diff --git a/README.md b/README.md index b96e508..197feb9 100644 --- a/README.md +++ b/README.md @@ -43,4 +43,4 @@ npm run format # Lint npm run lint - ``` +``` diff --git a/package.json b/package.json index 2af66dd..e97e9ed 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "format": "prettier --write 'src/**/*.ts'", "format:check": "prettier --check 'src/**/*.ts'", "lint": "eslint --max-warnings 0 --ext .ts src", + "lint:fix": "eslint --max-warnings 0 --ext .ts --fix src", "test": "ava" }, "description": "This is the high-level package to use for developing CloudQuery plugins in JavaScript", diff --git a/src/transformers/openapi.test.ts b/src/transformers/openapi.test.ts index 167b3bc..ac2aa77 100644 --- a/src/transformers/openapi.test.ts +++ b/src/transformers/openapi.test.ts @@ -52,32 +52,26 @@ test('should parse spec as expected', (t) => { createColumn({ name: 'string', type: new Utf8(), - description: '', }), createColumn({ name: 'number', type: new Int64(), - description: '', }), createColumn({ name: 'integer', type: new Int64(), - description: '', }), createColumn({ name: 'boolean', type: new Bool(), - description: '', }), createColumn({ name: 'object', type: new JSONType(), - description: '', }), createColumn({ name: 'array', type: new JSONType(), - description: '', }), ]; diff --git a/src/transformers/transform.test.ts b/src/transformers/transform.test.ts new file mode 100644 index 0000000..0e96c21 --- /dev/null +++ b/src/transformers/transform.test.ts @@ -0,0 +1,176 @@ +import { Utf8, Int64, Bool, List, Field, Float64, DataType } from '@apache-arrow/esnext-esm'; +import test from 'ava'; + +import { Column, createColumn } from '../schema/column.js'; +import { JSONType } from '../types/json.js'; + +import { objectToColumns } from './transform.js'; + +test('should parse object as expected', (t) => { + const expectedColumns: Column[] = [ + createColumn({ + name: 'string', + type: new Utf8(), + }), + createColumn({ + name: 'number', + type: new Int64(), + }), + createColumn({ + name: 'float', + type: new Float64(), + }), + createColumn({ + name: 'boolean', + type: new Bool(), + }), + createColumn({ + name: 'object', + type: new JSONType(), + }), + createColumn({ + name: 'array', + type: new List(new Field('element', new Utf8())), + }), + ]; + + const columns = objectToColumns({ + string: 'test', + number: 1, + float: 3.14, + boolean: false, + object: { inner: 'foo' }, + array: ['foo', 'bar'], + }); + t.deepEqual(columns, expectedColumns); +}); + +test('should parse object with custom types', (t) => { + const expectedColumns: Column[] = [ + createColumn({ + name: 'string', + type: new Utf8(), + }), + createColumn({ + name: 'float', + type: new Float64(), + }), + ]; + + const columns = objectToColumns( + { + string: 'test', + float: 1, + }, + { + getTypeFromValue: function (key: string, value: unknown): DataType | null | undefined { + if (key === 'float') return new Float64(); + return undefined; + }, + }, + ); + t.deepEqual(columns, expectedColumns); +}); + +test('should parse object with custom types and allow skip columns in type transformer', (t) => { + const expectedColumns: Column[] = [ + createColumn({ + name: 'string', + type: new Utf8(), + }), + ]; + + const columns = objectToColumns( + { + string: 'test', + skip: 'test', + }, + { + getTypeFromValue: function (key: string, value: unknown): DataType | null | undefined { + return key === 'skip' ? null : undefined; + }, + }, + ); + t.deepEqual(columns, expectedColumns); +}); + +test('should parse object and skip columns', (t) => { + const expectedColumns: Column[] = [ + createColumn({ + name: 'string', + type: new Utf8(), + }), + ]; + + const columns = objectToColumns( + { + string: 'test', + skip: 'test', + }, + { + skipColumns: ['skip'], + }, + ); + t.deepEqual(columns, expectedColumns); +}); + +test('should parse object and set PKs', (t) => { + const expectedColumns: Column[] = [ + createColumn({ + name: 'id', + type: new Utf8(), + primaryKey: true, + }), + createColumn({ + name: 'string', + type: new Utf8(), + }), + ]; + + const columns = objectToColumns( + { + id: 'the-id', + string: 'test', + }, + { + primaryKeys: ['id'], + }, + ); + t.deepEqual(columns, expectedColumns); +}); + +test('should allow direct overrides', (t) => { + const expectedColumns: Column[] = [ + createColumn({ + name: 'id', + type: new Utf8(), + notNull: true, + unique: true, + }), + createColumn({ + name: 'string', + type: new Utf8(), + }), + ]; + + const columns = objectToColumns( + { + id: 'the-id', + string: 'test', + }, + { + overrides: new Map([ + [ + 'id', + createColumn({ + name: 'id', + type: new Utf8(), + notNull: true, + unique: true, + }), + ], + ]), + }, + ); + t.deepEqual(columns, expectedColumns); +}); diff --git a/src/transformers/transform.ts b/src/transformers/transform.ts new file mode 100644 index 0000000..aa29665 --- /dev/null +++ b/src/transformers/transform.ts @@ -0,0 +1,71 @@ +import { DataType, Field, Utf8, Int64, Float64, Bool, List } from '@apache-arrow/esnext-esm'; + +import { Column, createColumn } from '../schema/column.js'; +import { JSONType } from '../types/json.js'; + +function defaultGetTypeFromValue(key: string, value: unknown): DataType | null { + switch (typeof value) { + case 'string': { + return new Utf8(); + } + case 'number': { + return value % 1 === 0 ? new Int64() : new Float64(); + } + case 'boolean': { + return new Bool(); + } + case 'object': { + if (Array.isArray(value)) { + if (value.length === 0) return new JSONType(); // Empty array, can't infer type + + // Assuming array of same type, getting type of first element + const elementType = defaultGetTypeFromValue(key + '.element', value[0]); + if (elementType === null) return new JSONType(); + + const field = new Field('element', elementType); // 'element' can be any name as it's just for internal representation + return new List(field); + } else { + return new JSONType(); + } + } + default: { + throw new Error(`Unsupported type: ${typeof value}`); + } + } +} + +type Options = { + skipColumns?: string[]; + primaryKeys?: string[]; + getTypeFromValue?: (key: string, value: unknown) => DataType | null | undefined; + overrides?: Map; +}; + +export function objectToColumns( + object: Record, + { + skipColumns = [], + primaryKeys = [], + getTypeFromValue = defaultGetTypeFromValue, + overrides = new Map(), + }: Options = {}, +): Column[] { + return Object.entries(object) + .filter(([key]) => !skipColumns.includes(key)) + .map(([key, value]): Column | null => { + if (overrides.has(key)) { + return overrides.get(key)!; + } + + let type = getTypeFromValue(key, value); + if (type === undefined && getTypeFromValue !== defaultGetTypeFromValue) { + type = defaultGetTypeFromValue(key, value); + } + if (type === null || type === undefined) return null; + + const isPrimaryKey = primaryKeys.includes(key); + + return createColumn({ name: key, type, primaryKey: isPrimaryKey }); + }) + .filter((column): column is Column => column !== null); // This is a type-guard +}