feat: Transform objects

README.md

@@ -43,4 +43,4 @@ npm run format
 
 # Lint
 npm run lint
- ```
+```

package.json

@@ -17,6 +17,7 @@
     "format": "prettier --write 'src/**/*.ts'",
     "format:check": "prettier --check 'src/**/*.ts'",
     "lint": "eslint --max-warnings 0 --ext .ts src",
+    "lint:fix": "eslint --max-warnings 0 --ext .ts --fix src",
     "test": "ava"
   },
   "description": "This is the high-level package to use for developing CloudQuery plugins in JavaScript",

src/transformers/openapi.test.ts

@@ -52,32 +52,26 @@ test('should parse spec as expected', (t) => {
     createColumn({
       name: 'string',
       type: new Utf8(),
-      description: '',
     }),
     createColumn({
       name: 'number',
       type: new Int64(),
-      description: '',
     }),
     createColumn({
       name: 'integer',
       type: new Int64(),
-      description: '',
     }),
     createColumn({
       name: 'boolean',
       type: new Bool(),
-      description: '',
     }),
     createColumn({
       name: 'object',
       type: new JSONType(),
-      description: '',
     }),
     createColumn({
       name: 'array',
       type: new JSONType(),
-      description: '',
     }),
   ];
 

src/transformers/transform.test.ts

@@ -0,0 +1,176 @@
+import { Utf8, Int64, Bool, List, Field, Float64, DataType } from '@apache-arrow/esnext-esm';
+import test from 'ava';
+
+import { Column, createColumn } from '../schema/column.js';
+import { JSONType } from '../types/json.js';
+
+import { objectToColumns } from './transform.js';
+
+test('should parse object as expected', (t) => {
+  const expectedColumns: Column[] = [
+    createColumn({
+      name: 'string',
+      type: new Utf8(),
+    }),
+    createColumn({
+      name: 'number',
+      type: new Int64(),
+    }),
+    createColumn({
+      name: 'float',
+      type: new Float64(),
+    }),
+    createColumn({
+      name: 'boolean',
+      type: new Bool(),
+    }),
+    createColumn({
+      name: 'object',
+      type: new JSONType(),
+    }),
+    createColumn({
+      name: 'array',
+      type: new List(new Field('element', new Utf8())),
+    }),
+  ];
+
+  const columns = objectToColumns({
+    string: 'test',
+    number: 1,
+    float: 3.14,
+    boolean: false,
+    object: { inner: 'foo' },
+    array: ['foo', 'bar'],
+  });
+  t.deepEqual(columns, expectedColumns);
+});
+
+test('should parse object with custom types', (t) => {
+  const expectedColumns: Column[] = [
+    createColumn({
+      name: 'string',
+      type: new Utf8(),
+    }),
+    createColumn({
+      name: 'float',
+      type: new Float64(),
+    }),
+  ];
+
+  const columns = objectToColumns(
+    {
+      string: 'test',
+      float: 1,
+    },
+    {
+      getTypeFromValue: function (key: string, value: unknown): DataType | null | undefined {
+        if (key === 'float') return new Float64();
+        return undefined;
+      },
+    },
+  );
+  t.deepEqual(columns, expectedColumns);
+});
+
+test('should parse object with custom types and allow skip columns in type transformer', (t) => {
+  const expectedColumns: Column[] = [
+    createColumn({
+      name: 'string',
+      type: new Utf8(),
+    }),
+  ];
+
+  const columns = objectToColumns(
+    {
+      string: 'test',
+      skip: 'test',
+    },
+    {
+      getTypeFromValue: function (key: string, value: unknown): DataType | null | undefined {
+        return key === 'skip' ? null : undefined;
+      },
+    },
+  );
+  t.deepEqual(columns, expectedColumns);
+});
+
+test('should parse object and skip columns', (t) => {
+  const expectedColumns: Column[] = [
+    createColumn({
+      name: 'string',
+      type: new Utf8(),
+    }),
+  ];
+
+  const columns = objectToColumns(
+    {
+      string: 'test',
+      skip: 'test',
+    },
+    {
+      skipColumns: ['skip'],
+    },
+  );
+  t.deepEqual(columns, expectedColumns);
+});
+
+test('should parse object and set PKs', (t) => {
+  const expectedColumns: Column[] = [
+    createColumn({
+      name: 'id',
+      type: new Utf8(),
+      primaryKey: true,
+    }),
+    createColumn({
+      name: 'string',
+      type: new Utf8(),
+    }),
+  ];
+
+  const columns = objectToColumns(
+    {
+      id: 'the-id',
+      string: 'test',
+    },
+    {
+      primaryKeys: ['id'],
+    },
+  );
+  t.deepEqual(columns, expectedColumns);
+});
+
+test('should allow direct overrides', (t) => {
+  const expectedColumns: Column[] = [
+    createColumn({
+      name: 'id',
+      type: new Utf8(),
+      notNull: true,
+      unique: true,
+    }),
+    createColumn({
+      name: 'string',
+      type: new Utf8(),
+    }),
+  ];
+
+  const columns = objectToColumns(
+    {
+      id: 'the-id',
+      string: 'test',
+    },
+    {
+      overrides: new Map<string, Column>([
+        [
+          'id',
+          createColumn({
+            name: 'id',
+            type: new Utf8(),
+            notNull: true,
+            unique: true,
+          }),
+        ],
+      ]),
+    },
+  );
+  t.deepEqual(columns, expectedColumns);
+});

src/transformers/transform.ts

@@ -0,0 +1,71 @@
+import { DataType, Field, Utf8, Int64, Float64, Bool, List } from '@apache-arrow/esnext-esm';
+
+import { Column, createColumn } from '../schema/column.js';
+import { JSONType } from '../types/json.js';
+
+function defaultGetTypeFromValue(key: string, value: unknown): DataType | null {
+  switch (typeof value) {
+    case 'string': {
+      return new Utf8();
+    }
+    case 'number': {
+      return value % 1 === 0 ? new Int64() : new Float64();
+    }
+    case 'boolean': {
+      return new Bool();
+    }
+    case 'object': {
+      if (Array.isArray(value)) {
+        if (value.length === 0) return new JSONType(); // Empty array, can't infer type
+
+        // Assuming array of same type, getting type of first element
+        const elementType = defaultGetTypeFromValue(key + '.element', value[0]);
+        if (elementType === null) return new JSONType();
+
+        const field = new Field('element', elementType); // 'element' can be any name as it's just for internal representation
+        return new List(field);
+      } else {
+        return new JSONType();
+      }
+    }
+    default: {
+      throw new Error(`Unsupported type: ${typeof value}`);
+    }
+  }
+}
+
+type Options = {
+  skipColumns?: string[];
+  primaryKeys?: string[];
+  getTypeFromValue?: (key: string, value: unknown) => DataType | null | undefined;
+  overrides?: Map<string, Column>;
+};
+
+export function objectToColumns(
+  object: Record<string, unknown>,
+  {
+    skipColumns = [],
+    primaryKeys = [],
+    getTypeFromValue = defaultGetTypeFromValue,
+    overrides = new Map(),
+  }: Options = {},
+): Column[] {
+  return Object.entries(object)
+    .filter(([key]) => !skipColumns.includes(key))
+    .map(([key, value]): Column | null => {
+      if (overrides.has(key)) {
+        return overrides.get(key)!;
+      }
+
+      let type = getTypeFromValue(key, value);
+      if (type === undefined && getTypeFromValue !== defaultGetTypeFromValue) {
+        type = defaultGetTypeFromValue(key, value);
+      }
+      if (type === null || type === undefined) return null;
+
+      const isPrimaryKey = primaryKeys.includes(key);
+
+      return createColumn({ name: key, type, primaryKey: isPrimaryKey });
+    })
+    .filter((column): column is Column => column !== null); // This is a type-guard
+}