From 6fa8358bbe4275152d0310113ea9aa2005c87d5a Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Aug 2023 14:40:26 +0100 Subject: [PATCH 1/5] feat: Transformers --- .gitignore | 3 ++ src/schema/arrow.ts | 9 ++++ src/schema/column.ts | 78 ++++++++++++++++++++++++++++++++ src/transformers/openapi.test.ts | 60 ++++++++++++++++++++++++ src/transformers/openapi.ts | 56 +++++++++++++++++++++++ src/types/json.ts | 21 +++++++++ src/types/uuid.ts | 22 +++++++++ 7 files changed, 249 insertions(+) create mode 100644 src/schema/arrow.ts create mode 100644 src/schema/column.ts create mode 100644 src/transformers/openapi.test.ts create mode 100644 src/transformers/openapi.ts create mode 100644 src/types/json.ts create mode 100644 src/types/uuid.ts diff --git a/.gitignore b/.gitignore index c6bba59..25eac54 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,6 @@ dist .yarn/build-state.yml .yarn/install-state.gz .pnp.* + +# Editor settings +.idea diff --git a/src/schema/arrow.ts b/src/schema/arrow.ts new file mode 100644 index 0000000..13f8e63 --- /dev/null +++ b/src/schema/arrow.ts @@ -0,0 +1,9 @@ +export const METADATA_UNIQUE = "cq:extension:unique" +export const METADATA_PRIMARY_KEY = "cq:extension:primary_key" +export const METADATA_CONSTRAINT_NAME = "cq:extension:constraint_name" +export const METADATA_INCREMENTAL = "cq:extension:incremental" + +export const METADATA_TRUE = "true" +export const METADATA_FALSE = "false" +export const METADATA_TABLE_NAME = "cq:table_name" +export const METADATA_TABLE_DESCRIPTION = "cq:table_description" diff --git a/src/schema/column.ts b/src/schema/column.ts new file mode 100644 index 0000000..25822ae --- /dev/null +++ b/src/schema/column.ts @@ -0,0 +1,78 @@ +import { DataType, Field } from '@apache-arrow/esnext-esm'; +import * as arrow from './arrow.js' + +export class Column { + name: string; + type: DataType; + description: string; + primary_key: boolean; + not_null: boolean; + incremental_key: boolean; + unique: boolean; + + constructor( + name: string, + type: DataType, + description: string = "", + primary_key: boolean = false, + not_null: boolean = false, + incremental_key: boolean = false, + unique: boolean = false + ) { + this.name = name; + this.type = type; + this.description = description; + this.primary_key = primary_key; + this.not_null = not_null; + this.incremental_key = incremental_key; + this.unique = unique; + } + + toString(): string { + return `Column(name=${this.name}, type=${this.type}, description=${this.description}, primary_key=${this.primary_key}, not_null=${this.not_null}, incremental_key=${this.incremental_key}, unique=${this.unique})`; + } + + // JavaScript (and TypeScript) uses a single method for both string representation and debugging output + toJSON(): string { + return this.toString(); + } + + equals(value: any): boolean { + if (value instanceof Column) { + return this.name === value.name && + this.type === value.type && + this.description === value.description && + this.primary_key === value.primary_key && + this.not_null === value.not_null && + this.incremental_key === value.incremental_key && + this.unique === value.unique; + } + return false; + } + + toArrowField(): Field { + const metadataMap = new Map(); + metadataMap.set(arrow.METADATA_PRIMARY_KEY, this.primary_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); + metadataMap.set(arrow.METADATA_UNIQUE, this.unique ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); + metadataMap.set(arrow.METADATA_INCREMENTAL, this.incremental_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); + + return new Field(this.name, this.type, /*nullable=*/!this.not_null, metadataMap); + } + + static fromArrowField(field: Field): Column { + const metadata = field.metadata; + const primary_key = metadata.get(arrow.METADATA_PRIMARY_KEY) === arrow.METADATA_TRUE; + const unique = metadata.get(arrow.METADATA_UNIQUE) === arrow.METADATA_TRUE; + const incremental_key = metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE; + + return new Column( + field.name, + field.type, + "", + primary_key, + !field.nullable, + unique, + incremental_key + ); + } +} diff --git a/src/transformers/openapi.test.ts b/src/transformers/openapi.test.ts new file mode 100644 index 0000000..84a10c7 --- /dev/null +++ b/src/transformers/openapi.test.ts @@ -0,0 +1,60 @@ +import test from 'ava'; + +import { oapiDefinitionToColumns } from './openapi.js'; +import { JSONType } from '../types/json.js'; +import { Column } from '../schema/column.js'; +import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; + +const OAPI_SPEC = { + "swagger": "2.0", + "info": { + "version": "2.0", + "title": "Test API", + "description": "Unit tests APIs", + }, + "host": "cloudquery.io", + "schemes": ["https"], + "consumes": ["application/json"], + "produces": ["application/json"], + "paths": {}, + "definitions": { + "TestDefinition": { + "type": "object", + "properties": { + "string": { + "type": "string", + }, + "number": { + "type": "number", + }, + "integer": { + "type": "integer", + }, + "boolean": { + "type": "boolean", + }, + "object": { + "$ref": "#/definitions/SomeDefinition", + }, + "array": { + "type": "array", + "items": {"$ref": "#/definitions/SomeDefinition"}, + }, + }, + }, + }, + } + +test('should parse spec as expected', (t) => { + const expectedColumns: Column[] = [ + new Column("string", new Utf8, ""), + new Column("number", new Int64, ""), + new Column("integer", new Int64, ""), + new Column("boolean", new Bool, ""), + new Column("object", new JSONType, ""), + new Column("array", new JSONType, ""), + ]; + + let columns = oapiDefinitionToColumns(OAPI_SPEC["definitions"]["TestDefinition"]) + t.deepEqual(columns, expectedColumns); +}); diff --git a/src/transformers/openapi.ts b/src/transformers/openapi.ts new file mode 100644 index 0000000..62bd5b6 --- /dev/null +++ b/src/transformers/openapi.ts @@ -0,0 +1,56 @@ +import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; +import { JSONType } from '../types/json.js'; +import { Column } from '../schema/column.js'; + +interface Field { + type?: string; + [key: string]: any; +} + +function oapiTypeToArrowType(field: Field): DataType { + const oapiType = field.type; + switch (oapiType) { + case "string": + return new Utf8; + case "number": + case "integer": + return new Int64; + case "boolean": + return new Bool; + case "array": + case "object": + return new JSONType; + default: + if (!oapiType && "$ref" in field) { + return new JSONType; + } else { + return new Utf8; + } + } +} + +export function getColumnByName(columns: Column[], name: string): Column | null { + for (let column of columns) { + if (column.name === name) { + return column; + } + } + return null; +} + +export function oapiDefinitionToColumns(definition: any, overrideColumns: Column[] = []): Column[] { + let columns: Column[] = []; + for (let key in definition.properties) { + const value = definition.properties[key]; + const columnType = oapiTypeToArrowType(value); + let column = new Column(key, columnType, value.description); + const overrideColumn = getColumnByName(overrideColumns, key); + if (overrideColumn) { + column.type = overrideColumn.type; + column.primary_key = overrideColumn.primary_key; + column.unique = overrideColumn.unique; + } + columns.push(column); + } + return columns; +} diff --git a/src/types/json.ts b/src/types/json.ts new file mode 100644 index 0000000..076e0f8 --- /dev/null +++ b/src/types/json.ts @@ -0,0 +1,21 @@ +import { DataType, Binary } from '@apache-arrow/esnext-esm'; + +export class JSONType extends DataType { + readonly extensionName: string = "json"; + + constructor() { + super(); + // Assuming there's no direct way to set the storage type in the constructor, + // this is just a representation of the JSONType. + } + + serialize(): ArrayBuffer { + // Implement your serialization logic here. + return new TextEncoder().encode("json-serialized").buffer; + } + + static deserialize(storageType: Binary, serialized: ArrayBuffer): JSONType { + // Implement your deserialization logic here. + return new JSONType(); + } +} diff --git a/src/types/uuid.ts b/src/types/uuid.ts new file mode 100644 index 0000000..99b3fab --- /dev/null +++ b/src/types/uuid.ts @@ -0,0 +1,22 @@ +import { DataType, Binary } from '@apache-arrow/esnext-esm'; + +export class UUIDType extends DataType { + readonly extensionName: string = "uuid"; + + constructor() { + super(); + // The underlying storage type is a binary of 16 bytes, representing a UUID. + // Assuming there's no direct way to set the storage type in the constructor, + // this is just a representation of the UUIDType. + } + + serialize(): ArrayBuffer { + // Implement your serialization logic here. + return new TextEncoder().encode("uuid-serialized").buffer; + } + + static deserialize(storageType: Binary, serialized: ArrayBuffer): UUIDType { + // Implement your deserialization logic here. + return new UUIDType(); + } +} From 8747caa3994d38398360c41dc34e104b07209629 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Aug 2023 14:42:42 +0100 Subject: [PATCH 2/5] fmt --- src/schema/arrow.ts | 16 ++-- src/schema/column.ts | 124 +++++++++++++++---------------- src/transformers/openapi.test.ts | 92 +++++++++++------------ src/transformers/openapi.ts | 78 +++++++++---------- src/types/json.ts | 28 +++---- src/types/uuid.ts | 30 ++++---- 6 files changed, 181 insertions(+), 187 deletions(-) diff --git a/src/schema/arrow.ts b/src/schema/arrow.ts index 13f8e63..c36b03a 100644 --- a/src/schema/arrow.ts +++ b/src/schema/arrow.ts @@ -1,9 +1,9 @@ -export const METADATA_UNIQUE = "cq:extension:unique" -export const METADATA_PRIMARY_KEY = "cq:extension:primary_key" -export const METADATA_CONSTRAINT_NAME = "cq:extension:constraint_name" -export const METADATA_INCREMENTAL = "cq:extension:incremental" +export const METADATA_UNIQUE = 'cq:extension:unique'; +export const METADATA_PRIMARY_KEY = 'cq:extension:primary_key'; +export const METADATA_CONSTRAINT_NAME = 'cq:extension:constraint_name'; +export const METADATA_INCREMENTAL = 'cq:extension:incremental'; -export const METADATA_TRUE = "true" -export const METADATA_FALSE = "false" -export const METADATA_TABLE_NAME = "cq:table_name" -export const METADATA_TABLE_DESCRIPTION = "cq:table_description" +export const METADATA_TRUE = 'true'; +export const METADATA_FALSE = 'false'; +export const METADATA_TABLE_NAME = 'cq:table_name'; +export const METADATA_TABLE_DESCRIPTION = 'cq:table_description'; diff --git a/src/schema/column.ts b/src/schema/column.ts index 25822ae..97e689f 100644 --- a/src/schema/column.ts +++ b/src/schema/column.ts @@ -1,78 +1,72 @@ import { DataType, Field } from '@apache-arrow/esnext-esm'; -import * as arrow from './arrow.js' +import * as arrow from './arrow.js'; export class Column { - name: string; - type: DataType; - description: string; - primary_key: boolean; - not_null: boolean; - incremental_key: boolean; - unique: boolean; + name: string; + type: DataType; + description: string; + primary_key: boolean; + not_null: boolean; + incremental_key: boolean; + unique: boolean; - constructor( - name: string, - type: DataType, - description: string = "", - primary_key: boolean = false, - not_null: boolean = false, - incremental_key: boolean = false, - unique: boolean = false - ) { - this.name = name; - this.type = type; - this.description = description; - this.primary_key = primary_key; - this.not_null = not_null; - this.incremental_key = incremental_key; - this.unique = unique; - } + constructor( + name: string, + type: DataType, + description: string = '', + primary_key: boolean = false, + not_null: boolean = false, + incremental_key: boolean = false, + unique: boolean = false, + ) { + this.name = name; + this.type = type; + this.description = description; + this.primary_key = primary_key; + this.not_null = not_null; + this.incremental_key = incremental_key; + this.unique = unique; + } - toString(): string { - return `Column(name=${this.name}, type=${this.type}, description=${this.description}, primary_key=${this.primary_key}, not_null=${this.not_null}, incremental_key=${this.incremental_key}, unique=${this.unique})`; - } + toString(): string { + return `Column(name=${this.name}, type=${this.type}, description=${this.description}, primary_key=${this.primary_key}, not_null=${this.not_null}, incremental_key=${this.incremental_key}, unique=${this.unique})`; + } - // JavaScript (and TypeScript) uses a single method for both string representation and debugging output - toJSON(): string { - return this.toString(); - } + // JavaScript (and TypeScript) uses a single method for both string representation and debugging output + toJSON(): string { + return this.toString(); + } - equals(value: any): boolean { - if (value instanceof Column) { - return this.name === value.name && - this.type === value.type && - this.description === value.description && - this.primary_key === value.primary_key && - this.not_null === value.not_null && - this.incremental_key === value.incremental_key && - this.unique === value.unique; - } - return false; + equals(value: any): boolean { + if (value instanceof Column) { + return ( + this.name === value.name && + this.type === value.type && + this.description === value.description && + this.primary_key === value.primary_key && + this.not_null === value.not_null && + this.incremental_key === value.incremental_key && + this.unique === value.unique + ); } + return false; + } - toArrowField(): Field { - const metadataMap = new Map(); - metadataMap.set(arrow.METADATA_PRIMARY_KEY, this.primary_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); - metadataMap.set(arrow.METADATA_UNIQUE, this.unique ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); - metadataMap.set(arrow.METADATA_INCREMENTAL, this.incremental_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); + toArrowField(): Field { + const metadataMap = new Map(); + metadataMap.set(arrow.METADATA_PRIMARY_KEY, this.primary_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); + metadataMap.set(arrow.METADATA_UNIQUE, this.unique ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); + metadataMap.set(arrow.METADATA_INCREMENTAL, this.incremental_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE); - return new Field(this.name, this.type, /*nullable=*/!this.not_null, metadataMap); - } + return new Field(this.name, this.type, /*nullable=*/ !this.not_null, metadataMap); + } - static fromArrowField(field: Field): Column { - const metadata = field.metadata; - const primary_key = metadata.get(arrow.METADATA_PRIMARY_KEY) === arrow.METADATA_TRUE; - const unique = metadata.get(arrow.METADATA_UNIQUE) === arrow.METADATA_TRUE; - const incremental_key = metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE; + static fromArrowField(field: Field): Column { + const metadata = field.metadata; + const primary_key = metadata.get(arrow.METADATA_PRIMARY_KEY) === arrow.METADATA_TRUE; + const unique = metadata.get(arrow.METADATA_UNIQUE) === arrow.METADATA_TRUE; + const incremental_key = metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE; - return new Column( - field.name, - field.type, - "", - primary_key, - !field.nullable, - unique, - incremental_key - ); - } + return new Column(field.name, field.type, '', primary_key, !field.nullable, unique, incremental_key); + } } diff --git a/src/transformers/openapi.test.ts b/src/transformers/openapi.test.ts index 84a10c7..f5fef00 100644 --- a/src/transformers/openapi.test.ts +++ b/src/transformers/openapi.test.ts @@ -6,55 +6,55 @@ import { Column } from '../schema/column.js'; import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; const OAPI_SPEC = { - "swagger": "2.0", - "info": { - "version": "2.0", - "title": "Test API", - "description": "Unit tests APIs", + swagger: '2.0', + info: { + version: '2.0', + title: 'Test API', + description: 'Unit tests APIs', + }, + host: 'cloudquery.io', + schemes: ['https'], + consumes: ['application/json'], + produces: ['application/json'], + paths: {}, + definitions: { + TestDefinition: { + type: 'object', + properties: { + string: { + type: 'string', }, - "host": "cloudquery.io", - "schemes": ["https"], - "consumes": ["application/json"], - "produces": ["application/json"], - "paths": {}, - "definitions": { - "TestDefinition": { - "type": "object", - "properties": { - "string": { - "type": "string", - }, - "number": { - "type": "number", - }, - "integer": { - "type": "integer", - }, - "boolean": { - "type": "boolean", - }, - "object": { - "$ref": "#/definitions/SomeDefinition", - }, - "array": { - "type": "array", - "items": {"$ref": "#/definitions/SomeDefinition"}, - }, - }, - }, + number: { + type: 'number', }, - } + integer: { + type: 'integer', + }, + boolean: { + type: 'boolean', + }, + object: { + $ref: '#/definitions/SomeDefinition', + }, + array: { + type: 'array', + items: { $ref: '#/definitions/SomeDefinition' }, + }, + }, + }, + }, +}; test('should parse spec as expected', (t) => { - const expectedColumns: Column[] = [ - new Column("string", new Utf8, ""), - new Column("number", new Int64, ""), - new Column("integer", new Int64, ""), - new Column("boolean", new Bool, ""), - new Column("object", new JSONType, ""), - new Column("array", new JSONType, ""), - ]; + const expectedColumns: Column[] = [ + new Column('string', new Utf8(), ''), + new Column('number', new Int64(), ''), + new Column('integer', new Int64(), ''), + new Column('boolean', new Bool(), ''), + new Column('object', new JSONType(), ''), + new Column('array', new JSONType(), ''), + ]; - let columns = oapiDefinitionToColumns(OAPI_SPEC["definitions"]["TestDefinition"]) - t.deepEqual(columns, expectedColumns); + let columns = oapiDefinitionToColumns(OAPI_SPEC['definitions']['TestDefinition']); + t.deepEqual(columns, expectedColumns); }); diff --git a/src/transformers/openapi.ts b/src/transformers/openapi.ts index 62bd5b6..e2a7665 100644 --- a/src/transformers/openapi.ts +++ b/src/transformers/openapi.ts @@ -3,54 +3,54 @@ import { JSONType } from '../types/json.js'; import { Column } from '../schema/column.js'; interface Field { - type?: string; - [key: string]: any; + type?: string; + [key: string]: any; } function oapiTypeToArrowType(field: Field): DataType { - const oapiType = field.type; - switch (oapiType) { - case "string": - return new Utf8; - case "number": - case "integer": - return new Int64; - case "boolean": - return new Bool; - case "array": - case "object": - return new JSONType; - default: - if (!oapiType && "$ref" in field) { - return new JSONType; - } else { - return new Utf8; - } - } + const oapiType = field.type; + switch (oapiType) { + case 'string': + return new Utf8(); + case 'number': + case 'integer': + return new Int64(); + case 'boolean': + return new Bool(); + case 'array': + case 'object': + return new JSONType(); + default: + if (!oapiType && '$ref' in field) { + return new JSONType(); + } else { + return new Utf8(); + } + } } export function getColumnByName(columns: Column[], name: string): Column | null { - for (let column of columns) { - if (column.name === name) { - return column; - } + for (let column of columns) { + if (column.name === name) { + return column; } - return null; + } + return null; } export function oapiDefinitionToColumns(definition: any, overrideColumns: Column[] = []): Column[] { - let columns: Column[] = []; - for (let key in definition.properties) { - const value = definition.properties[key]; - const columnType = oapiTypeToArrowType(value); - let column = new Column(key, columnType, value.description); - const overrideColumn = getColumnByName(overrideColumns, key); - if (overrideColumn) { - column.type = overrideColumn.type; - column.primary_key = overrideColumn.primary_key; - column.unique = overrideColumn.unique; - } - columns.push(column); + let columns: Column[] = []; + for (let key in definition.properties) { + const value = definition.properties[key]; + const columnType = oapiTypeToArrowType(value); + let column = new Column(key, columnType, value.description); + const overrideColumn = getColumnByName(overrideColumns, key); + if (overrideColumn) { + column.type = overrideColumn.type; + column.primary_key = overrideColumn.primary_key; + column.unique = overrideColumn.unique; } - return columns; + columns.push(column); + } + return columns; } diff --git a/src/types/json.ts b/src/types/json.ts index 076e0f8..cc7b613 100644 --- a/src/types/json.ts +++ b/src/types/json.ts @@ -1,21 +1,21 @@ import { DataType, Binary } from '@apache-arrow/esnext-esm'; export class JSONType extends DataType { - readonly extensionName: string = "json"; + readonly extensionName: string = 'json'; - constructor() { - super(); - // Assuming there's no direct way to set the storage type in the constructor, - // this is just a representation of the JSONType. - } + constructor() { + super(); + // Assuming there's no direct way to set the storage type in the constructor, + // this is just a representation of the JSONType. + } - serialize(): ArrayBuffer { - // Implement your serialization logic here. - return new TextEncoder().encode("json-serialized").buffer; - } + serialize(): ArrayBuffer { + // Implement your serialization logic here. + return new TextEncoder().encode('json-serialized').buffer; + } - static deserialize(storageType: Binary, serialized: ArrayBuffer): JSONType { - // Implement your deserialization logic here. - return new JSONType(); - } + static deserialize(storageType: Binary, serialized: ArrayBuffer): JSONType { + // Implement your deserialization logic here. + return new JSONType(); + } } diff --git a/src/types/uuid.ts b/src/types/uuid.ts index 99b3fab..4bd5c15 100644 --- a/src/types/uuid.ts +++ b/src/types/uuid.ts @@ -1,22 +1,22 @@ import { DataType, Binary } from '@apache-arrow/esnext-esm'; export class UUIDType extends DataType { - readonly extensionName: string = "uuid"; + readonly extensionName: string = 'uuid'; - constructor() { - super(); - // The underlying storage type is a binary of 16 bytes, representing a UUID. - // Assuming there's no direct way to set the storage type in the constructor, - // this is just a representation of the UUIDType. - } + constructor() { + super(); + // The underlying storage type is a binary of 16 bytes, representing a UUID. + // Assuming there's no direct way to set the storage type in the constructor, + // this is just a representation of the UUIDType. + } - serialize(): ArrayBuffer { - // Implement your serialization logic here. - return new TextEncoder().encode("uuid-serialized").buffer; - } + serialize(): ArrayBuffer { + // Implement your serialization logic here. + return new TextEncoder().encode('uuid-serialized').buffer; + } - static deserialize(storageType: Binary, serialized: ArrayBuffer): UUIDType { - // Implement your deserialization logic here. - return new UUIDType(); - } + static deserialize(storageType: Binary, serialized: ArrayBuffer): UUIDType { + // Implement your deserialization logic here. + return new UUIDType(); + } } From 3a9e202ba4e89758c71f0eb0c0ff647336ce49dd Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Aug 2023 14:59:38 +0100 Subject: [PATCH 3/5] clean up --- src/schema/column.ts | 3 +- src/transformers/openapi.test.ts | 9 ++--- src/transformers/openapi.ts | 56 +++++++++++++++++++------------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/src/schema/column.ts b/src/schema/column.ts index 97e689f..7bf6a4e 100644 --- a/src/schema/column.ts +++ b/src/schema/column.ts @@ -1,4 +1,5 @@ import { DataType, Field } from '@apache-arrow/esnext-esm'; + import * as arrow from './arrow.js'; export class Column { @@ -37,7 +38,7 @@ export class Column { return this.toString(); } - equals(value: any): boolean { + equals(value: object): boolean { if (value instanceof Column) { return ( this.name === value.name && diff --git a/src/transformers/openapi.test.ts b/src/transformers/openapi.test.ts index f5fef00..62937d5 100644 --- a/src/transformers/openapi.test.ts +++ b/src/transformers/openapi.test.ts @@ -1,9 +1,10 @@ +import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; import test from 'ava'; -import { oapiDefinitionToColumns } from './openapi.js'; -import { JSONType } from '../types/json.js'; import { Column } from '../schema/column.js'; -import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; +import { JSONType } from '../types/json.js'; + +import { oapiDefinitionToColumns } from './openapi.js'; const OAPI_SPEC = { swagger: '2.0', @@ -55,6 +56,6 @@ test('should parse spec as expected', (t) => { new Column('array', new JSONType(), ''), ]; - let columns = oapiDefinitionToColumns(OAPI_SPEC['definitions']['TestDefinition']); + const columns = oapiDefinitionToColumns(OAPI_SPEC['definitions']['TestDefinition']); t.deepEqual(columns, expectedColumns); }); diff --git a/src/transformers/openapi.ts b/src/transformers/openapi.ts index e2a7665..7f44071 100644 --- a/src/transformers/openapi.ts +++ b/src/transformers/openapi.ts @@ -1,49 +1,61 @@ -import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; -import { JSONType } from '../types/json.js'; +import { DataType, Field, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm'; + import { Column } from '../schema/column.js'; +import { JSONType } from '../types/json.js'; -interface Field { +interface OAPIProperty { type?: string; - [key: string]: any; + description?: string; + $ref?: string; + items?: { + $ref: string; + }; +} + +interface OAPIDefinition { + properties: { + [key: string]: OAPIProperty; + }; } -function oapiTypeToArrowType(field: Field): DataType { +function oapiTypeToArrowType(field: OAPIProperty): DataType { const oapiType = field.type; switch (oapiType) { - case 'string': + case 'string': { return new Utf8(); + } case 'number': - case 'integer': + case 'integer': { return new Int64(); - case 'boolean': + } + case 'boolean': { return new Bool(); + } case 'array': - case 'object': + case 'object': { return new JSONType(); - default: - if (!oapiType && '$ref' in field) { - return new JSONType(); - } else { - return new Utf8(); - } + } + default: { + return !oapiType && '$ref' in field ? new JSONType() : new Utf8(); + } } } -export function getColumnByName(columns: Column[], name: string): Column | null { - for (let column of columns) { +export function getColumnByName(columns: Column[], name: string): Column | undefined { + for (const column of columns) { if (column.name === name) { return column; } } - return null; + return undefined; } -export function oapiDefinitionToColumns(definition: any, overrideColumns: Column[] = []): Column[] { - let columns: Column[] = []; - for (let key in definition.properties) { +export function oapiDefinitionToColumns(definition: OAPIDefinition, overrideColumns: Column[] = []): Column[] { + const columns: Column[] = []; + for (const key in definition.properties) { const value = definition.properties[key]; const columnType = oapiTypeToArrowType(value); - let column = new Column(key, columnType, value.description); + const column = new Column(key, columnType, value.description); const overrideColumn = getColumnByName(overrideColumns, key); if (overrideColumn) { column.type = overrideColumn.type; From d5e58e13aa1efb247c085bd788d47be2e1ae0ad7 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Aug 2023 15:01:25 +0100 Subject: [PATCH 4/5] add linting section --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 72e33a0..e48ba67 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,13 @@ npm run build ```bash npm test ``` + +### Formatting and Linting + +```bash +# Format code. Omit -w to just check for formatting issues +prettier -w 'src/**/*.ts' + +# Lint +eslint --max-warnings 0 --ext .ts src + ``` From fd81fc854d0f54a18c7ef95d197ab00c5c35e7da Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 8 Aug 2023 15:15:49 +0100 Subject: [PATCH 5/5] use Datatype, fix readme to use npm --- README.md | 9 ++++++--- src/types/json.ts | 4 ++-- src/types/uuid.ts | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e48ba67..b96e508 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,12 @@ npm test ### Formatting and Linting ```bash -# Format code. Omit -w to just check for formatting issues -prettier -w 'src/**/*.ts' +# This is just to check if the code is formatted +npm run format:check + +# Automatically format code +npm run format # Lint -eslint --max-warnings 0 --ext .ts src +npm run lint ``` diff --git a/src/types/json.ts b/src/types/json.ts index cc7b613..3a001f9 100644 --- a/src/types/json.ts +++ b/src/types/json.ts @@ -1,6 +1,6 @@ -import { DataType, Binary } from '@apache-arrow/esnext-esm'; +import { DataType, Binary, Type } from '@apache-arrow/esnext-esm'; -export class JSONType extends DataType { +export class JSONType extends DataType { readonly extensionName: string = 'json'; constructor() { diff --git a/src/types/uuid.ts b/src/types/uuid.ts index 4bd5c15..923f10d 100644 --- a/src/types/uuid.ts +++ b/src/types/uuid.ts @@ -1,6 +1,6 @@ -import { DataType, Binary } from '@apache-arrow/esnext-esm'; +import { DataType, Binary, Type } from '@apache-arrow/esnext-esm'; -export class UUIDType extends DataType { +export class UUIDType extends DataType { readonly extensionName: string = 'uuid'; constructor() {