From 6c77a6905cd038f8c2a9d38a71922dbdd0381fbf Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 8 Aug 2023 19:12:31 +0200 Subject: [PATCH] feat: Add table types --- .eslintrc | 2 +- src/scalar/bool.ts | 7 +++--- src/scalar/scalar.ts | 11 +++++++++ src/schema/column.ts | 12 +++++++--- src/schema/meta.ts | 52 ++++++++++++++++++++++++++++++++++++++++++ src/schema/resource.ts | 43 ++++++++++++++++++++++++++++++++++ src/schema/table.ts | 45 ++++++++++++++++++++++++++++++++++++ src/schema/types.ts | 1 + 8 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 src/scalar/scalar.ts create mode 100644 src/schema/meta.ts create mode 100644 src/schema/resource.ts create mode 100644 src/schema/table.ts create mode 100644 src/schema/types.ts diff --git a/.eslintrc b/.eslintrc index c8a8882..87f12c7 100644 --- a/.eslintrc +++ b/.eslintrc @@ -19,6 +19,7 @@ "plugin:you-dont-need-lodash-underscore/all" ], "rules": { + "unicorn/no-null": 0, "unused-imports/no-unused-imports": "error", "no-console": "error", "@typescript-eslint/no-unused-vars": 0, @@ -40,7 +41,6 @@ { "files": ["src/grpc/**/*.ts"], "rules": { - "unicorn/no-null": 0, "@typescript-eslint/naming-convention": 0 } }, diff --git a/src/scalar/bool.ts b/src/scalar/bool.ts index 4afacac..a0a01fe 100644 --- a/src/scalar/bool.ts +++ b/src/scalar/bool.ts @@ -1,14 +1,15 @@ import { Bool as ArrowBool } from '@apache-arrow/esnext-esm'; import { boolean, isBooleanable } from 'boolean'; +import { Scalar } from './scalar.js'; import { isInvalid, NULL_VALUE } from './util.js'; -export class Bool { +export class Bool implements Scalar { private _valid = false; private _value = false; public constructor(v: unknown) { - this.valid = v; + this.value = v; return this; } @@ -24,7 +25,7 @@ export class Bool { return this._value; } - public set valid(value: unknown) { + public set value(value: unknown) { if (isInvalid(value)) { this._valid = false; return; diff --git a/src/scalar/scalar.ts b/src/scalar/scalar.ts new file mode 100644 index 0000000..3789be6 --- /dev/null +++ b/src/scalar/scalar.ts @@ -0,0 +1,11 @@ +import { DataType } from '@apache-arrow/esnext-esm'; + +export interface Scalar { + toString: () => string; + get valid(): boolean; + get value(): T; + set value(value: unknown); + get dataType(): DataType; +} + +export type Vector = Scalar[]; diff --git a/src/schema/column.ts b/src/schema/column.ts index 94a0244..8292883 100644 --- a/src/schema/column.ts +++ b/src/schema/column.ts @@ -3,6 +3,10 @@ import { isDeepStrictEqual } from 'node:util'; import { DataType, Field, Bool } from '@apache-arrow/esnext-esm'; import * as arrow from './arrow.js'; +import { ClientMeta } from './meta.js'; +import { Resource } from './resource.js'; + +export type ColumnResolver = (meta: ClientMeta, resource: Resource, c: Column) => void; export type Column = { name: string; @@ -12,6 +16,8 @@ export type Column = { notNull: boolean; incrementalKey: boolean; unique: boolean; + resolver?: ColumnResolver; + ignoreInTests: boolean; }; export const createColumn = ({ @@ -30,6 +36,7 @@ export const createColumn = ({ notNull, incrementalKey, unique, + ignoreInTests: false, }); export const formatColumn = (column: Column): string => { @@ -58,13 +65,12 @@ export const fromArrowField = (field: Field): Column => { const unique = metadata.get(arrow.METADATA_UNIQUE) === arrow.METADATA_TRUE; const incrementalKey = metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE; - return { + return createColumn({ name, type, - description: '', primaryKey, notNull: !nullable, unique, incrementalKey, - }; + }); }; diff --git a/src/schema/meta.ts b/src/schema/meta.ts new file mode 100644 index 0000000..ccf5a7b --- /dev/null +++ b/src/schema/meta.ts @@ -0,0 +1,52 @@ +import { Binary, TimeNanosecond } from '@apache-arrow/esnext-esm'; + +import { UUIDType } from '../types/uuid.js'; + +import { Column, createColumn, ColumnResolver } from './column.js'; +import { Resource } from './resource.js'; + +export type ClientMeta = { + id: () => string; +}; + +export const parentCqUUIDResolver = (): ColumnResolver => { + return (_: ClientMeta, r: Resource, c: Column) => { + if (r.parent === null) { + return r.setColumData(c.name, null); + } + const parentCqID = r.parent.getColumnData(cqIDColumn.name); + if (parentCqID == null) { + return r.setColumData(c.name, null); + } + return r.setColumData(c.name, parentCqID); + }; +}; + +export const cqIDColumn = createColumn({ + name: '_cq_id', + type: new UUIDType(), + description: 'Internal CQ ID of the row', + notNull: true, + unique: true, +}); +export const cqParentIDColumn = createColumn({ + name: '_cq_parent_id', + type: new UUIDType(), + description: 'Internal CQ ID of the parent row', + resolver: parentCqUUIDResolver(), + ignoreInTests: true, +}); +export const cqSyncTimeColumn = createColumn({ + name: '_cq_sync_time', + type: new TimeNanosecond(), + description: 'Internal CQ row of when sync was started (this will be the same for all rows in a single fetch)', + resolver: parentCqUUIDResolver(), + ignoreInTests: true, +}); +export const cqSourceNameColumn = createColumn({ + name: '_cq_source_name', + type: new Binary(), + description: 'Internal CQ row that references the source plugin name data was retrieved', + resolver: parentCqUUIDResolver(), + ignoreInTests: true, +}); diff --git a/src/schema/resource.ts b/src/schema/resource.ts new file mode 100644 index 0000000..c3627c4 --- /dev/null +++ b/src/schema/resource.ts @@ -0,0 +1,43 @@ +import { Scalar, Vector } from '../scalar/scalar.js'; + +import { Table } from './table.js'; +import { Nullable } from './types.js'; + +export class Resource { + item: unknown; + parent: Nullable; + table: Table; + data: Vector; + + constructor(table: Table, parent: Nullable, item: unknown) { + this.table = table; + this.parent = parent; + this.item = item; + // TODO: Init from table columns + this.data = []; + } + + getColumnData(columnName: string): Scalar { + const columnIndex = this.table.columns.findIndex((c) => c.name === columnName); + if (columnIndex === undefined) { + throw new Error(`Column '${columnName}' not found`); + } + return this.data[columnIndex]; + } + + setColumData(columnName: string, value: unknown): void { + const columnIndex = this.table.columns.findIndex((c) => c.name === columnName); + if (columnIndex === undefined) { + throw new Error(`Column '${columnName}' not found`); + } + this.data[columnIndex].value = value; + } + + getItem(): unknown { + return this.item; + } + + setItem(item: unknown): void { + this.item = item; + } +} diff --git a/src/schema/table.ts b/src/schema/table.ts new file mode 100644 index 0000000..fe5b43e --- /dev/null +++ b/src/schema/table.ts @@ -0,0 +1,45 @@ +import { Writable } from 'node:stream'; + +import { Column } from './column.js'; +import { ClientMeta } from './meta.js'; +import { Resource } from './resource.js'; +import { Nullable } from './types.js'; + +export type TableResolver = (clientMeta: ClientMeta, parent: Nullable, stream: Writable) => void; +export type RowResolver = (clientMeta: ClientMeta, resource: Resource) => void; +export type Multiplexer = (clientMeta: ClientMeta) => ClientMeta[]; +export type Transform = (table: Table) => void; + +export type Table = { + name: string; + title: string; + description: string; + columns: Column[]; + relations: Table[]; + transform: Transform; + resolver: TableResolver; + multiplexer: Multiplexer; + postResourceResolver: RowResolver; + preResourceResolver: RowResolver; + isIncremental: boolean; + ignoreInTests: boolean; + parent: Nullable; + pkConstraintName: string; +}; + +export const getTablesNames = (tables: Table[]) => tables.map((table) => table.name); +export const getTopLevelTableByName = (tables: Table[], name: string): Table | undefined => + tables.find((table) => table.name === name); + +export const getTableByName = (tables: Table[], name: string): Table | undefined => { + const table = tables.find((table) => table.name === name); + if (table) { + return table; + } + for (const table of tables) { + const found = getTableByName(table.relations, name); + if (found) { + return found; + } + } +}; diff --git a/src/schema/types.ts b/src/schema/types.ts new file mode 100644 index 0000000..aa32b2d --- /dev/null +++ b/src/schema/types.ts @@ -0,0 +1 @@ +export type Nullable = T | null;