feat: Add table types

.eslintrc

@@ -19,6 +19,7 @@
     "plugin:you-dont-need-lodash-underscore/all"
   ],
   "rules": {
+    "unicorn/no-null": 0,
     "unused-imports/no-unused-imports": "error",
     "no-console": "error",
     "@typescript-eslint/no-unused-vars": 0,
@@ -40,7 +41,6 @@
     {
       "files": ["src/grpc/**/*.ts"],
       "rules": {
-        "unicorn/no-null": 0,
         "@typescript-eslint/naming-convention": 0
       }
     },

src/scalar/bool.ts

@@ -1,14 +1,15 @@
 import { Bool as ArrowBool } from '@apache-arrow/esnext-esm';
 import { boolean, isBooleanable } from 'boolean';
 
+import { Scalar } from './scalar.js';
 import { isInvalid, NULL_VALUE } from './util.js';
 
-export class Bool {
+export class Bool implements Scalar<boolean> {
   private _valid = false;
   private _value = false;
 
   public constructor(v: unknown) {
-    this.valid = v;
+    this.value = v;
     return this;
   }
 
@@ -24,7 +25,7 @@ export class Bool {
     return this._value;
   }
 
-  public set valid(value: unknown) {
+  public set value(value: unknown) {
     if (isInvalid(value)) {
       this._valid = false;
       return;

src/scalar/scalar.ts

@@ -0,0 +1,11 @@
+import { DataType } from '@apache-arrow/esnext-esm';
+
+export interface Scalar<T> {
+  toString: () => string;
+  get valid(): boolean;
+  get value(): T;
+  set value(value: unknown);
+  get dataType(): DataType;
+}
+
+export type Vector = Scalar<unknown>[];

src/schema/column.ts

@@ -3,6 +3,10 @@ import { isDeepStrictEqual } from 'node:util';
 import { DataType, Field, Bool } from '@apache-arrow/esnext-esm';
 
 import * as arrow from './arrow.js';
+import { ClientMeta } from './meta.js';
+import { Resource } from './resource.js';
+
+export type ColumnResolver = (meta: ClientMeta, resource: Resource, c: Column) => void;
 
 export type Column = {
   name: string;
@@ -12,6 +16,8 @@ export type Column = {
   notNull: boolean;
   incrementalKey: boolean;
   unique: boolean;
+  resolver?: ColumnResolver;
+  ignoreInTests: boolean;
 };
 
 export const createColumn = ({
@@ -30,6 +36,7 @@ export const createColumn = ({
   notNull,
   incrementalKey,
   unique,
+  ignoreInTests: false,
 });
 
 export const formatColumn = (column: Column): string => {
@@ -58,13 +65,12 @@ export const fromArrowField = (field: Field): Column => {
   const unique = metadata.get(arrow.METADATA_UNIQUE) === arrow.METADATA_TRUE;
   const incrementalKey = metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE;
 
-  return {
+  return createColumn({
     name,
     type,
-    description: '',
     primaryKey,
     notNull: !nullable,
     unique,
     incrementalKey,
-  };
+  });
 };

src/schema/meta.ts

@@ -0,0 +1,52 @@
+import { Binary, TimeNanosecond } from '@apache-arrow/esnext-esm';
+
+import { UUIDType } from '../types/uuid.js';
+
+import { Column, createColumn, ColumnResolver } from './column.js';
+import { Resource } from './resource.js';
+
+export type ClientMeta = {
+  id: () => string;
+};
+
+export const parentCqUUIDResolver = (): ColumnResolver => {
+  return (_: ClientMeta, r: Resource, c: Column) => {
+    if (r.parent === null) {
+      return r.setColumData(c.name, null);
+    }
+    const parentCqID = r.parent.getColumnData(cqIDColumn.name);
+    if (parentCqID == null) {
+      return r.setColumData(c.name, null);
+    }
+    return r.setColumData(c.name, parentCqID);
+  };
+};
+
+export const cqIDColumn = createColumn({
+  name: '_cq_id',
+  type: new UUIDType(),
+  description: 'Internal CQ ID of the row',
+  notNull: true,
+  unique: true,
+});
+export const cqParentIDColumn = createColumn({
+  name: '_cq_parent_id',
+  type: new UUIDType(),
+  description: 'Internal CQ ID of the parent row',
+  resolver: parentCqUUIDResolver(),
+  ignoreInTests: true,
+});
+export const cqSyncTimeColumn = createColumn({
+  name: '_cq_sync_time',
+  type: new TimeNanosecond(),
+  description: 'Internal CQ row of when sync was started (this will be the same for all rows in a single fetch)',
+  resolver: parentCqUUIDResolver(),
+  ignoreInTests: true,
+});
+export const cqSourceNameColumn = createColumn({
+  name: '_cq_source_name',
+  type: new Binary(),
+  description: 'Internal CQ row that references the source plugin name data was retrieved',
+  resolver: parentCqUUIDResolver(),
+  ignoreInTests: true,
+});

src/schema/resource.ts

@@ -0,0 +1,43 @@
+import { Scalar, Vector } from '../scalar/scalar.js';
+
+import { Table } from './table.js';
+import { Nullable } from './types.js';
+
+export class Resource {
+  item: unknown;
+  parent: Nullable<Resource>;
+  table: Table;
+  data: Vector;
+
+  constructor(table: Table, parent: Nullable<Resource>, item: unknown) {
+    this.table = table;
+    this.parent = parent;
+    this.item = item;
+    // TODO: Init from table columns
+    this.data = [];
+  }
+
+  getColumnData(columnName: string): Scalar<unknown> {
+    const columnIndex = this.table.columns.findIndex((c) => c.name === columnName);
+    if (columnIndex === undefined) {
+      throw new Error(`Column '${columnName}' not found`);
+    }
+    return this.data[columnIndex];
+  }
+
+  setColumData(columnName: string, value: unknown): void {
+    const columnIndex = this.table.columns.findIndex((c) => c.name === columnName);
+    if (columnIndex === undefined) {
+      throw new Error(`Column '${columnName}' not found`);
+    }
+    this.data[columnIndex].value = value;
+  }
+
+  getItem(): unknown {
+    return this.item;
+  }
+
+  setItem(item: unknown): void {
+    this.item = item;
+  }
+}

src/schema/table.ts

@@ -0,0 +1,45 @@
+import { Writable } from 'node:stream';
+
+import { Column } from './column.js';
+import { ClientMeta } from './meta.js';
+import { Resource } from './resource.js';
+import { Nullable } from './types.js';
+
+export type TableResolver = (clientMeta: ClientMeta, parent: Nullable<Resource>, stream: Writable) => void;
+export type RowResolver = (clientMeta: ClientMeta, resource: Resource) => void;
+export type Multiplexer = (clientMeta: ClientMeta) => ClientMeta[];
+export type Transform = (table: Table) => void;
+
+export type Table = {
+  name: string;
+  title: string;
+  description: string;
+  columns: Column[];
+  relations: Table[];
+  transform: Transform;
+  resolver: TableResolver;
+  multiplexer: Multiplexer;
+  postResourceResolver: RowResolver;
+  preResourceResolver: RowResolver;
+  isIncremental: boolean;
+  ignoreInTests: boolean;
+  parent: Nullable<Table>;
+  pkConstraintName: string;
+};
+
+export const getTablesNames = (tables: Table[]) => tables.map((table) => table.name);
+export const getTopLevelTableByName = (tables: Table[], name: string): Table | undefined =>
+  tables.find((table) => table.name === name);
+
+export const getTableByName = (tables: Table[], name: string): Table | undefined => {
+  const table = tables.find((table) => table.name === name);
+  if (table) {
+    return table;
+  }
+  for (const table of tables) {
+    const found = getTableByName(table.relations, name);
+    if (found) {
+      return found;
+    }
+  }
+};

src/schema/types.ts

@@ -0,0 +1 @@
+export type Nullable<T> = T | null;