Skip to content

Commit fbb7c94

Browse files
authored
feat: Add CQID and Parent CQID (#53)
1 parent 773a0e5 commit fbb7c94

File tree

19 files changed

+340
-85
lines changed

19 files changed

+340
-85
lines changed

package-lock.json

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
"@ava/typescript": "^4.1.0",
4747
"@grpc/grpc-js": "^1.9.0",
4848
"@tsconfig/node16": "^16.1.0",
49+
"@types/uuid": "^9.0.2",
4950
"@types/yargs": "^17.0.24",
5051
"@typescript-eslint/eslint-plugin": "^6.2.1",
5152
"@typescript-eslint/parser": "^6.2.1",
@@ -89,6 +90,7 @@
8990
"matcher": "^5.0.0",
9091
"p-map": "^6.0.0",
9192
"p-timeout": "^6.1.2",
93+
"uuid": "^9.0.0",
9294
"winston": "^3.10.0",
9395
"yargs": "^17.7.2"
9496
}

src/memdb/tables.ts

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import { Utf8 } from '@apache-arrow/esnext-esm';
1+
import { Int64 } from '@apache-arrow/esnext-esm';
22

33
import { createColumn } from '../schema/column.js';
44
import { addCQIDsColumns } from '../schema/meta.js';
5-
import { pathResolver } from '../schema/resolvers.js';
5+
import { pathResolver, parentColumnResolver } from '../schema/resolvers.js';
66
import { createTable } from '../schema/table.js';
77

88
export const createTables = () => {
@@ -12,14 +12,13 @@ export const createTables = () => {
1212
title: 'Table 1',
1313
description: 'Table 1 description',
1414
resolver: (clientMeta, parent, stream) => {
15-
stream.write({ id: 'table1-name1' });
16-
stream.write({ id: 'table1-name2' });
15+
stream.write({ id: 'id-1' });
16+
stream.write({ id: 'id-2' });
1717
return Promise.resolve();
1818
},
1919
columns: [
2020
createColumn({
2121
name: 'id',
22-
type: new Utf8(),
2322
resolver: pathResolver('id'),
2423
}),
2524
],
@@ -29,18 +28,61 @@ export const createTables = () => {
2928
title: 'Table 2',
3029
description: 'Table 2 description',
3130
resolver: (clientMeta, parent, stream) => {
32-
stream.write({ name: 'table2-name1' });
33-
stream.write({ name: 'table2-name2' });
31+
stream.write({ name: 'name-1' });
32+
stream.write({ name: 'name-2' });
3433
return Promise.resolve();
3534
},
3635
columns: [
3736
createColumn({
3837
name: 'name',
39-
type: new Utf8(),
4038
resolver: pathResolver('name'),
4139
}),
4240
],
4341
}),
42+
createTable({
43+
name: 'table3',
44+
title: 'Table 3',
45+
description: 'Table 3 description',
46+
resolver: (clientMeta, parent, stream) => {
47+
stream.write({ name: 'name-1' });
48+
stream.write({ name: 'name-2' });
49+
return Promise.resolve();
50+
},
51+
columns: [
52+
createColumn({
53+
name: 'name',
54+
primaryKey: true,
55+
resolver: pathResolver('name'),
56+
}),
57+
],
58+
relations: [
59+
createTable({
60+
name: 'table3_child1',
61+
resolver: (clientMeta, parent, stream) => {
62+
stream.write({ name: 'name-1', id: 1 });
63+
stream.write({ name: 'name-2', id: 2 });
64+
return Promise.resolve();
65+
},
66+
columns: [
67+
createColumn({
68+
name: 'name',
69+
resolver: pathResolver('name'),
70+
}),
71+
createColumn({
72+
name: 'id',
73+
resolver: pathResolver('id'),
74+
type: new Int64(),
75+
primaryKey: true,
76+
}),
77+
createColumn({
78+
name: 'parent_name',
79+
resolver: parentColumnResolver('name'),
80+
primaryKey: true,
81+
}),
82+
],
83+
}),
84+
],
85+
}),
4486
];
4587

4688
const tableWithCQIDs = allTables.map((table) => addCQIDsColumns(table));

src/scalar/list.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import { DataType, List as ArrowList } from '@apache-arrow/esnext-esm';
22

3-
import { Scalar } from './scalar.js';
3+
import { Scalar, Stringable } from './scalar.js';
44
import { isInvalid, NULL_VALUE } from './util.js';
55

6-
type TVector<T extends Scalar<unknown>> = T[];
6+
type TVector<T extends Scalar<Stringable>> = T[];
77

8-
export class List<T extends Scalar<unknown>> implements Scalar<TVector<T>> {
8+
export class List<T extends Scalar<Stringable>> implements Scalar<TVector<T>> {
99
private _type: new (value?: unknown) => T;
1010
private _valid = false;
1111
private _value: TVector<T> = [];

src/scalar/scalar.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,19 @@ import { Int64 } from './int64.js';
66
import { Text } from './text.js';
77
import { Timestamp } from './timestamp.js';
88

9-
export interface Scalar<T> {
9+
export type Stringable = { toString: () => string };
10+
11+
export interface Scalar<T extends Stringable> {
1012
toString: () => string;
1113
get valid(): boolean;
1214
get value(): T;
1315
set value(value: unknown);
1416
get dataType(): DataType;
1517
}
1618

17-
export type Vector = Scalar<unknown>[];
19+
export type Vector = Scalar<Stringable>[];
1820

19-
export const newScalar = (dataType: DataType): Scalar<unknown> => {
21+
export const newScalar = (dataType: DataType): Scalar<Stringable> => {
2022
if (DataType.isBool(dataType)) {
2123
return new Bool();
2224
}

src/scalar/text.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import test from 'ava';
44
import { Text } from './text.js';
55

66
// eslint-disable-next-line unicorn/no-null
7-
[null, undefined].forEach((v) => {
7+
[null, undefined, new Text()].forEach((v) => {
88
test(`should set values to empty string when ${v} is passed`, (t) => {
99
const s = new Text(v);
1010
t.is(s.value, '');

src/scalar/text.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ export class Text implements Scalar<string> {
4242
return;
4343
}
4444

45+
if (value instanceof Text) {
46+
this._value = value.value;
47+
this._valid = value.valid;
48+
return;
49+
}
50+
4551
if (typeof value!.toString === 'function' && value!.toString !== Object.prototype.toString) {
4652
this._value = value!.toString();
4753
this._valid = true;

src/scheduler/cqid.test.ts

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import { createHash } from 'node:crypto';
2+
3+
import test from 'ava';
4+
5+
import { createColumn } from '../schema/column.js';
6+
import { addCQIDsColumns, cqIDColumn } from '../schema/meta.js';
7+
import { Resource } from '../schema/resource.js';
8+
import { createTable } from '../schema/table.js';
9+
10+
import { setCQId } from './cqid.js';
11+
12+
test('setCQId - should set to random value if deterministicCQId is false', (t): void => {
13+
const resource = new Resource(addCQIDsColumns(createTable({ name: 'table1' })), null, null);
14+
15+
setCQId(resource, false, () => 'random');
16+
17+
t.is(resource.getColumnData(cqIDColumn.name).valid, true);
18+
t.is(resource.getColumnData(cqIDColumn.name).value.toString(), 'random');
19+
});
20+
21+
test('setCQId - should set to random value if deterministicCQId is true and table does not have non _cq_id PKs', (t): void => {
22+
const resource = new Resource(addCQIDsColumns(createTable({ name: 'table1' })), null, null);
23+
24+
setCQId(resource, true, () => 'random');
25+
26+
t.is(resource.getColumnData(cqIDColumn.name).valid, true);
27+
t.is(resource.getColumnData(cqIDColumn.name).value.toString(), 'random');
28+
});
29+
30+
test('setCQId - should set to fixed value if deterministicCQId is true and table has non _cq_id PKs', (t): void => {
31+
const resource = new Resource(
32+
addCQIDsColumns(
33+
createTable({
34+
name: 'table1',
35+
columns: [
36+
createColumn({ name: 'pk1', primaryKey: true, unique: true, notNull: true }),
37+
createColumn({ name: 'pk2', primaryKey: true, unique: true, notNull: true }),
38+
createColumn({ name: 'pk3', primaryKey: true, unique: true, notNull: true }),
39+
createColumn({ name: 'non_pk' }),
40+
],
41+
}),
42+
),
43+
null,
44+
null,
45+
);
46+
47+
resource.setColumData('pk1', 'pk1-value');
48+
resource.setColumData('pk2', 'pk2-value');
49+
resource.setColumData('pk3', 'pk3-value');
50+
resource.setColumData('non_pk', 'non-pk-value');
51+
52+
const expectedSha256 = createHash('sha256');
53+
expectedSha256.update('pk1');
54+
expectedSha256.update('pk1-value');
55+
expectedSha256.update('pk2');
56+
expectedSha256.update('pk2-value');
57+
expectedSha256.update('pk3');
58+
expectedSha256.update('pk3-value');
59+
60+
setCQId(resource, true);
61+
62+
t.is(resource.getColumnData(cqIDColumn.name).valid, true);
63+
t.is(resource.getColumnData(cqIDColumn.name).value.toString(), expectedSha256.digest('hex'));
64+
});

src/scheduler/cqid.ts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { createHash } from 'node:crypto';
2+
3+
import { v4 as uuidv4 } from 'uuid';
4+
5+
import { cqIDColumn } from '../schema/meta.js';
6+
import { Resource } from '../schema/resource.js';
7+
import { getPrimaryKeys } from '../schema/table.js';
8+
9+
export const setCQId = (resource: Resource, deterministicCQId: boolean, generator: () => string = uuidv4) => {
10+
const randomCQId = generator();
11+
if (!deterministicCQId) {
12+
resource.setCqId(randomCQId);
13+
}
14+
15+
const primaryKeys = getPrimaryKeys(resource.table);
16+
const hasNonCqPKs = primaryKeys.some((pk) => pk !== cqIDColumn.name);
17+
if (hasNonCqPKs) {
18+
const sha256 = createHash('sha256');
19+
primaryKeys.sort();
20+
for (const pk of primaryKeys) {
21+
sha256.update(pk);
22+
sha256.update(resource.getColumnData(pk).toString());
23+
}
24+
return resource.setCqId(sha256.digest('hex'));
25+
}
26+
27+
return resource.setCqId(randomCQId);
28+
};

0 commit comments

Comments
 (0)