Skip to content

Commit 45ab919

Browse files
erezrokaherezrokah
authored
feat: Add UUID, JSON types (#56)
1 parent 5a8f3c3 commit 45ab919

File tree

10 files changed

+83
-51
lines changed

10 files changed

+83
-51
lines changed

‎src/scalar/json.ts

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ const validate = (value: string) => {
1212
}
1313
};
1414

15-
class JSONType implements Scalar<string> {
15+
class JSONType implements Scalar<Uint8Array> {
1616
private _valid = false;
17-
private _value = '';
17+
private _value = new TextEncoder().encode(NULL_VALUE);
1818

1919
public constructor(v?: unknown) {
2020
this.value = v;
@@ -29,7 +29,7 @@ class JSONType implements Scalar<string> {
2929
return this._valid;
3030
}
3131

32-
public get value(): string {
32+
public get value(): Uint8Array {
3333
return this._value;
3434
}
3535

@@ -40,14 +40,14 @@ class JSONType implements Scalar<string> {
4040
}
4141

4242
if (typeof value === 'string') {
43-
this._value = value;
43+
this._value = new TextEncoder().encode(value);
4444
this._valid = validate(value);
4545
return;
4646
}
4747

4848
if (value instanceof Uint8Array) {
49-
this._value = new TextDecoder().decode(value);
50-
this._valid = validate(this._value);
49+
this._value = value;
50+
this._valid = validate(new TextDecoder().decode(value));
5151
return;
5252
}
5353

@@ -58,7 +58,7 @@ class JSONType implements Scalar<string> {
5858
}
5959

6060
try {
61-
this._value = JSON.stringify(value);
61+
this._value = new TextEncoder().encode(JSON.stringify(value));
6262
this._valid = true;
6363
} catch {
6464
throw new Error(`Unable to set '${value}' as JSON`);
@@ -67,7 +67,7 @@ class JSONType implements Scalar<string> {
6767

6868
public toString() {
6969
if (this._valid) {
70-
return this._value;
70+
return new TextDecoder().decode(this._value);
7171
}
7272

7373
return NULL_VALUE;

‎src/scalar/uuid.ts

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,26 @@
1-
import { Utf8 as ArrowString } from '@apache-arrow/esnext-esm';
1+
import { FixedSizeBinary } from '@apache-arrow/esnext-esm';
22
import { validate } from 'uuid';
33

44
import { Scalar } from './scalar.js';
55
import { isInvalid, NULL_VALUE } from './util.js';
66

7-
export class UUID implements Scalar<string> {
7+
export class UUID implements Scalar<Uint8Array> {
88
private _valid = false;
9-
private _value = '';
9+
private _value = new TextEncoder().encode(NULL_VALUE);
1010

1111
public constructor(v?: unknown) {
1212
this.value = v;
13-
return this;
1413
}
1514

1615
public get dataType() {
17-
return new ArrowString();
16+
return new FixedSizeBinary(16);
1817
}
1918

2019
public get valid(): boolean {
2120
return this._valid;
2221
}
2322

24-
public get value(): string {
23+
public get value(): Uint8Array {
2524
return this._value;
2625
}
2726

@@ -32,14 +31,14 @@ export class UUID implements Scalar<string> {
3231
}
3332

3433
if (typeof value === 'string') {
35-
this._value = value;
34+
this._value = new TextEncoder().encode(value);
3635
this._valid = validate(value);
3736
return;
3837
}
3938

4039
if (value instanceof Uint8Array) {
41-
this._value = new TextDecoder().decode(value);
42-
this._valid = validate(this._value);
40+
this._value = value;
41+
this._valid = validate(new TextDecoder().decode(value));
4342
return;
4443
}
4544

@@ -50,8 +49,8 @@ export class UUID implements Scalar<string> {
5049
}
5150

5251
if (typeof value!.toString === 'function' && value!.toString !== Object.prototype.toString) {
53-
this._value = value!.toString();
54-
this._valid = validate(this._value);
52+
this._value = Buffer.from(value!.toString());
53+
this._valid = validate(value!.toString());
5554
return;
5655
}
5756

@@ -60,7 +59,7 @@ export class UUID implements Scalar<string> {
6059

6160
public toString() {
6261
if (this._valid) {
63-
return this._value;
62+
return new TextDecoder().decode(this._value);
6463
}
6564

6665
return NULL_VALUE;

‎src/scheduler/cqid.test.ts

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ test('setCQId - should set to random value if deterministicCQId is false', (t):
2727

2828
setCQId(resource, false, () => NIL_UUID);
2929

30-
t.is(resource.getColumnData(cqIDColumn.name).valid, true);
31-
t.is(resource.getColumnData(cqIDColumn.name).value.toString(), NIL_UUID);
30+
const cqId = resource.getColumnData(cqIDColumn.name);
31+
t.is(cqId.valid, true);
32+
t.is(cqId.toString(), NIL_UUID);
3233
});
3334

3435
test('setCQId - should set to random value if deterministicCQId is true and table does not have non _cq_id PKs', (t): void => {
@@ -50,8 +51,9 @@ test('setCQId - should set to random value if deterministicCQId is true and tabl
5051

5152
setCQId(resource, true, () => NIL_UUID);
5253

53-
t.is(resource.getColumnData(cqIDColumn.name).valid, true);
54-
t.is(resource.getColumnData(cqIDColumn.name).value.toString(), NIL_UUID);
54+
const cqId = resource.getColumnData(cqIDColumn.name);
55+
t.is(cqId.valid, true);
56+
t.is(cqId.toString(), NIL_UUID);
5557
});
5658

5759
test('setCQId - should set to fixed value if deterministicCQId is true and table has non _cq_id PKs', (t): void => {
@@ -78,6 +80,7 @@ test('setCQId - should set to fixed value if deterministicCQId is true and table
7880

7981
setCQId(resource, true);
8082

81-
t.is(resource.getColumnData(cqIDColumn.name).valid, true);
82-
t.is(resource.getColumnData(cqIDColumn.name).value.toString(), '415bd5dd-9bac-5806-b9d1-c53f17d37455');
83+
const cqId = resource.getColumnData(cqIDColumn.name);
84+
t.is(cqId.valid, true);
85+
t.is(cqId.toString(), '415bd5dd-9bac-5806-b9d1-c53f17d37455');
8386
});

‎src/schema/arrow.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,6 @@ export const METADATA_TABLE_NAME = 'cq:table_name';
99
export const METADATA_TABLE_TITLE = 'cq:table_title';
1010
export const METADATA_TABLE_DESCRIPTION = 'cq:table_description';
1111
export const METADATA_TABLE_DEPENDS_ON = 'cq:table_depends_on';
12+
13+
export const METADATA_ARROW_EXTENSION_NAME = 'ARROW:extension:name';
14+
export const METADATA_ARROW_EXTENSION_METADATA = 'ARROW:extension:metadata';

‎src/schema/column.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import { isDeepStrictEqual } from 'node:util';
22

33
import { DataType, Field, Utf8 } from '@apache-arrow/esnext-esm';
44

5+
import { ExtensionType, isExtensionType } from '../types/extensions.js';
6+
57
import * as arrow from './arrow.js';
68
import { ClientMeta } from './meta.js';
79
import { Resource } from './resource.js';
@@ -60,7 +62,13 @@ export const toArrowField = (column: Column): Field => {
6062
metadataMap.set(arrow.METADATA_UNIQUE, unique ? arrow.METADATA_TRUE : arrow.METADATA_FALSE);
6163
metadataMap.set(arrow.METADATA_INCREMENTAL, incrementalKey ? arrow.METADATA_TRUE : arrow.METADATA_FALSE);
6264

63-
return new Field(name, type, /*nullable=*/ !notNull, metadataMap);
65+
if (isExtensionType(type)) {
66+
const { name, metadata } = type as unknown as ExtensionType;
67+
metadataMap.set(arrow.METADATA_ARROW_EXTENSION_NAME, name);
68+
metadataMap.set(arrow.METADATA_ARROW_EXTENSION_METADATA, metadata);
69+
}
70+
71+
return new Field(name, type, !notNull, metadataMap);
6472
};
6573

6674
export const fromArrowField = (field: Field): Column => {

‎src/schema/meta.test.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ test('parentCqUUIDResolver - should set to _cq_id column value when parent has i
9595

9696
parentCqUUIDResolver()({ id: () => '' }, childResource, cqParentIDColumn);
9797

98-
t.is(childResource.getColumnData(cqParentIDColumn.name).value, '9241a9cb-f580-420f-8fd7-46d2c4f55ccb');
99-
t.is(childResource.getColumnData(cqParentIDColumn.name).valid, true);
98+
const cqParentId = childResource.getColumnData(cqParentIDColumn.name);
99+
t.is(cqParentId.valid, true);
100+
t.is(cqParentId.toString(), '9241a9cb-f580-420f-8fd7-46d2c4f55ccb');
100101
});

‎src/schema/resource.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { tableToIPC, Table as ArrowTable, RecordBatch, vectorFromArray } from '@apache-arrow/esnext-esm';
22

33
import { Scalar, Vector, newScalar, Stringable } from '../scalar/scalar.js';
4+
import { isExtensionType } from '../types/extensions.js';
45

56
import { cqIDColumn } from './meta.js';
67
import { Table, toArrowSchema } from './table.js';
@@ -59,7 +60,11 @@ export const encodeResource = (resource: Resource): Uint8Array => {
5960
let batch = new RecordBatch(schema, undefined);
6061
for (let index = 0; index < table.columns.length; index++) {
6162
const column = table.columns[index];
62-
const data = resource.getColumnData(column.name);
63+
// For extension types, we need to get the underlying value
64+
const data = isExtensionType(column.type)
65+
? resource.getColumnData(column.name).value
66+
: resource.getColumnData(column.name);
67+
6368
const vector = vectorFromArray([data], column.type);
6469
batch = batch.setChildAt(index, vector);
6570
}

‎src/types/extensions.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { DataType } from '@apache-arrow/esnext-esm';
2+
3+
import { JSONType } from './json.js';
4+
import { UUIDType } from './uuid.js';
5+
6+
export interface ExtensionType {
7+
get name(): string;
8+
get metadata(): string;
9+
}
10+
11+
const extensions = [JSONType, UUIDType];
12+
13+
export const isExtensionType = (type: DataType) => extensions.some((extension) => type instanceof extension);

‎src/types/json.ts

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
1-
import { DataType, Type } from '@apache-arrow/esnext-esm';
1+
import { Type, DataType } from '@apache-arrow/esnext-esm';
22

3-
export class JSONType extends DataType<Type.Utf8> {
4-
readonly extensionName: string = 'json';
3+
import { ExtensionType } from './extensions.js';
54

6-
constructor() {
7-
super();
5+
export class JSONType extends DataType<Type.Binary> implements ExtensionType {
6+
get name(): string {
7+
return 'json';
8+
}
9+
get metadata(): string {
10+
return 'json-serialized';
811
}
912

10-
get typeId(): Type.Utf8 {
11-
return Type.Utf8;
13+
get typeId(): Type.Binary {
14+
return Type.Binary;
1215
}
1316
}

‎src/types/uuid.ts

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,20 @@
1-
import { DataType, Type } from '@apache-arrow/esnext-esm';
1+
import { Type, FixedSizeBinary } from '@apache-arrow/esnext-esm';
22

3-
export class UUIDType extends DataType<Type.Utf8> {
4-
readonly extensionName: string = 'uuid';
3+
import { ExtensionType } from './extensions.js';
54

5+
export class UUIDType extends FixedSizeBinary implements ExtensionType {
66
constructor() {
7-
super();
7+
super(16);
88
}
99

10-
get typeId(): Type.Utf8 {
11-
return Type.Utf8;
10+
get name(): string {
11+
return 'uuid';
1212
}
13-
14-
public toString() {
15-
return `uuid`;
13+
get metadata(): string {
14+
return 'uuid-serialized';
1615
}
1716

18-
protected static [Symbol.toStringTag] = ((proto: UUIDType) => {
19-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
20-
(<any>proto).ArrayType = Uint8Array;
21-
return (proto[Symbol.toStringTag] = 'uuid');
22-
})(UUIDType.prototype);
17+
get typeId(): Type.FixedSizeBinary {
18+
return Type.FixedSizeBinary;
19+
}
2320
}

0 commit comments

Comments
 (0)