From ad3a29099cd99061276417a2dc0b18fdec2c7807 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 9 Aug 2023 14:11:32 +0100 Subject: [PATCH 1/2] feat(scalars): Timestamp, float64, int64 Courtesy of ChatGPT --- package-lock.json | 9 +++++ package.json | 1 + src/scalar/float64.ts | 62 ++++++++++++++++++++++++++++++ src/scalar/int64.ts | 71 ++++++++++++++++++++++++++++++++++ src/scalar/timestamp.ts | 84 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 227 insertions(+) create mode 100644 src/scalar/float64.ts create mode 100644 src/scalar/int64.ts create mode 100644 src/scalar/timestamp.ts diff --git a/package-lock.json b/package-lock.json index fe61dd2..7dbc969 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@apache-arrow/esnext-esm": "^12.0.1", "@cloudquery/plugin-pb-javascript": "^0.0.7", "boolean": "^3.2.0", + "luxon": "^3.4.0", "winston": "^3.10.0", "yargs": "^17.7.2" }, @@ -4093,6 +4094,14 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.0.tgz", + "integrity": "sha512-7eDo4Pt7aGhoCheGFIuq4Xa2fJm4ZpmldpGhjTYBNUYNCN6TIEP6v7chwwwt3KRp7YR+rghbfvjyo3V5y9hgBw==", + "engines": { + "node": ">=12" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", diff --git a/package.json b/package.json index e97e9ed..0bf2793 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "@apache-arrow/esnext-esm": "^12.0.1", "@cloudquery/plugin-pb-javascript": "^0.0.7", "boolean": "^3.2.0", + "luxon": "^3.4.0", "winston": "^3.10.0", "yargs": "^17.7.2" } diff --git a/src/scalar/float64.ts b/src/scalar/float64.ts new file mode 100644 index 0000000..3f6a35d --- /dev/null +++ b/src/scalar/float64.ts @@ -0,0 +1,62 @@ +import { DataType, Float64 as ArrowFloat64 } from '@apache-arrow/esnext-esm'; + +import { Scalar } from './scalar.js'; +import { isInvalid, NULL_VALUE } from './util.js'; + +export class Float64 implements Scalar { + private _valid = false; + private _value: number = 0; + + public constructor(v: unknown) { + this.value = v; + return this; + } + + public get dataType(): DataType { + return new ArrowFloat64(); + } + + public get valid(): boolean { + return this._valid; + } + + public get value(): number { + return this._value; + } + + public set value(value: unknown) { + if (isInvalid(value)) { + this._valid = false; + return; + } + + if (value instanceof Float64) { + this._valid = value.valid; + this._value = value.value; + return; + } + + if (typeof value === 'number') { + this._value = value; + this._valid = true; + return; + } + + const floatValue = Number.parseFloat(String(value)); + if (!Number.isNaN(floatValue)) { + this._value = floatValue; + this._valid = true; + return; + } + + throw new Error(`Unable to set '${value}' as Float64`); + } + + public toString() { + if (this._valid) { + return String(this._value); + } + + return NULL_VALUE; + } +} diff --git a/src/scalar/int64.ts b/src/scalar/int64.ts new file mode 100644 index 0000000..9c0521d --- /dev/null +++ b/src/scalar/int64.ts @@ -0,0 +1,71 @@ +import { DataType, Int64 as ArrowInt64 } from '@apache-arrow/esnext-esm'; + +import { Scalar } from './scalar.js'; +import { isInvalid, NULL_VALUE } from './util.js'; + +export class Int64 implements Scalar { + private _valid = false; + private _value: bigint = BigInt(0); + + public constructor(v: unknown) { + this.value = v; + return this; + } + + public get dataType(): DataType { + return new ArrowInt64(); + } + + public get valid(): boolean { + return this._valid; + } + + public get value(): bigint { + return this._value; + } + + public set value(value: unknown) { + if (isInvalid(value)) { + this._valid = false; + return; + } + + if (value instanceof Int64) { + this._valid = value.valid; + this._value = value.value; + return; + } + + if (typeof value === 'bigint') { + this._value = value; + this._valid = true; + return; + } + + if (typeof value === 'number') { + if (!Number.isSafeInteger(value)) { + throw new TypeError(`Value '${value}' cannot be safely converted to Int64`); + } + this._value = BigInt(value); + this._valid = true; + return; + } + + const bigintValue = BigInt(value); + if (bigintValue !== undefined) { + this._value = bigintValue; + this._valid = true; + return; + } + + throw new Error(`Unable to set '${value}' as Int64`); + } + + public toString() { + if (this._valid) { + return String(this._value); + } + + return NULL_VALUE; + } +} diff --git a/src/scalar/timestamp.ts b/src/scalar/timestamp.ts new file mode 100644 index 0000000..991a028 --- /dev/null +++ b/src/scalar/timestamp.ts @@ -0,0 +1,84 @@ +import { DataType, Timestamp as ArrowTimestamp } from '@apache-arrow/esnext-esm'; +import { DateTime } from 'luxon'; + +import { Scalar } from './scalar.js'; +import { isInvalid, NULL_VALUE } from './util.js'; + +export class Timestamp implements Scalar { + private _valid = false; + private _value: DateTime = DateTime.fromMillis(0); + + public constructor(v: unknown) { + this.value = v; + return this; + } + + public get dataType(): DataType { + return new ArrowTimestamp(); + } + + public get valid(): boolean { + return this._valid; + } + + public get value(): DateTime { + return this._value; + } + + public set value(value: unknown) { + if (isInvalid(value)) { + this._valid = false; + return; + } + + if (value instanceof Timestamp) { + this._valid = value.valid; + this._value = value.value; + return; + } + + let dateValue: DateTime | null = null; + + if (typeof value === 'string') { + dateValue = DateTime.fromFormat(value, 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS ZZZZ', { setZone: true }); + + if (!dateValue.isValid) { + dateValue = DateTime.fromFormat(value, 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS', { zone: 'utc' }); + } + + if (!dateValue.isValid) { + dateValue = DateTime.fromFormat(value, "yyyy-MM-dd HH:mm:ss.SSSSSSSSS'Z'", { zone: 'utc' }); + } + + if (!dateValue.isValid) { + dateValue = DateTime.fromISO(value, { setZone: true }); + } + } + + if (dateValue && dateValue.isValid) { + this._value = dateValue; + this._valid = true; + return; + } + + throw new Error(`Unable to set '${value}' as Timestamp`); + } + + public toString() { + if (this._valid) { + return this._value.toISO(); + } + + return NULL_VALUE; + } + + public equals(scalar: Timestamp): boolean { + if (!scalar) { + return false; + } + if (scalar instanceof Timestamp) { + return this._value.equals(scalar.value) && this._valid === scalar.valid; + } + return false; + } +} From 0ee0bb2ab410a12c81ec1f66f66e32375f4d7f22 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 9 Aug 2023 14:25:11 +0100 Subject: [PATCH 2/2] fixes --- package-lock.json | 6 ++++++ package.json | 1 + src/scalar/int64.ts | 7 ------- src/scalar/timestamp.ts | 24 +++++++++--------------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7dbc969..01756dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@apache-arrow/esnext-esm": "^12.0.1", "@cloudquery/plugin-pb-javascript": "^0.0.7", + "@types/luxon": "^3.3.1", "boolean": "^3.2.0", "luxon": "^3.4.0", "winston": "^3.10.0", @@ -645,6 +646,11 @@ "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", "dev": true }, + "node_modules/@types/luxon": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.1.tgz", + "integrity": "sha512-XOS5nBcgEeP2PpcqJHjCWhUCAzGfXIU8ILOSLpx2FhxqMW9KdxgCGXNOEKGVBfveKtIpztHzKK5vSRVLyW/NqA==" + }, "node_modules/@types/node": { "version": "20.4.6", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.6.tgz", diff --git a/package.json b/package.json index 0bf2793..7a2edea 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "dependencies": { "@apache-arrow/esnext-esm": "^12.0.1", "@cloudquery/plugin-pb-javascript": "^0.0.7", + "@types/luxon": "^3.3.1", "boolean": "^3.2.0", "luxon": "^3.4.0", "winston": "^3.10.0", diff --git a/src/scalar/int64.ts b/src/scalar/int64.ts index 9c0521d..828fd6c 100644 --- a/src/scalar/int64.ts +++ b/src/scalar/int64.ts @@ -51,13 +51,6 @@ export class Int64 implements Scalar { return; } - const bigintValue = BigInt(value); - if (bigintValue !== undefined) { - this._value = bigintValue; - this._valid = true; - return; - } - throw new Error(`Unable to set '${value}' as Int64`); } diff --git a/src/scalar/timestamp.ts b/src/scalar/timestamp.ts index 991a028..eb6e873 100644 --- a/src/scalar/timestamp.ts +++ b/src/scalar/timestamp.ts @@ -1,4 +1,4 @@ -import { DataType, Timestamp as ArrowTimestamp } from '@apache-arrow/esnext-esm'; +import { DataType, Timestamp as ArrowTimestamp, TimeUnit } from '@apache-arrow/esnext-esm'; import { DateTime } from 'luxon'; import { Scalar } from './scalar.js'; @@ -7,14 +7,18 @@ import { isInvalid, NULL_VALUE } from './util.js'; export class Timestamp implements Scalar { private _valid = false; private _value: DateTime = DateTime.fromMillis(0); + private _unit: TimeUnit = TimeUnit.NANOSECOND; - public constructor(v: unknown) { + public constructor(v: unknown, unit?: TimeUnit) { this.value = v; + if (unit) { + this._unit = unit; + } return this; } public get dataType(): DataType { - return new ArrowTimestamp(); + return new ArrowTimestamp(this._unit); } public get valid(): boolean { @@ -64,21 +68,11 @@ export class Timestamp implements Scalar { throw new Error(`Unable to set '${value}' as Timestamp`); } - public toString() { + public toString(): string { if (this._valid) { - return this._value.toISO(); + return this._value.toISO()!; } return NULL_VALUE; } - - public equals(scalar: Timestamp): boolean { - if (!scalar) { - return false; - } - if (scalar instanceof Timestamp) { - return this._value.equals(scalar.value) && this._valid === scalar.valid; - } - return false; - } }