diff --git a/server/node-service/package.json b/server/node-service/package.json index 20c5df8d5..3d3887699 100644 --- a/server/node-service/package.json +++ b/server/node-service/package.json @@ -48,6 +48,7 @@ "@types/node-fetch": "^2.6.2", "axios": "^1.2.0", "base64-arraybuffer": "^1.0.2", + "duckdb-async": "^0.10.0", "dynamodb-data-types": "^4.0.1", "express": "^4.18.2", "express-async-errors": "^3.1.1", diff --git a/server/node-service/src/plugins/duckdb/dataSourceConfig.ts b/server/node-service/src/plugins/duckdb/dataSourceConfig.ts new file mode 100644 index 000000000..7186ed4a7 --- /dev/null +++ b/server/node-service/src/plugins/duckdb/dataSourceConfig.ts @@ -0,0 +1,27 @@ +// @server/node-service/src/plugins/duckdb/dataSourceConfig.ts +import { ConfigToType } from "lowcoder-sdk/dataSource"; + +const dataSourceConfig = { + type: "dataSource", + params: [ + { + key: "databaseFile", + type: "textInput", + label: "Database File", + rules: [{ required: true, message: "Please provide a database file path 'db.duckdb' or ':memory:' for an in-memory database" }], + tooltip: "Please provide a database file path 'db.duckdb' or ':memory:' for an in-memory database", + defaultValue: ":memory:", + }, + { + key: "options", + type: "textInput", + label: "Database Options", + tooltip: "Additional options to pass to the DuckDB constructor (in JSON format)", + defaultValue: `{"access_mode": "READ_WRITE","max_memory": "512MB","threads": "4"}`, + }, + ], +} as const; + +export default dataSourceConfig; + +export type DataSourceDataType = ConfigToType; diff --git a/server/node-service/src/plugins/duckdb/index.ts b/server/node-service/src/plugins/duckdb/index.ts new file mode 100644 index 000000000..6d3e3872f --- /dev/null +++ b/server/node-service/src/plugins/duckdb/index.ts @@ -0,0 +1,42 @@ +import { DataSourcePlugin } from "lowcoder-sdk/dataSource"; +import dataSourceConfig, { DataSourceDataType } from "./dataSourceConfig"; +import queryConfig, { ActionDataType } from "./queryConfig"; +import { Database } from "duckdb-async"; +import { ServiceError } from "../../common/error"; + +// Helper function to handle BigInt serialization +function serializeBigInts(row: any): any { + const newRow: { [key: string]: any } = {}; // Add index signature + for (const [key, value] of Object.entries(row)) { + newRow[key] = typeof value === 'bigint' ? value.toString() : value; + } + return newRow; +} + +const duckdbPlugin: DataSourcePlugin = { + id: "duckdb", + name: "DuckDB", + category: "database", + icon: "duckdb.svg", + dataSourceConfig, + queryConfig, + run: async function (actionData, dataSourceConfig): Promise { + const { databaseFile, options } = dataSourceConfig; + const parsedOptions = JSON.parse(options); + const db = await Database.create(databaseFile, parsedOptions); + + if (actionData.actionName === "Query") { + try { + const result = await db.all(actionData.queryString); + // Apply BigInt serialization to each row + return result.map(serializeBigInts); + } catch (error) { + throw new ServiceError((error as Error).message); + } finally { + await db.close(); + } + } + }, +}; + +export default duckdbPlugin; diff --git a/server/node-service/src/plugins/duckdb/queryConfig.ts b/server/node-service/src/plugins/duckdb/queryConfig.ts new file mode 100644 index 000000000..456a68882 --- /dev/null +++ b/server/node-service/src/plugins/duckdb/queryConfig.ts @@ -0,0 +1,24 @@ +// @server/node-service/src/plugins/duckdb/queryConfig.ts +import { ConfigToType } from "lowcoder-sdk/dataSource"; + +const queryConfig = { + type: "query", + label: "Action", + actions: [ + { + actionName: "Query", + label: "Query", + params: [ + { + label: "Query String", + key: "queryString", + type: "sqlInput", + }, + ], + }, + ], +} as const; + +export type ActionDataType = ConfigToType; + +export default queryConfig; diff --git a/server/node-service/src/plugins/index.ts b/server/node-service/src/plugins/index.ts index 258ad8e93..5944908bd 100644 --- a/server/node-service/src/plugins/index.ts +++ b/server/node-service/src/plugins/index.ts @@ -8,6 +8,7 @@ import couchdbPlugin from "./couchdb"; import wooCommercePlugin from "./woocommerce"; import openAiPlugin from "./openAi"; import athenaPlugin from "./athena"; +import duckdbPlugin from "./duckdb"; import lambdaPlugin from "./lambda"; import googleCloudStorage from "./googleCloudStorage"; import stripePlugin from "./stripe"; @@ -43,6 +44,7 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [ wooCommercePlugin, openAiPlugin, athenaPlugin, + duckdbPlugin, lambdaPlugin, googleCloudStorage, stripePlugin, @@ -72,6 +74,6 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [ try { plugins = require("../ee/plugins").default; console.info("using ee plugins"); -} catch {} +} catch { } export default plugins; diff --git a/server/node-service/src/static/plugin-icons/duckdb.svg b/server/node-service/src/static/plugin-icons/duckdb.svg new file mode 100644 index 000000000..982fca747 --- /dev/null +++ b/server/node-service/src/static/plugin-icons/duckdb.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/node-service/yarn.lock b/server/node-service/yarn.lock index e022d3cce..55e5ef66b 100644 --- a/server/node-service/yarn.lock +++ b/server/node-service/yarn.lock @@ -3001,6 +3001,25 @@ __metadata: languageName: node linkType: hard +"@mapbox/node-pre-gyp@npm:^1.0.0": + version: 1.0.11 + resolution: "@mapbox/node-pre-gyp@npm:1.0.11" + dependencies: + detect-libc: ^2.0.0 + https-proxy-agent: ^5.0.0 + make-dir: ^3.1.0 + node-fetch: ^2.6.7 + nopt: ^5.0.0 + npmlog: ^5.0.1 + rimraf: ^3.0.2 + semver: ^7.3.5 + tar: ^6.1.11 + bin: + node-pre-gyp: bin/node-pre-gyp + checksum: b848f6abc531a11961d780db813cc510ca5a5b6bf3184d72134089c6875a91c44d571ba6c1879470020803f7803609e7b2e6e429651c026fe202facd11d444b8 + languageName: node + linkType: hard + "@npmcli/fs@npm:^2.1.0": version: 2.1.2 resolution: "@npmcli/fs@npm:2.1.2" @@ -4663,6 +4682,16 @@ __metadata: languageName: node linkType: hard +"are-we-there-yet@npm:^2.0.0": + version: 2.0.0 + resolution: "are-we-there-yet@npm:2.0.0" + dependencies: + delegates: ^1.0.0 + readable-stream: ^3.6.0 + checksum: 6c80b4fd04ecee6ba6e737e0b72a4b41bdc64b7d279edfc998678567ff583c8df27e27523bc789f2c99be603ffa9eaa612803da1d886962d2086e7ff6fa90c7c + languageName: node + linkType: hard + "are-we-there-yet@npm:^3.0.0": version: 3.0.1 resolution: "are-we-there-yet@npm:3.0.1" @@ -5304,7 +5333,7 @@ __metadata: languageName: node linkType: hard -"color-support@npm:^1.1.3": +"color-support@npm:^1.1.2, color-support@npm:^1.1.3": version: 1.1.3 resolution: "color-support@npm:1.1.3" bin: @@ -5658,6 +5687,13 @@ __metadata: languageName: node linkType: hard +"detect-libc@npm:^2.0.0": + version: 2.0.3 + resolution: "detect-libc@npm:2.0.3" + checksum: 2ba6a939ae55f189aea996ac67afceb650413c7a34726ee92c40fb0deb2400d57ef94631a8a3f052055eea7efb0f99a9b5e6ce923415daa3e68221f963cfc27d + languageName: node + linkType: hard + "detect-newline@npm:^3.0.0": version: 3.1.0 resolution: "detect-newline@npm:3.1.0" @@ -5717,6 +5753,26 @@ __metadata: languageName: node linkType: hard +"duckdb-async@npm:^0.10.0": + version: 0.10.0 + resolution: "duckdb-async@npm:0.10.0" + dependencies: + duckdb: 0.10.0 + checksum: 40b9f44506422048af2fd7a3ea48066ec35cc83b931a1b6e19ab1c007a8234f61b126f7d8915a690a1f9c64a5f99ef2cc0c4a9d564da95db9406720eee703432 + languageName: node + linkType: hard + +"duckdb@npm:0.10.0": + version: 0.10.0 + resolution: "duckdb@npm:0.10.0" + dependencies: + "@mapbox/node-pre-gyp": ^1.0.0 + node-addon-api: ^7.0.0 + node-gyp: ^9.3.0 + checksum: 934b3de2aac87e6aa30db6347730aeeba60774f0e16751858fca208d7d46b1808209876602c53d905af46587b6a1aed2453b6252dd9254875bc5438060070556 + languageName: node + linkType: hard + "duplexify@npm:^4.0.0, duplexify@npm:^4.1.1": version: 4.1.2 resolution: "duplexify@npm:4.1.2" @@ -6057,6 +6113,13 @@ __metadata: languageName: node linkType: hard +"exponential-backoff@npm:^3.1.1": + version: 3.1.1 + resolution: "exponential-backoff@npm:3.1.1" + checksum: 3d21519a4f8207c99f7457287291316306255a328770d320b401114ec8481986e4e467e854cb9914dd965e0a1ca810a23ccb559c642c88f4c7f55c55778a9b48 + languageName: node + linkType: hard + "express-async-errors@npm:^3.1.1": version: 3.1.1 resolution: "express-async-errors@npm:3.1.1" @@ -6414,6 +6477,23 @@ __metadata: languageName: node linkType: hard +"gauge@npm:^3.0.0": + version: 3.0.2 + resolution: "gauge@npm:3.0.2" + dependencies: + aproba: ^1.0.3 || ^2.0.0 + color-support: ^1.1.2 + console-control-strings: ^1.0.0 + has-unicode: ^2.0.1 + object-assign: ^4.1.1 + signal-exit: ^3.0.0 + string-width: ^4.2.3 + strip-ansi: ^6.0.1 + wide-align: ^1.1.2 + checksum: 81296c00c7410cdd48f997800155fbead4f32e4f82109be0719c63edc8560e6579946cc8abd04205297640691ec26d21b578837fd13a4e96288ab4b40b1dc3e9 + languageName: node + linkType: hard + "gauge@npm:^4.0.3": version: 4.0.4 resolution: "gauge@npm:4.0.4" @@ -7974,7 +8054,7 @@ __metadata: languageName: node linkType: hard -"make-dir@npm:^3.0.0": +"make-dir@npm:^3.0.0, make-dir@npm:^3.1.0": version: 3.1.0 resolution: "make-dir@npm:3.1.0" dependencies: @@ -8417,6 +8497,15 @@ __metadata: languageName: node linkType: hard +"node-addon-api@npm:^7.0.0": + version: 7.1.0 + resolution: "node-addon-api@npm:7.1.0" + dependencies: + node-gyp: latest + checksum: 26640c8d2ed7e2059e2ed65ee79e2a195306b3f1fc27ad11448943ba91d37767bd717a9a0453cc97e83a1109194dced8336a55f8650000458ef625c0b8b5e3df + languageName: node + linkType: hard + "node-domexception@npm:1.0.0": version: 1.0.0 resolution: "node-domexception@npm:1.0.0" @@ -8456,6 +8545,27 @@ __metadata: languageName: node linkType: hard +"node-gyp@npm:^9.3.0": + version: 9.4.1 + resolution: "node-gyp@npm:9.4.1" + dependencies: + env-paths: ^2.2.0 + exponential-backoff: ^3.1.1 + glob: ^7.1.4 + graceful-fs: ^4.2.6 + make-fetch-happen: ^10.0.3 + nopt: ^6.0.0 + npmlog: ^6.0.0 + rimraf: ^3.0.2 + semver: ^7.3.5 + tar: ^6.1.2 + which: ^2.0.2 + bin: + node-gyp: bin/node-gyp.js + checksum: 8576c439e9e925ab50679f87b7dfa7aa6739e42822e2ad4e26c36341c0ba7163fdf5a946f0a67a476d2f24662bc40d6c97bd9e79ced4321506738e6b760a1577 + languageName: node + linkType: hard + "node-gyp@npm:latest": version: 9.3.1 resolution: "node-gyp@npm:9.3.1" @@ -8520,6 +8630,17 @@ __metadata: languageName: node linkType: hard +"nopt@npm:^5.0.0": + version: 5.0.0 + resolution: "nopt@npm:5.0.0" + dependencies: + abbrev: 1 + bin: + nopt: bin/nopt.js + checksum: d35fdec187269503843924e0114c0c6533fb54bbf1620d0f28b4b60ba01712d6687f62565c55cc20a504eff0fbe5c63e22340c3fad549ad40469ffb611b04f2f + languageName: node + linkType: hard + "nopt@npm:^6.0.0": version: 6.0.0 resolution: "nopt@npm:6.0.0" @@ -8570,6 +8691,18 @@ __metadata: languageName: node linkType: hard +"npmlog@npm:^5.0.1": + version: 5.0.1 + resolution: "npmlog@npm:5.0.1" + dependencies: + are-we-there-yet: ^2.0.0 + console-control-strings: ^1.1.0 + gauge: ^3.0.0 + set-blocking: ^2.0.0 + checksum: 516b2663028761f062d13e8beb3f00069c5664925871a9b57989642ebe09f23ab02145bf3ab88da7866c4e112cafff72401f61a672c7c8a20edc585a7016ef5f + languageName: node + linkType: hard + "npmlog@npm:^6.0.0": version: 6.0.2 resolution: "npmlog@npm:6.0.2" @@ -8598,7 +8731,7 @@ __metadata: languageName: node linkType: hard -"object-assign@npm:^4.1.0": +"object-assign@npm:^4.1.0, object-assign@npm:^4.1.1": version: 4.1.1 resolution: "object-assign@npm:4.1.1" checksum: fcc6e4ea8c7fe48abfbb552578b1c53e0d194086e2e6bbbf59e0a536381a292f39943c6e9628af05b5528aa5e3318bb30d6b2e53cadaf5b8fe9e12c4b69af23f @@ -10025,6 +10158,7 @@ __metadata: base64-arraybuffer: ^1.0.2 commander: ^10.0.0 copyfiles: ^2.4.1 + duckdb-async: ^0.10.0 dynamodb-data-types: ^4.0.1 express: ^4.18.2 express-async-errors: ^3.1.1 @@ -10721,7 +10855,7 @@ __metadata: languageName: node linkType: hard -"wide-align@npm:^1.1.0, wide-align@npm:^1.1.5": +"wide-align@npm:^1.1.0, wide-align@npm:^1.1.2, wide-align@npm:^1.1.5": version: 1.1.5 resolution: "wide-align@npm:1.1.5" dependencies: