Merge pull request #1103 from lowcoder-org/versions-for-datasources

Versions for API based Datasources

Author: FalkWolsky

server/node-service/src/common/util.test.ts

@@ -0,0 +1,112 @@
+import { specsToOptions, version2spec } from "./util";
+
+describe('version2spec', () => {
+  test('should return the spec for the given version', () => {
+    const specs = {
+      v1: 'spec for version 1',
+      v2: 'spec for version 2',
+      v3: 'spec for version 3'
+    };
+
+    expect(version2spec(specs, 'v2')).toBe('spec for version 2');
+  });
+
+  test('should return the first spec if version is undefined', () => {
+    const specs = {
+      v1: 'spec for version 1',
+      v2: 'spec for version 2',
+      v3: 'spec for version 3'
+    };
+
+    expect(version2spec(specs, undefined)).toBe('spec for version 1');
+  });
+
+  test('should return the first spec if version is an empty string', () => {
+    const specs = {
+      v1: 'spec for version 1',
+      v2: 'spec for version 2',
+      v3: 'spec for version 3'
+    };
+
+    expect(version2spec(specs, "")).toBe('spec for version 1');
+  });
+
+  test('should return undefined if specs is an empty object and version is undefined', () => {
+    const specs = {};
+
+    expect(version2spec(specs, undefined)).toBeUndefined();
+  });
+
+  test('should return undefined if specs is an empty object and version is an empty string', () => {
+    const specs = {};
+
+    expect(version2spec(specs, "")).toBeUndefined();
+  });
+
+  test('should return undefined if the specified version does not exist in specs', () => {
+    const specs = {
+      v1: 'spec for version 1',
+      v2: 'spec for version 2',
+      v3: 'spec for version 3'
+    };
+
+    expect(version2spec(specs, 'v4')).toBeUndefined();
+  });
+});
+
+describe('specsToOptions', () => {
+  test('should convert specs object to options array', () => {
+    const specs = {
+      color: 'red',
+      size: 'large',
+      weight: 'light'
+    };
+
+    const expectedOptions = [
+      { value: 'color', label: 'color' },
+      { value: 'size', label: 'size' },
+      { value: 'weight', label: 'weight' }
+    ];
+
+    expect(specsToOptions(specs)).toEqual(expectedOptions);
+  });
+
+  test('should return an empty array if specs object is empty', () => {
+    const specs = {};
+    const expectedOptions: any[] = [];
+
+    expect(specsToOptions(specs)).toEqual(expectedOptions);
+  });
+
+  test('should handle specs object with non-string values', () => {
+    const specs = {
+      color: 'red',
+      size: 42,
+      available: true
+    };
+
+    const expectedOptions = [
+      { value: 'color', label: 'color' },
+      { value: 'size', label: 'size' },
+      { value: 'available', label: 'available' }
+    ];
+
+    expect(specsToOptions(specs)).toEqual(expectedOptions);
+  });
+
+  test('should handle specs object with numeric keys', () => {
+    const specs = {
+      1: 'one',
+      2: 'two',
+      3: 'three'
+    };
+
+    const expectedOptions = [
+      { value: '1', label: '1' },
+      { value: '2', label: '2' },
+      { value: '3', label: '3' }
+    ];
+
+    expect(specsToOptions(specs)).toEqual(expectedOptions);
+  });
+});

server/node-service/src/common/util.ts

@@ -1,5 +1,9 @@
 import yaml from "yaml";
 import fs from "fs";
+import { MultiOpenApiSpecItem } from "../plugins/openApi/parse";
+import path from "path";
+import { appendTags } from "../plugins/openApi/util";
+import _ from "lodash";
 
 export function kvToRecord(
   kvs: { key: string; value: string }[],
@@ -85,3 +89,45 @@ export function safeJsonStringify(data: any) {
     return null;
   }
 }
+
+export function specsToOptions(specs: any) {
+  return Object.keys(specs).map(k => ({value: k, label: k}));
+}
+
+export function version2spec(specs: any, version: any) {
+  if(version == undefined || version == "") {
+    const keys = Object.keys(specs);
+    if(keys.length == 0) return;
+    return specs[keys[0]];
+  }
+  return specs[version];
+}
+
+function genTagFromFileName(name: string) {
+  const fileName = name.replace(/\.yaml|twilio_|\.json/g, "");
+  const parts = fileName.split("_");
+  return parts.reduce((a, b) => {
+    if (/v\d+/.test(b)) {
+      return `${a}(${b})`;
+    }
+    return a + _.upperFirst(b);
+  }, "");
+}
+
+export function dirToSpecList(specDir: string) {
+  const specList: MultiOpenApiSpecItem[] = [];
+
+  const start = performance.now();
+  const specFiles = fs.readdirSync(specDir);
+  specFiles.forEach((specFile) => {
+    const spec = readYaml(path.join(specDir, specFile));
+    const tag = genTagFromFileName(specFile);
+    appendTags(spec, tag);
+    specList.push({
+      id: tag,
+      spec,
+    });
+  });
+  logger.info("spec list loaded %s, duration: %d ms",specDir, performance.now() - start);
+  return specList;
+}

server/node-service/src/plugins/asana/index.ts

@@ -1,4 +1,4 @@
-import { readYaml } from "../../common/util";
+import { readYaml, specsToOptions, version2spec } from "../../common/util";
 import _ from "lodash";
 import path from "path";
 import { OpenAPIV3, OpenAPI } from "openapi-types";
@@ -8,7 +8,9 @@ import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 
 
 const spec = readYaml(path.join(__dirname, "./asana.spec.yaml"));
-
+const specs = {
+  "v1.0": spec,
+}
 
 const dataSourceConfig = {
   type: "dataSource",
@@ -23,7 +25,15 @@ const dataSourceConfig = {
       "key": "personalAccessToken.value",
       "label": "Token",
       "tooltip": "A [personal access token](https://developers.asana.com/docs/personal-access-token) allows access to the api for the user who created it. This should be kept a secret and be treated like a password.",
-    }
+    },
+    {
+      label: "Spec Version",
+      key: "specVersion",
+      type: "select",
+      tooltip: "Version of the spec file.",
+      placeholder: "v1.0",
+      options: specsToOptions(specs)
+    },
   ]
 } as const;
 
@@ -41,8 +51,8 @@ const asanaPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
   icon: "asana.svg",
   category: "Project Management",
   dataSourceConfig,
-  queryConfig: async () => {
-    const { actions, categories } = await parseOpenApi(spec as OpenAPI.Document, parseOptions);
+  queryConfig: async (data) => {
+    const { actions, categories } = await parseOpenApi(version2spec(specs, data.specVersion) as OpenAPI.Document, parseOptions);
     return {
       type: "query",
       label: "Action",
@@ -58,8 +68,9 @@ const asanaPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
       url: "",
       serverURL: "",
       dynamicParamsConfig: dataSourceConfig,
+      specVersion: dataSourceConfig.specVersion,
     };
-    return runOpenApi(actionData, runApiDsConfig, spec as OpenAPIV3.Document);
+    return runOpenApi(actionData, runApiDsConfig, version2spec(specs, dataSourceConfig.specVersion) as OpenAPIV3.Document);
   },
 };
 

server/node-service/src/plugins/circleCi/index.ts

@@ -5,6 +5,10 @@ import { runOpenApi } from "../openApi";
 import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 
 import spec from "./circleCi.spec.json";
+import { specsToOptions, version2spec } from "../../common/util";
+const specs = {
+  "v1.0": spec,
+}
 
 const dataSourceConfig = {
   type: "dataSource",
@@ -16,6 +20,14 @@ const dataSourceConfig = {
       tooltip:
         "[Personal API Token](https://circleci.com/docs/managing-api-tokens/#creating-a-personal-api-token)",
     },
+    {
+      label: "Spec Version",
+      key: "specVersion",
+      type: "select",
+      tooltip: "Version of the spec file.",
+      placeholder: "v1.0",
+      options: specsToOptions(specs)
+    },
   ],
 } as const;
 
@@ -33,9 +45,9 @@ const circleCiPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
   icon: "circleCI.svg",
   category: "DevOps",
   dataSourceConfig,
-  queryConfig: async () => {
+  queryConfig: async (data) => {
     const { actions, categories } = await parseOpenApi(
-      spec as unknown as OpenAPI.Document,
+      version2spec(specs, data.specVersion) as unknown as OpenAPI.Document,
       parseOptions
     );
     return {
@@ -53,8 +65,9 @@ const circleCiPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
       url: "",
       serverURL: "",
       dynamicParamsConfig: dataSourceConfig,
+      specVersion: dataSourceConfig.specVersion,
     };
-    return runOpenApi(actionData, runApiDsConfig, spec as unknown as OpenAPIV3.Document);
+    return runOpenApi(actionData, runApiDsConfig, version2spec(specs, dataSourceConfig.specVersion) as unknown as OpenAPIV3.Document);
   },
 };
 

server/node-service/src/plugins/cloudinary/index.ts

@@ -1,4 +1,4 @@
-import { readYaml } from "../../common/util";
+import { readYaml, specsToOptions, version2spec } from "../../common/util";
 import _ from "lodash";
 import path from "path";
 import { OpenAPI } from "openapi-types";
@@ -14,6 +14,9 @@ const specList = [
   { spec: adminApiSpec, id: "admin" },
   { spec: uploadApiSpec, id: "upload" },
 ];
+const specs = {
+  "v1.0": specList,
+}
 
 const dataSourceConfig = {
   type: "dataSource",
@@ -32,6 +35,14 @@ const dataSourceConfig = {
       tooltip: "Basic auth password",
       placeholder: "<Basic Auth Password>",
     },
+    {
+      label: "Spec Version",
+      key: "specVersion",
+      type: "select",
+      tooltip: "Version of the spec file.",
+      placeholder: "v1.0",
+      options: specsToOptions(specs)
+    },
   ],
 } as const;
 
@@ -46,18 +57,18 @@ const parseOptions: ParseOpenApiOptions = {
 
 type DataSourceConfigType = ConfigToType<typeof dataSourceConfig>;
 
-let queryConfig: QueryConfig;
+let queryConfig: any = {};
 
 const cloudinaryPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
   id: "cloudinary",
   name: "Cloudinary",
   icon: "cloudinary.svg",
   category: "Assets",
   dataSourceConfig,
-  queryConfig: async () => {
-    if (!queryConfig) {
-      const { actions, categories } = await parseMultiOpenApi(specList, parseOptions);
-      queryConfig = {
+  queryConfig: async (data) => {
+    if (!queryConfig[data.specVersion as keyof typeof queryConfig]) {
+      const { actions, categories } = await parseMultiOpenApi(version2spec(specs, data.specVersion), parseOptions);
+      queryConfig[data.specVersion as keyof typeof queryConfig] = {
         type: "query",
         label: "Action",
         categories: {
@@ -67,15 +78,16 @@ const cloudinaryPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
         actions,
       };
     }
-    return queryConfig;
+    return queryConfig[data.specVersion as keyof typeof queryConfig];
   },
   run: function (actionData, dataSourceConfig): Promise<any> {
     const runApiDsConfig = {
       url: "",
       serverURL: "",
       dynamicParamsConfig: dataSourceConfig,
+      specVersion: dataSourceConfig.specVersion,
     };
-    return runOpenApi(actionData, runApiDsConfig, specList);
+    return runOpenApi(actionData, runApiDsConfig, version2spec(specs, dataSourceConfig.specVersion));
   },
 };
 

server/node-service/src/plugins/couchdb/index.ts

@@ -4,7 +4,10 @@ import { ConfigToType, DataSourcePlugin } from "lowcoder-sdk/dataSource";
 import { runOpenApi } from "../openApi";
 import { defaultParseOpenApiOptions, parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 import spec from "./CouchDB-3.1.1-resolved.json";
-
+import { specsToOptions, version2spec } from "../../common/util";
+const specs = {
+  "v1.0": spec,
+}
 const dataSourceConfig = {
   type: "dataSource",
   params: [
@@ -38,6 +41,14 @@ const dataSourceConfig = {
       tooltip: "",
       placeholder: "",
     },
+    {
+      label: "Spec Version",
+      key: "specVersion",
+      type: "select",
+      tooltip: "Version of the spec file.",
+      placeholder: "v1.0",
+      options: specsToOptions(specs)
+    },
   ],
 } as const;
 
@@ -59,8 +70,8 @@ const couchdbPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
   icon: "couchdb.svg",
   category: "database",
   dataSourceConfig,
-  queryConfig: async () => {
-    const { actions, categories } = await parseOpenApi(spec as OpenAPI.Document, parseOptions);
+  queryConfig: async (data) => {
+    const { actions, categories } = await parseOpenApi(version2spec(specs, data.specVersion) as OpenAPI.Document, parseOptions);
     return {
       type: "query",
       label: "Operation",
@@ -77,8 +88,9 @@ const couchdbPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
       url: "",
       serverURL: serverURL,
       dynamicParamsConfig: otherDataSourceConfig,
+      specVersion: dataSourceConfig.specVersion
     };
-    return runOpenApi(actionData, runApiDsConfig, spec as OpenAPIV2.Document);
+    return runOpenApi(actionData, runApiDsConfig, version2spec(specs, dataSourceConfig.specVersion) as OpenAPIV2.Document);
   },
 };