Merge pull request #294 from openblocks-dev/0308-datasources

0308 datasources

Author: hanshuai

server/node-service/jest.config.js

@@ -4,4 +4,5 @@ module.exports = {
   testEnvironment: "node",
   testTimeout: 10 * 60000,
   testPathIgnorePatterns: ["/node_modules/", "/build/"],
+  setupFiles: ["./jest.setup.ts"],
 };

server/node-service/jest.setup.ts

@@ -0,0 +1 @@
+import "./src/common/logger";

server/node-service/nodemon.json

@@ -0,0 +1,13 @@
+{
+  "restartable": "rs",
+  "ignore": [".git", "node_modules/**/node_modules"],
+  "verbose": true,
+  "execMap": {
+    "ts": "node --require ts-node/register"
+  },
+  "watch": ["src/"],
+  "env": {
+    "NODE_ENV": "development"
+  },
+  "ext": "js,json,ts"
+}

server/node-service/package.json

@@ -7,18 +7,22 @@
   },
   "main": "src/server.ts",
   "scripts": {
-    "dev": "nodemon --files src/server.ts",
+    "dev": "nodemon src/server.ts",
+    "debug": "nodemon --inspect src/server.ts",
     "start": "node ./build/src/server.js",
     "test": "jest",
-    "build": "rm -rf build/ && tsc && yarn run copy",
-    "copy": "copyfiles -u 1 src/**/*.!(ts|test.ts) build/src"
+    "copy": "copyfiles -u 1 src/**/*.!(ts|test.ts) build/src",
+    "genOpenApiPlugin": "npx ts-node --files ./scripts/openApiDataSourceGen.ts",
+    "build": "rm -rf build/ && yarn test && tsc && cp -r src/static build/src/static"
   },
   "devDependencies": {
     "@types/jest": "^29.2.4",
+    "commander": "^10.0.0",
     "copyfiles": "^2.4.1",
     "jest": "^29.3.1",
     "nock": "^13.3.0",
     "nodemon": "^2.0.20",
+    "postman-to-openapi": "^3.0.1",
     "svgo": "^3.0.2",
     "ts-jest": "^29.0.3",
     "ts-node": "^10.9.1"
@@ -30,30 +34,42 @@
     "@aws-sdk/client-lambda": "^3.272.0",
     "@aws-sdk/client-s3": "^3.238.0",
     "@aws-sdk/s3-request-presigner": "^3.241.0",
+    "@google-cloud/bigquery": "^6.1.0",
     "@google-cloud/storage": "^6.9.3",
+    "@supabase/supabase-js": "^2.10.0",
     "@types/axios": "^0.14.0",
     "@types/express": "^4.17.14",
     "@types/jsonpath": "^0.2.0",
     "@types/lodash": "^4.14.190",
     "@types/morgan": "^1.9.3",
     "@types/node": "^18.11.18",
+    "@types/node-fetch": "^2.6.2",
     "axios": "^1.2.0",
+    "base64-arraybuffer": "^1.0.2",
     "dynamodb-data-types": "^4.0.1",
     "express": "^4.18.2",
     "express-async-errors": "^3.1.1",
     "firebase-admin": "^11.5.0",
     "formdata-node": "4",
+    "graphql": "^16.6.0",
+    "graphql-request": "^5.1.0",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
     "loglevel": "^1.8.1",
     "morgan": "^1.10.0",
+    "node-fetch": "2",
     "openapi-types": "^12.1.0",
     "openblocks-core": "^0.0.7",
-    "openblocks-sdk": "^0.0.35",
+    "openblocks-sdk": "0.0.38",
+    "pino": "^8.11.0",
+    "proxy-agent": "^5.0.0",
     "stylis": "^4.1.3",
     "swagger-client": "^3.18.5",
     "typescript": "^4.9.3",
     "yaml": "^2.2.1"
   },
+  "resolutions": {
+    "@apidevtools/json-schema-ref-parser": "9.0.7"
+  },
   "packageManager": "yarn@3.3.1"
 }

server/node-service/scripts/openApiDataSourceGen.ts

@@ -0,0 +1,114 @@
+import { readYaml } from "../src/common/util";
+import fs from "fs";
+import _ from "lodash";
+import path from "path";
+import { authParamsConfig, retrieveSpec } from "../src/plugins/openApi/parse";
+import postManToOpenApi from "postman-to-openapi";
+import { program } from "commander";
+
+interface Options {
+  force: boolean;
+  name: string;
+  url: string;
+  postMan: boolean;
+  postManCollectionFile: string;
+  id?: string;
+}
+
+async function gen(options: Options) {
+  const { postMan, force, postManCollectionFile, name, url: specUrl, id: pluginId } = options;
+  const id = pluginId ?? _.camelCase(name);
+  const pluginDir = path.join(path.dirname(__dirname), "src/plugins", id);
+  const pluginEntryFile = path.join(pluginDir, "index.ts");
+  const pluginSpecYamlFile = path.join(pluginDir, `${id}.spec.yaml`);
+  const pluginSpecJsonFile = path.join(pluginDir, `${id}.spec.json`);
+  const pluginDefaultCollectionFile = path.join(pluginDir, `${id}.collection.json`);
+
+  console.info();
+  console.info("start generate plugin, id:", id, "name:", name);
+
+  if (postMan) {
+    console.info("is PostMan Collection start transforming...");
+    const collection = postManCollectionFile ?? pluginDefaultCollectionFile;
+    await postManToOpenApi(collection, pluginSpecYamlFile, { defaultTag: "General" });
+  }
+
+  if (!fs.existsSync(pluginDir)) {
+    fs.mkdirSync(pluginDir, { recursive: true });
+    console.info(`plugin dir ${id} created.`);
+  }
+
+  if (!force && fs.existsSync(pluginEntryFile)) {
+    console.info(`plugin: ${id} is already existed.`);
+    return;
+  }
+
+  // 1. fetch spec url
+  let spec: any;
+  const isYamlSpecExist = fs.existsSync(pluginSpecYamlFile);
+  const isJsonSpecExist = fs.existsSync(pluginSpecJsonFile);
+  if (!isYamlSpecExist && !isJsonSpecExist) {
+    if (!specUrl) {
+      console.error("specUrl is required to fetch OpenAPI spec.");
+      return;
+    }
+    console.info(`start fetching:`, specUrl);
+    const { spec } = await retrieveSpec(specUrl);
+    fs.writeFileSync(pluginSpecJsonFile, JSON.stringify(spec, null, 2));
+    console.info(`${name}spec.json saved`);
+  } else {
+    if (isJsonSpecExist) {
+      const specJson = fs.readFileSync(pluginSpecJsonFile).toString();
+      spec = JSON.parse(specJson);
+      console.info("got spec from json file:", pluginSpecJsonFile);
+    }
+    if (isYamlSpecExist) {
+      spec = readYaml(pluginSpecYamlFile);
+      console.info("got spec from yaml file:", pluginSpecYamlFile);
+    }
+  }
+
+  if (!spec) {
+    console.error("can not get spec");
+    return;
+  }
+
+  // 2. get data source params
+  const dataSourceParams = await authParamsConfig(spec);
+
+  // 3. gen code
+  const template = fs
+    .readFileSync(path.join(__dirname, "./openApiDataSourceTemplate.tpl"))
+    .toString();
+  const data = {
+    id,
+    name,
+    isJsonSpec: isJsonSpecExist,
+    isYamlSpec: isYamlSpecExist,
+    dataSourceParams: JSON.stringify(dataSourceParams, null, 2),
+  };
+  const compiledTemplate = _.template(template);
+  const code = compiledTemplate(data);
+  fs.writeFileSync(pluginEntryFile, code);
+  console.info("success generate plugin:", pluginDir);
+  console.info();
+}
+
+const plugins = [
+  // ["Jira", "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json"],
+  [],
+];
+
+program
+  .option("--force")
+  .option("--post-man")
+  .option("-f, --post-man-collection-file [postman collection file path]")
+  .option("-n, --name <char>")
+  .option("-i, --id [plugin id]")
+  .option("--url [spec-download-url]");
+
+program.parse();
+
+const options = program.opts<Options>();
+
+gen(options);

server/node-service/scripts/openApiDataSourceTemplate.tpl

@@ -0,0 +1,57 @@
+import { readYaml } from "../../common/util";
+import _ from "lodash";
+import path from "path";
+import { OpenAPIV3, OpenAPI } from "openapi-types";
+import { ConfigToType, DataSourcePlugin } from "openblocks-sdk/dataSource";
+import { runOpenApi } from "../openApi";
+import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
+<% if (isJsonSpec) {%>
+import spec from './<%=id %>.spec.json';
+<% } %>
+
+<% if (isYamlSpec) {%>
+const spec = readYaml(path.join(__dirname, "./<%=id %>.spec.yaml"));
+<% } %>
+
+const dataSourceConfig = {
+  type: "dataSource",
+  params: <%=dataSourceParams %>
+} as const;
+
+const parseOptions: ParseOpenApiOptions = {
+  actionLabel: (method: string, path: string, operation: OpenAPI.Operation) => {
+    return _.upperFirst(operation.operationId || "");
+  },
+};
+
+type DataSourceConfigType = ConfigToType<typeof dataSourceConfig>;
+
+const <%=id %>Plugin: DataSourcePlugin<any, DataSourceConfigType> = {
+  id: "<%=id %>",
+  name: "<%=name %>",
+  icon: "<%=id %>.svg",
+  category: "api",
+  dataSourceConfig,
+  queryConfig: async () => {
+    const { actions, categories } = await parseOpenApi(spec<% if (isJsonSpec) { %> as unknown<% } %> as OpenAPI.Document, parseOptions);
+    return {
+      type: "query",
+      label: "Action",
+      categories: {
+        label: "Resources",
+        items: categories,
+      },
+      actions,
+    };
+  },
+  run: function (actionData, dataSourceConfig): Promise<any> {
+    const runApiDsConfig = {
+      url: "",
+      serverURL: "",
+      dynamicParamsConfig: dataSourceConfig,
+    };
+    return runOpenApi(actionData, runApiDsConfig, spec as OpenAPIV3.Document);
+  },
+};
+
+export default <%=id %>Plugin;

server/node-service/src/common/fetch.ts

@@ -0,0 +1,6 @@
+import nodeFetch, { RequestInit, RequestInfo } from "node-fetch";
+import proxyAgent from "proxy-agent";
+
+export function fetch(url: RequestInfo, init?: RequestInit) {
+  return nodeFetch(url, { agent: proxyAgent(), ...init });
+}

server/node-service/src/common/logger.ts

@@ -0,0 +1,3 @@
+import pino from "pino";
+
+global.logger = pino();

server/node-service/src/common/util.ts

@@ -1,6 +1,20 @@
 import yaml from "yaml";
 import fs from "fs";
 
+export function kvToRecord(
+  kvs: { key: string; value: string }[],
+  trimEmpty: boolean = true
+): Record<string, string> {
+  const ret: Record<string, string> = {};
+  (kvs || []).forEach(({ key, value }) => {
+    if (trimEmpty && !value) {
+      return;
+    }
+    ret[key] = value;
+  });
+  return ret;
+}
+
 export function toString(value: any): string {
   if (value === undefined || value === null) {
     return "";

server/node-service/src/global.d.ts

@@ -0,0 +1,7 @@
+import { Logger } from "pino";
+
+declare global {
+  var logger: Logger;
+}
+
+export {};