refactor openApi parse multi spec

Author: sunlei

server/node-service/src/plugins/cloudinary/index.ts

@@ -2,14 +2,18 @@ import { readYaml } from "../../common/util";
 import _ from "lodash";
 import path from "path";
 import { OpenAPI } from "openapi-types";
-import { ConfigToType, DataSourcePlugin } from "openblocks-sdk/dataSource";
+import { ConfigToType, DataSourcePlugin, QueryConfig } from "openblocks-sdk/dataSource";
 import { runOpenApi } from "../openApi";
-import { parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
+import { parseMultiOpenApi, parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 import { mergeCategories } from "../../plugins/openApi/util";
 
 // all OpenAPI specs generated from https://www.postman.com/cloudinaryteam/workspace/programmable-media/collection/16080251-d28221d4-b2f8-4244-a4eb-7e77abe3a857?ctx=documentation
 const adminApiSpec = readYaml(path.join(__dirname, "./adminApi.spec.yaml"));
 const uploadApiSpec = readYaml(path.join(__dirname, "./uploadApi.spec.yaml"));
+const specList = [
+  { spec: adminApiSpec, id: "admin" },
+  { spec: uploadApiSpec, id: "upload" },
+];
 
 const dataSourceConfig = {
   type: "dataSource",
@@ -42,34 +46,36 @@ const parseOptions: ParseOpenApiOptions = {
 
 type DataSourceConfigType = ConfigToType<typeof dataSourceConfig>;
 
+let queryConfig: QueryConfig;
+
 const cloudinaryPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
   id: "cloudinary",
   name: "cloudinary",
   icon: "cloudinary.svg",
   category: "api",
   dataSourceConfig,
   queryConfig: async () => {
-    const [parsedAdminApi, parsedUploadApi] = await Promise.all([
-      parseOpenApi(adminApiSpec, parseOptions),
-      parseOpenApi(uploadApiSpec, parseOptions),
-    ]);
-    return {
-      type: "query",
-      label: "Action",
-      categories: {
-        label: "Resources",
-        items: mergeCategories(parsedUploadApi.categories, parsedAdminApi.categories),
-      },
-      actions: parsedAdminApi.actions.concat(parsedUploadApi.actions),
-    };
+    if (!queryConfig) {
+      const { actions, categories } = await parseMultiOpenApi(specList, parseOptions);
+      queryConfig = {
+        type: "query",
+        label: "Action",
+        categories: {
+          label: "Resources",
+          items: categories,
+        },
+        actions,
+      };
+    }
+    return queryConfig;
   },
   run: function (actionData, dataSourceConfig): Promise<any> {
     const runApiDsConfig = {
       url: "",
       serverURL: "",
       dynamicParamsConfig: dataSourceConfig,
     };
-    return runOpenApi(actionData, runApiDsConfig, [adminApiSpec, uploadApiSpec]);
+    return runOpenApi(actionData, runApiDsConfig, specList);
   },
 };
 

server/node-service/src/plugins/did/index.ts

@@ -4,17 +4,21 @@ import path from "path";
 import { OpenAPIV3, OpenAPI } from "openapi-types";
 import { QueryConfig, ConfigToType, DataSourcePlugin } from "openblocks-sdk/dataSource";
 import { runOpenApi } from "../openApi";
-import { parseMultiOpenApi, parseOpenApi, ParseOpenApiOptions } from "../openApi/parse";
+import { MultiOpenApiSpecItem, parseMultiOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 import { appendTags } from "../../plugins/openApi/util";
 import { readdirSync } from "fs";
 
-const specList: OpenAPI.Document[] = [];
+const specList: MultiOpenApiSpecItem[] = [];
 const specFiles = readdirSync(path.join(__dirname, "./did.spec"));
 const start = performance.now();
 specFiles.forEach((specFile) => {
   const spec = readYaml(path.join(__dirname, "./did.spec", specFile));
-  appendTags(spec, _.upperFirst(specFile.replace(".json", "")));
-  specList.push(spec);
+  const tag = _.upperFirst(specFile.replace(".json", ""));
+  appendTags(spec, tag);
+  specList.push({
+    spec,
+    id: tag,
+  });
 });
 logger.info("did spec list loaded, duration: %d ms", performance.now() - start);
 

server/node-service/src/plugins/openApi/index.ts

@@ -1,7 +1,7 @@
 import SwaggerClient from "swagger-client";
 import SwaggerParser from "@apidevtools/swagger-parser";
 import { ConfigToType, DataSourcePlugin } from "openblocks-sdk/dataSource";
-import { authParamsConfig, parseOpenApi, retrieveSpec } from "./parse";
+import { authParamsConfig, MultiOpenApiSpecItem, parseOpenApi, retrieveSpec } from "./parse";
 import {
   extractSecurityParams,
   findOperation,
@@ -53,21 +53,32 @@ interface ActionDataType {
 export async function runOpenApi(
   actionData: ActionDataType,
   dataSourceConfig: DataSourceDataType,
-  spec: OpenAPI.Document | OpenAPI.Document[],
+  spec: OpenAPI.Document | MultiOpenApiSpecItem[],
   defaultHeaders?: Record<string, string>
 ) {
-  const specList = Array.isArray(spec) ? spec : [spec];
-  const definitions = await Promise.all(specList.map((i) => SwaggerParser.dereference(i)));
+  const specList = Array.isArray(spec) ? spec : [{ spec, id: "" }];
+  const definitions = await Promise.all(
+    specList.map(async ({ id, spec }) => {
+      const deRefedSpec = await SwaggerParser.dereference(spec);
+      return {
+        def: deRefedSpec,
+        id,
+      };
+    })
+  );
   const { actionName, ...otherActionData } = actionData;
   const { serverURL } = dataSourceConfig;
 
   let definition: OpenAPI.Document | undefined;
   let operation;
+  let realOperationId;
 
-  for (const def of definitions) {
-    operation = findOperation(actionName, def);
-    if (operation) {
+  for (const { id, def } of definitions) {
+    const ret = findOperation(actionName, def, id);
+    if (ret) {
       definition = def;
+      operation = ret.operation;
+      realOperationId = ret.realOperationId;
       break;
     }
   }
@@ -94,7 +105,7 @@ export async function runOpenApi(
     const securities = extractSecurityParams(dataSourceConfig.dynamicParamsConfig, definition);
     const response = await SwaggerClient.execute({
       spec: definition,
-      operationId: actionName,
+      operationId: realOperationId,
       parameters,
       requestBody,
       securities,

server/node-service/src/plugins/openApi/parse.ts

@@ -14,6 +14,7 @@ import _ from "lodash";
 import SwaggerClient from "swagger-client";
 import {
   appendCategories,
+  getOperationId,
   isOas3HttpMethods,
   isOas3RefObject,
   isSwagger2HttpMethods,
@@ -175,11 +176,18 @@ interface OpenAPIParseResult {
   categories: ActionCategory[];
 }
 
+export type MultiOpenApiSpecItem = {
+  spec: OpenAPI.Document | string;
+  id: string;
+};
+
 export async function parseMultiOpenApi(
-  spec: (OpenAPI.Document | string)[],
-  options?: ParseOpenApiOptions
+  specList: MultiOpenApiSpecItem[],
+  options?: Omit<ParseOpenApiOptions, "specId">
 ): Promise<OpenAPIParseResult> {
-  const results = await Promise.all(spec.map((i) => parseOpenApi(i, options)));
+  const results = await Promise.all(
+    specList.map(({ id, spec }) => parseOpenApi(spec, options, id))
+  );
   const reducedResults = results.reduce(
     (a, b) => {
       return {
@@ -194,7 +202,12 @@ export async function parseMultiOpenApi(
 
 export async function parseOpenApi(
   specJsonOrObj: OpenAPI.Document | string,
-  options?: ParseOpenApiOptions
+  options?: ParseOpenApiOptions,
+  /**
+   * when parse multi open api this field is required.
+   * This will be used to ensure operationId is unique among all operations in all specs.
+   */
+  specId?: string
 ): Promise<OpenAPIParseResult> {
   let spec = specJsonOrObj;
   if (typeof specJsonOrObj === "string") {
@@ -245,7 +258,7 @@ export async function parseOpenApi(
         return;
       }
 
-      const operationId: string = SwaggerClient.helpers.opId(operation, path, httpMethod);
+      const operationId: string = getOperationId(operation, path, httpMethod, specId);
       if (!operationId) {
         console.warn("can not get operationId:", operation);
         return;

server/node-service/src/plugins/openApi/util.ts

@@ -4,6 +4,7 @@ import { File } from "formdata-node";
 import { OpenAPI, OpenAPIV2, OpenAPIV3, OpenAPIV3_1 } from "openapi-types";
 import swaggerClient from "swagger-client";
 import { ActionCategory } from "openblocks-sdk/dataSource";
+import SwaggerClient from "swagger-client";
 
 export const MediaTypeOctetStream = "application/octet-stream";
 export const MediaTypeUrlEncoded = "application/x-www-form-urlencoded";
@@ -186,7 +187,7 @@ export function findOas3FilePropertiesFromSchema(
   });
 }
 
-export function findOperation(id: string, spec: OpenAPI.Document) {
+export function findOperation(id: string, spec: OpenAPI.Document, specId: string = "") {
   if (!spec.paths) {
     return null;
   }
@@ -197,8 +198,11 @@ export function findOperation(id: string, spec: OpenAPI.Document) {
     }
     for (const method of Object.keys(pathObj)) {
       const operation: OpenAPI.Operation = (pathObj as any)[method];
-      if (swaggerClient.helpers.opId(operation, path, method) === id) {
-        return operation;
+      if (getOperationId(operation, path, method, specId) === id) {
+        return {
+          operation,
+          realOperationId: getOperationId(operation, path, method),
+        };
       }
     }
   }
@@ -345,13 +349,13 @@ export function getSchemaType(
   return schema.type;
 }
 
-export function getSchemaExample(
-  schema?:
-    | OpenAPIV2.SchemaObject
-    | OpenAPIV2.ReferenceObject
-    | OpenAPIV3.SchemaObject
-    | OpenAPIV3.ReferenceObject
-): any {
+type SchemaObject =
+  | OpenAPIV2.SchemaObject
+  | OpenAPIV2.ReferenceObject
+  | OpenAPIV3.SchemaObject
+  | OpenAPIV3.ReferenceObject;
+
+export function getSchemaExample(schema?: SchemaObject): any {
   if (!schema || isOas3RefObject(schema) || isSwagger2RefObject(schema)) {
     return;
   }
@@ -376,6 +380,22 @@ export function getSchemaExample(
     );
     return _.isNil(itemExample) ? [] : [itemExample];
   }
+  if (schema.oneOf && schema.oneOf.length > 0) {
+    const firstSchema = schema.oneOf[0];
+    return getSchemaExample(firstSchema as SchemaObject);
+  }
+  if (schema.anyOf && schema.anyOf.length > 0) {
+    const firstSchema = schema.anyOf[0];
+    return getSchemaExample(firstSchema as SchemaObject);
+  }
+  if (schema.allOf && schema.allOf.length > 0) {
+    return (schema.allOf as SchemaObject[]).reduce((a: any, b: SchemaObject) => {
+      return {
+        ...a,
+        ...getSchemaExample(b),
+      };
+    }, {});
+  }
   let ret: any;
   if (schema.properties) {
     Object.entries(schema.properties).forEach(([name, def]) => {
@@ -419,3 +439,16 @@ export function appendCategories(a: ActionCategory[], b: ActionCategory[]) {
     a.push(i);
   });
 }
+
+export function getOperationId(
+  operation: OpenAPI.Operation,
+  path: string,
+  method: string,
+  specId: string = ""
+) {
+  const operationId: string = SwaggerClient.helpers.opId(operation, path, method);
+  if (!operationId) {
+    return "";
+  }
+  return operationId + specId;
+}

server/node-service/src/plugins/twilio/index.ts

@@ -3,9 +3,9 @@ import _ from "lodash";
 import fs from "fs";
 import path from "path";
 import { OpenAPI } from "openapi-types";
-import { ConfigToType, DataSourcePlugin } from "openblocks-sdk/dataSource";
+import { ConfigToType, DataSourcePlugin, QueryConfig } from "openblocks-sdk/dataSource";
 import { runOpenApi } from "../openApi";
-import { parseMultiOpenApi, ParseOpenApiOptions } from "../openApi/parse";
+import { MultiOpenApiSpecItem, parseMultiOpenApi, ParseOpenApiOptions } from "../openApi/parse";
 import { appendTags } from "../openApi/util";
 
 function genTagFromFileName(name: string) {
@@ -19,14 +19,18 @@ function genTagFromFileName(name: string) {
   }, "");
 }
 
-const specList: OpenAPI.Document[] = [];
+const specList: MultiOpenApiSpecItem[] = [];
 
 const start = performance.now();
 const specFiles = fs.readdirSync(path.join(__dirname, "./twilio.spec"));
 specFiles.forEach((specFile) => {
   const spec = readYaml(path.join(__dirname, "./twilio.spec", specFile));
-  appendTags(spec, genTagFromFileName(specFile));
-  specList.push(spec);
+  const tag = genTagFromFileName(specFile);
+  appendTags(spec, tag);
+  specList.push({
+    id: tag,
+    spec,
+  });
 });
 logger.info("twilio spec list loaded, duration: %d ms", performance.now() - start);
 
@@ -54,6 +58,8 @@ const parseOptions: ParseOpenApiOptions = {
   },
 };
 
+let queryConfig: QueryConfig;
+
 type DataSourceConfigType = ConfigToType<typeof dataSourceConfig>;
 
 const twilioPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
@@ -62,18 +68,23 @@ const twilioPlugin: DataSourcePlugin<any, DataSourceConfigType> = {
   icon: "twilio.svg",
   category: "api",
   dataSourceConfig,
+
   queryConfig: async () => {
-    const { actions, categories } = await parseMultiOpenApi(specList, parseOptions);
-    return {
-      type: "query",
-      label: "Action",
-      categories: {
-        label: "Resources",
-        items: categories,
-      },
-      actions,
-    };
+    if (!queryConfig) {
+      const { actions, categories } = await parseMultiOpenApi(specList, parseOptions);
+      queryConfig = {
+        type: "query",
+        label: "Action",
+        categories: {
+          label: "Resources",
+          items: categories,
+        },
+        actions,
+      };
+    }
+    return queryConfig;
   },
+
   run: function (actionData, dataSourceConfig): Promise<any> {
     const runApiDsConfig = {
       url: "",