Merge remote-tracking branch 'origin/main' into koesie10/data-extension-editor-generate-flow-model

Author: koesie10

.github/workflows/main.yml

@@ -100,6 +100,11 @@ jobs:
         run: |
           npm run lint
 
+      - name: Lint Markdown
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run lint:markdown
+
       - name: Lint scenarios
         working-directory: extensions/ql-vscode
         run: |

extensions/ql-vscode/package-lock.json

@@ -16,10 +16,11 @@ import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
 import { qlpackOfDatabase } from "../contextual/queryResolver";
 import { file } from "tmp-promise";
 import { readFile, writeFile } from "fs-extra";
-import { dump, load } from "js-yaml";
+import { dump as dumpYaml, load as loadYaml } from "js-yaml";
 import {
   getOnDiskWorkspaceFolders,
   showAndLogExceptionWithTelemetry,
+  showAndLogWarningMessage,
 } from "../helpers";
 import { DatabaseItem, DatabaseManager } from "../local-databases";
 import { CodeQLCliServer } from "../cli";
@@ -29,6 +30,8 @@ import { promptImportGithubDatabase } from "../databaseFetcher";
 import { App } from "../common/app";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
 import { redactableError } from "../pure/errors";
+import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
+import { ExternalApiUsage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
 
 export class DataExtensionsEditorView extends AbstractWebview<
@@ -76,8 +79,11 @@ export class DataExtensionsEditorView extends AbstractWebview<
         await this.onWebViewLoaded();
 
         break;
-      case "applyDataExtensionYaml":
-        await this.saveYaml(msg.yaml);
+      case "saveModeledMethods":
+        await this.saveModeledMethods(
+          msg.externalApiUsages,
+          msg.modeledMethods,
+        );
         await this.loadExternalApiUsages();
 
         break;
@@ -93,36 +99,51 @@ export class DataExtensionsEditorView extends AbstractWebview<
   protected async onWebViewLoaded() {
     super.onWebViewLoaded();
 
-    await Promise.all([this.loadExternalApiUsages(), this.readExistingYaml()]);
+    await Promise.all([
+      this.loadExternalApiUsages(),
+      this.loadExistingModeledMethods(),
+    ]);
   }
 
-  protected async saveYaml(yaml: string): Promise<void> {
-    const modelFilename = this.modelFileName;
+  protected async saveModeledMethods(
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+  ): Promise<void> {
+    const modelFilename = this.calculateModelFilename();
     if (!modelFilename) {
       return;
     }
 
+    const yaml = createDataExtensionYaml(externalApiUsages, modeledMethods);
+
     await writeFile(modelFilename, yaml);
 
     void extLogger.log(`Saved data extension YAML to ${modelFilename}`);
   }
 
-  protected async readExistingYaml(): Promise<void> {
-    const modelFilename = this.modelFileName;
+  protected async loadExistingModeledMethods(): Promise<void> {
+    const modelFilename = this.calculateModelFilename();
     if (!modelFilename) {
       return;
     }
 
     try {
       const yaml = await readFile(modelFilename, "utf8");
 
-      const data = load(yaml, {
+      const data = loadYaml(yaml, {
         filename: modelFilename,
       });
 
+      const existingModeledMethods = loadDataExtensionYaml(data);
+
+      if (!existingModeledMethods) {
+        void showAndLogWarningMessage("Failed to parse data extension YAML.");
+        return;
+      }
+
       await this.postMessage({
-        t: "setExistingYamlData",
-        data,
+        t: "setExistingModeledMethods",
+        existingModeledMethods,
       });
     } catch (e: unknown) {
       void extLogger.log(`Unable to read data extension YAML: ${e}`);
@@ -275,7 +296,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
         },
       });
     }
-    await writeFile(suiteFile, dump(suiteYaml), "utf8");
+    await writeFile(suiteFile, dumpYaml(suiteYaml), "utf8");
 
     const queries = await this.cliServer.resolveQueriesInSuite(
       suiteFile,
@@ -354,7 +375,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     });
   }
 
-  private get modelFileName(): string | undefined {
+  private calculateModelFilename(): string | undefined {
     const workspaceFolder = workspace.workspaceFolders?.find(
       (folder) => folder.name === "ql",
     );

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -493,19 +493,20 @@ export interface ShowProgressMessage {
   message: string;
 }
 
-export interface SetExistingYamlDataMessage {
-  t: "setExistingYamlData";
-  data: any;
+export interface SetExistingModeledMethods {
+  t: "setExistingModeledMethods";
+  existingModeledMethods: Record<string, ModeledMethod>;
 }
 
 export interface AddModeledMethodsMessage {
   t: "addModeledMethods";
   modeledMethods: Record<string, ModeledMethod>;
 }
 
-export interface ApplyDataExtensionYamlMessage {
-  t: "applyDataExtensionYaml";
-  yaml: string;
+export interface SaveModeledMethods {
+  t: "saveModeledMethods";
+  externalApiUsages: ExternalApiUsage[];
+  modeledMethods: Record<string, ModeledMethod>;
 }
 
 export interface GenerateExternalApiMessage {
@@ -515,10 +516,10 @@ export interface GenerateExternalApiMessage {
 export type ToDataExtensionsEditorMessage =
   | SetExternalApiUsagesMessage
   | ShowProgressMessage
-  | SetExistingYamlDataMessage
+  | SetExistingModeledMethods
   | AddModeledMethodsMessage;
 
 export type FromDataExtensionsEditorMessage =
   | ViewLoadedMsg
-  | ApplyDataExtensionYamlMessage
+  | SaveModeledMethods
   | GenerateExternalApiMessage;

extensions/ql-vscode/src/pure/interface-types.ts

@@ -112,7 +112,7 @@ export function generateVariantAnalysisMarkdownSummary(
   lines.push(`### Results for "${query.name}"`, "");
 
   // Expandable section containing query text
-  const queryCodeBlock = ["```ql", ...query.text.split("\n"), "```"];
+  const queryCodeBlock = ["```ql", ...query.text.split("\n"), "```", ""];
   lines.push(...buildExpandableMarkdownSection("Query", queryCodeBlock));
 
   // Padding between sections
@@ -128,6 +128,9 @@ export function generateVariantAnalysisMarkdownSummary(
     lines.push(`| ${fullName} | [${summary.resultCount} result(s)](${link}) |`);
   }
 
+  // Add a trailing newline
+  lines.push("");
+
   return {
     fileName: "_summary",
     content: lines,
@@ -279,8 +282,8 @@ function generateMarkdownForPathResults(
       );
       // Indent the snippet to fit with the numbered list.
       // The indentation is "n + 2" where the list number is an n-digit number.
-      const codeSnippetIndented = codeSnippet.map(
-        (line) => " ".repeat(listNumber.toString().length + 2) + line,
+      const codeSnippetIndented = codeSnippet.map((line) =>
+        (" ".repeat(listNumber.toString().length + 2) + line).trimEnd(),
       );
       pathLines.push(`${listNumber}. ${link}`, ...codeSnippetIndented);
     }
@@ -382,7 +385,6 @@ function buildExpandableMarkdownSection(
     `<summary>${title}</summary>`,
     "",
     ...contents,
-    "",
     "</details>",
     "",
   );

extensions/ql-vscode/src/variant-analysis/markdown-generation.ts

@@ -16,10 +16,6 @@ import { ModeledMethod } from "../../data-extensions-editor/modeled-method";
 import { MethodRow } from "./MethodRow";
 import { assertNever } from "../../pure/helpers-pure";
 import { vscode } from "../vscode-api";
-import {
-  createDataExtensionYaml,
-  loadDataExtensionYaml,
-} from "../../data-extensions-editor/yaml";
 import { calculateSupportedPercentage } from "./supported";
 
 export const DataExtensionsEditorContainer = styled.div`
@@ -61,12 +57,10 @@ export function DataExtensionsEditor(): JSX.Element {
           case "showProgress":
             setProgress(msg);
             break;
-          case "setExistingYamlData":
+          case "setExistingModeledMethods":
             setModeledMethods((oldModeledMethods) => {
-              const existingModeledMethods = loadDataExtensionYaml(msg.data);
-
               return {
-                ...existingModeledMethods,
+                ...msg.existingModeledMethods,
                 ...oldModeledMethods,
               };
             });
@@ -120,14 +114,10 @@ export function DataExtensionsEditor(): JSX.Element {
   );
 
   const onApplyClick = useCallback(() => {
-    const yamlString = createDataExtensionYaml(
+    vscode.postMessage({
+      t: "saveModeledMethods",
       externalApiUsages,
       modeledMethods,
-    );
-
-    vscode.postMessage({
-      t: "applyDataExtensionYaml",
-      yaml: yamlString,
     });
   }, [externalApiUsages, modeledMethods]);
 

extensions/ql-vscode/src/view/data-extensions-editor/DataExtensionsEditor.tsx

@@ -0,0 +1,7 @@
+{
+  "extends": "../../../../../../.markdownlint.json",
+  "MD010": false, // Hard tabs are from the original snippet in the SARIF file
+  "MD024": false, // Duplicate headings
+  "MD033": false, // Inline HTML for <details> and <summary>
+  "MD051": false // Link fragments are used in Gist to link to other files
+}