Show external API calls in data extensions editor

This updates the view of the data extensions editor to show a table of
possible sources/sinks/flow summaries that can be edited. It's not yet
possible to save the changes or load the existing file.

Author: koesie10

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -1,14 +1,36 @@
 import { ExtensionContext } from "vscode";
 import { DataExtensionsEditorView } from "./data-extensions-editor-view";
 import { DataExtensionsEditorCommands } from "../common/commands";
+import { CodeQLCliServer } from "../cli";
+import { QueryRunner } from "../queryRunner";
+import { DatabaseManager } from "../local-databases";
+import { extLogger } from "../common";
 
 export class DataExtensionsEditorModule {
-  public constructor(private readonly ctx: ExtensionContext) {}
+  public constructor(
+    private readonly ctx: ExtensionContext,
+    private readonly databaseManager: DatabaseManager,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly queryRunner: QueryRunner,
+    private readonly queryStorageDir: string,
+  ) {}
 
   public getCommands(): DataExtensionsEditorCommands {
     return {
       "codeQL.openDataExtensionsEditor": async () => {
-        const view = new DataExtensionsEditorView(this.ctx);
+        const db = this.databaseManager.currentDatabaseItem;
+        if (!db) {
+          void extLogger.log("No database selected");
+          return;
+        }
+
+        const view = new DataExtensionsEditorView(
+          this.ctx,
+          this.cliServer,
+          this.queryRunner,
+          this.queryStorageDir,
+          db,
+        );
         await view.openView();
       },
     };

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -1,15 +1,31 @@
-import { ExtensionContext, ViewColumn } from "vscode";
+import { CancellationTokenSource, ExtensionContext, ViewColumn } from "vscode";
 import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
 import {
   FromDataExtensionsEditorMessage,
   ToDataExtensionsEditorMessage,
 } from "../pure/interface-types";
+import { ProgressUpdate } from "../progress";
+import { extLogger, TeeLogger } from "../common";
+import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
+import { qlpackOfDatabase } from "../contextual/queryResolver";
+import { file } from "tmp-promise";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { getOnDiskWorkspaceFolders } from "../helpers";
+import { DatabaseItem } from "../local-databases";
+import { CodeQLCliServer } from "../cli";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
   FromDataExtensionsEditorMessage
 > {
-  public constructor(ctx: ExtensionContext) {
+  public constructor(
+    ctx: ExtensionContext,
+    private readonly cliServer: CodeQLCliServer,
+    private readonly queryRunner: QueryRunner,
+    private readonly queryStorageDir: string,
+    private readonly databaseItem: DatabaseItem,
+  ) {
     super(ctx);
   }
 
@@ -49,5 +65,136 @@ export class DataExtensionsEditorView extends AbstractWebview<
 
   protected async onWebViewLoaded() {
     super.onWebViewLoaded();
+
+    await this.loadExternalApiUsages();
+  }
+
+  protected async loadExternalApiUsages(): Promise<void> {
+    const queryResult = await this.runQuery();
+    if (!queryResult) {
+      await this.clearProgress();
+      return;
+    }
+
+    await this.showProgress({
+      message: "Loading results",
+      step: 1100,
+      maxStep: 1500,
+    });
+
+    const bqrsPath = queryResult.outputDir.bqrsPath;
+
+    const results = await this.getResults(bqrsPath);
+    if (!results) {
+      await this.clearProgress();
+      return;
+    }
+
+    await this.showProgress({
+      message: "Finalizing results",
+      step: 1450,
+      maxStep: 1500,
+    });
+
+    await this.postMessage({
+      t: "setExternalApiRepoResults",
+      results,
+    });
+
+    await this.clearProgress();
+  }
+
+  private async runQuery(): Promise<CoreCompletedQuery | undefined> {
+    const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem);
+
+    const packsToSearch = [qlpacks.dbschemePack];
+    if (qlpacks.queryPack) {
+      packsToSearch.push(qlpacks.queryPack);
+    }
+
+    const suiteFile = (
+      await file({
+        postfix: ".qls",
+      })
+    ).path;
+    const suiteYaml = [];
+    for (const qlpack of packsToSearch) {
+      suiteYaml.push({
+        from: qlpack,
+        queries: ".",
+        include: {
+          id: `${this.databaseItem.language}/telemetry/fetch-external-apis`,
+        },
+      });
+    }
+    await writeFile(suiteFile, dump(suiteYaml), "utf8");
+
+    const queries = await this.cliServer.resolveQueriesInSuite(
+      suiteFile,
+      getOnDiskWorkspaceFolders(),
+    );
+
+    if (queries.length !== 1) {
+      void extLogger.log(`Expected exactly one query, got ${queries.length}`);
+      return;
+    }
+
+    const query = queries[0];
+
+    const tokenSource = new CancellationTokenSource();
+
+    const queryRun = this.queryRunner.createQueryRun(
+      this.databaseItem.databaseUri.fsPath,
+      { queryPath: query, quickEvalPosition: undefined },
+      false,
+      getOnDiskWorkspaceFolders(),
+      undefined,
+      this.queryStorageDir,
+      undefined,
+      undefined,
+    );
+
+    return queryRun.evaluate(
+      (update) => this.showProgress(update, 1500),
+      tokenSource.token,
+      new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath),
+    );
+  }
+
+  private async getResults(bqrsPath: string) {
+    const bqrsInfo = await this.cliServer.bqrsInfo(bqrsPath);
+    if (bqrsInfo["result-sets"].length !== 1) {
+      void extLogger.log(
+        `Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
+      );
+      return undefined;
+    }
+
+    const resultSet = bqrsInfo["result-sets"][0];
+
+    await this.showProgress({
+      message: "Decoding results",
+      step: 1200,
+      maxStep: 1500,
+    });
+
+    return this.cliServer.bqrsDecode(bqrsPath, resultSet.name);
+  }
+
+  private async showProgress(update: ProgressUpdate, maxStep?: number) {
+    await this.postMessage({
+      t: "showProgress",
+      step: update.step,
+      maxStep: maxStep ?? update.maxStep,
+      message: update.message,
+    });
+  }
+
+  private async clearProgress() {
+    await this.showProgress({
+      step: 0,
+      maxStep: 0,
+      message: "",
+    });
   }
 }

extensions/ql-vscode/src/data-extensions-editor/interface.ts

@@ -0,0 +1,30 @@
+import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
+
+export type Call = {
+  label: string;
+  url: ResolvableLocationValue;
+};
+
+export type ExternalApiUsage = {
+  externalApiInfo: string;
+  packageName: string;
+  typeName: string;
+  methodName: string;
+  methodParameters: string;
+  supported: boolean;
+  usages: Call[];
+};
+
+export type ModeledMethodType =
+  | "none"
+  | "source"
+  | "sink"
+  | "summary"
+  | "neutral";
+
+export type ModeledMethod = {
+  type: ModeledMethodType;
+  input: string;
+  output: string;
+  kind: string;
+};

extensions/ql-vscode/src/extension.ts

@@ -861,7 +861,18 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(localQueries);
 
-  const dataExtensionsEditorModule = new DataExtensionsEditorModule(ctx);
+  const dataExtensionsEditorQueryStorageDir = join(
+    tmpDir.name,
+    "data-extensions-editor-results",
+  );
+  await ensureDir(dataExtensionsEditorQueryStorageDir);
+  const dataExtensionsEditorModule = new DataExtensionsEditorModule(
+    ctx,
+    dbm,
+    cliServer,
+    qs,
+    dataExtensionsEditorQueryStorageDir,
+  );
 
   void extLogger.log("Initializing QLTest interface.");
   const testExplorerExtension = extensions.getExtension<TestHub>(

extensions/ql-vscode/src/pure/interface-types.ts

@@ -5,6 +5,7 @@ import {
   ResultSetSchema,
   Column,
   ResolvableLocationValue,
+  DecodedBqrsChunk,
 } from "./bqrs-cli-types";
 import {
   VariantAnalysis,
@@ -479,6 +480,20 @@ export type ToDataFlowPathsMessage = SetDataFlowPathsMessage;
 
 export type FromDataFlowPathsMessage = CommonFromViewMessages;
 
-export type ToDataExtensionsEditorMessage = never;
+export interface SetExternalApiResultsMessage {
+  t: "setExternalApiRepoResults";
+  results: DecodedBqrsChunk;
+}
+
+export interface ShowProgressMessage {
+  t: "showProgress";
+  step: number;
+  maxStep: number;
+  message: string;
+}
+
+export type ToDataExtensionsEditorMessage =
+  | SetExternalApiResultsMessage
+  | ShowProgressMessage;
 
 export type FromDataExtensionsEditorMessage = ViewLoadedMsg;