Merge branch 'main' into robertbrignull/data-view-jump

Author: robertbrignull

extensions/ql-vscode/package-lock.json

@@ -1759,7 +1759,7 @@
     "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
-    "@octokit/plugin-retry": "^6.0.0",
+    "@octokit/plugin-retry": "^4.1.6",
     "@octokit/rest": "^19.0.4",
     "@vscode/codicons": "^0.0.31",
     "@vscode/debugadapter": "^1.59.0",
@@ -1851,13 +1851,13 @@
     "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
-    "cosmiconfig": "^7.1.0",
+    "cosmiconfig": "^8.2.0",
     "cross-env": "^7.0.3",
     "css-loader": "^6.8.1",
     "del": "^6.0.0",
     "esbuild": "^0.15.15",
     "eslint": "^8.23.1",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-etc": "^2.0.2",
     "eslint-plugin-github": "^4.4.1",
     "eslint-plugin-jest-dom": "^5.0.1",
@@ -1881,7 +1881,7 @@
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "patch-package": "^7.0.0",
+    "patch-package": "^8.0.0",
     "prettier": "^3.0.0",
     "storybook": "^7.1.0",
     "tar-stream": "^3.0.0",
@@ -1890,7 +1890,7 @@
     "ts-json-schema-generator": "^1.1.2",
     "ts-loader": "^9.4.2",
     "ts-node": "^10.7.0",
-    "ts-unused-exports": "^9.0.5",
+    "ts-unused-exports": "^10.0.0",
     "typescript": "^5.0.2",
     "webpack": "^5.76.0",
     "webpack-cli": "^5.0.1"

extensions/ql-vscode/package.json

@@ -515,6 +515,7 @@ interface AddModeledMethodsMessage {
 
 interface SetInProgressMethodsMessage {
   t: "setInProgressMethods";
+  packageName: string;
   inProgressMethods: string[];
 }
 

extensions/ql-vscode/src/common/interface-types.ts

@@ -177,9 +177,6 @@ export async function runAutoModelQueries({
   progress,
   cancellationTokenSource,
 }: AutoModelQueriesOptions): Promise<AutoModelQueriesResult | undefined> {
-  // maxStep for this part is 1500
-  const maxStep = 1500;
-
   const qlpack = await qlpackOfDatabase(cliServer, databaseItem);
 
   // CodeQL needs to have access to the database to be able to retrieve the
@@ -208,12 +205,6 @@ export async function runAutoModelQueries({
     await cliServer.resolveQlpacks(additionalPacks, true),
   );
 
-  progress({
-    step: 0,
-    maxStep,
-    message: "Finding candidates and examples",
-  });
-
   const candidates = await runAutoModelQuery({
     mode,
     queryTag: "candidates",
@@ -225,13 +216,7 @@ export async function runAutoModelQueries({
     additionalPacks,
     extensionPacks,
     queryStorageDir,
-    progress: (update) => {
-      progress({
-        step: update.step,
-        maxStep,
-        message: "Finding candidates and examples",
-      });
-    },
+    progress,
     token: cancellationTokenSource.token,
   });
 

extensions/ql-vscode/src/data-extensions-editor/auto-model-codeml-queries.ts

@@ -22,7 +22,13 @@ import { CancellationTokenSource } from "vscode";
 // Note that the model may return fewer than this number of candidates.
 const candidateBatchSize = 20;
 
+/**
+ * The auto-modeler holds state around auto-modeling jobs and allows
+ * starting and stopping them.
+ */
 export class AutoModeler {
+  // Keep track of auto-modeling jobs that are in progress
+  // so that we can stop them.
   private readonly jobs: Map<string, CancellationTokenSource>;
 
   constructor(
@@ -32,6 +38,7 @@ export class AutoModeler {
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
     private readonly setInProgressMethods: (
+      packageName: string,
       inProgressMethods: string[],
     ) => Promise<void>,
     private readonly addModeledMethods: (
@@ -41,6 +48,14 @@ export class AutoModeler {
     this.jobs = new Map<string, CancellationTokenSource>();
   }
 
+  /**
+   * Models the given package's external API usages, except
+   * the ones that are already modeled.
+   * @param packageName The name of the package to model.
+   * @param externalApiUsages The external API usages.
+   * @param modeledMethods The currently modeled methods.
+   * @param mode The mode we are modeling in.
+   */
   public async startModeling(
     packageName: string,
     externalApiUsages: ExternalApiUsage[],
@@ -67,6 +82,10 @@ export class AutoModeler {
     }
   }
 
+  /**
+   * Stops modeling the given package.
+   * @param packageName The name of the package to stop modeling.
+   */
   public async stopModeling(packageName: string): Promise<void> {
     void extLogger.log(`Stopping modeling for package ${packageName}`);
     const cancellationTokenSource = this.jobs.get(packageName);
@@ -75,6 +94,9 @@ export class AutoModeler {
     }
   }
 
+  /**
+   * Stops all in-progress modeling jobs.
+   */
   public async stopAllModeling(): Promise<void> {
     for (const cancellationTokenSource of this.jobs.values()) {
       cancellationTokenSource.cancel();
@@ -90,8 +112,6 @@ export class AutoModeler {
   ): Promise<void> {
     void extLogger.log(`Modeling package ${packageName}`);
     await withProgress(async (progress) => {
-      const maxStep = 3000;
-
       // Fetch the candidates to send to the model
       const allCandidateMethods = getCandidates(
         mode,
@@ -111,6 +131,7 @@ export class AutoModeler {
       );
       try {
         for (let i = 0; i < batchNumber; i++) {
+          // Check if we should stop
           if (cancellationTokenSource.token.isCancellationRequested) {
             break;
           }
@@ -119,29 +140,23 @@ export class AutoModeler {
           const end = start + candidateBatchSize;
           const candidatesToProcess = allCandidateMethods.slice(start, end);
 
+          // Let the UI know which candidates we are modeling
           await this.setInProgressMethods(
+            packageName,
             candidatesToProcess.map((c) => c.signature),
           );
 
-          progress({
-            step: 1800 + i * 100,
-            maxStep,
-            message: `Automodeling candidates, batch ${
-              i + 1
-            } of ${batchNumber}`,
-          });
-
+          // Kick off the process to model the slice of candidates
           await this.modelCandidates(
             candidatesToProcess,
             mode,
             progress,
-            maxStep,
             cancellationTokenSource,
           );
         }
       } finally {
         // Clear out in progress methods
-        await this.setInProgressMethods([]);
+        await this.setInProgressMethods(packageName, []);
       }
     });
   }
@@ -150,48 +165,33 @@ export class AutoModeler {
     candidateMethods: MethodSignature[],
     mode: Mode,
     progress: ProgressCallback,
-    maxStep: number,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
+    void extLogger.log("Executing auto-model queries");
+
     const usages = await runAutoModelQueries({
       mode,
       candidateMethods,
       cliServer: this.cliServer,
       queryRunner: this.queryRunner,
       queryStorageDir: this.queryStorageDir,
       databaseItem: this.databaseItem,
-      progress: (update) => progress({ ...update, maxStep }),
+      progress: (update) => progress({ ...update }),
       cancellationTokenSource,
     });
     if (!usages) {
       return;
     }
 
-    progress({
-      step: 1800,
-      maxStep,
-      message: "Creating request",
-    });
-
     const request = await createAutoModelV2Request(mode, usages);
 
-    progress({
-      step: 2000,
-      maxStep,
-      message: "Sending request",
-    });
+    void extLogger.log("Calling auto-model API");
 
     const response = await this.callAutoModelApi(request);
     if (!response) {
       return;
     }
 
-    progress({
-      step: 2500,
-      maxStep,
-      message: "Parsing response",
-    });
-
     const models = loadYaml(response.models, {
       filename: "auto-model.yml",
     });
@@ -222,12 +222,6 @@ export class AutoModeler {
       }
     }
 
-    progress({
-      step: 2800,
-      maxStep,
-      message: "Applying results",
-    });
-
     await this.addModeledMethods(loadedMethods);
   }
 

extensions/ql-vscode/src/data-extensions-editor/auto-modeler.ts

@@ -45,7 +45,7 @@ export class DataExtensionsEditorModule extends DisposableObject {
       baseQueryStorageDir,
       "data-extensions-editor-results",
     );
-    this.modelDetailsPanel = this.push(new ModelDetailsPanel());
+    this.modelDetailsPanel = this.push(new ModelDetailsPanel(cliServer));
   }
 
   public static async initialize(

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -72,7 +72,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly updateModelDetailsPanelState: (
       externalApiUsages: ExternalApiUsage[],
       databaseItem: DatabaseItem,
-    ) => void,
+    ) => Promise<void>,
     private readonly revealItemInDetailsPanel: (usage: Usage) => Promise<void>,
   ) {
     super(ctx);
@@ -83,9 +83,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
       queryRunner,
       queryStorageDir,
       databaseItem,
-      async (inProgressMethods) => {
+      async (packageName, inProgressMethods) => {
         await this.postMessage({
           t: "setInProgressMethods",
+          packageName,
           inProgressMethods,
         });
       },
@@ -300,7 +301,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
             t: "setExternalApiUsages",
             externalApiUsages,
           });
-          this.updateModelDetailsPanelState(
+          await this.updateModelDetailsPanelState(
             externalApiUsages,
             this.databaseItem,
           );

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -1,57 +1,84 @@
 import {
   Event,
   EventEmitter,
+  ThemeColor,
+  ThemeIcon,
   TreeDataProvider,
   TreeItem,
   TreeItemCollapsibleState,
+  Uri,
 } from "vscode";
 import { DisposableObject } from "../../common/disposable-object";
 import { ExternalApiUsage, Usage } from "../external-api-usage";
 import { DatabaseItem } from "../../databases/local-databases";
+import { relative } from "path";
+import { CodeQLCliServer } from "../../codeql-cli/cli";
 
 export class ModelDetailsDataProvider
   extends DisposableObject
   implements TreeDataProvider<ModelDetailsTreeViewItem>
 {
   private externalApiUsages: ExternalApiUsage[] = [];
   private databaseItem: DatabaseItem | undefined = undefined;
+  private sourceLocationPrefix: string | undefined = undefined;
 
   private readonly onDidChangeTreeDataEmitter = this.push(
     new EventEmitter<void>(),
   );
 
+  public constructor(private readonly cliServer: CodeQLCliServer) {
+    super();
+  }
+
   public get onDidChangeTreeData(): Event<void> {
     return this.onDidChangeTreeDataEmitter.event;
   }
 
-  public setState(
+  public async setState(
     externalApiUsages: ExternalApiUsage[],
     databaseItem: DatabaseItem,
-  ): void {
+  ): Promise<void> {
     this.externalApiUsages = externalApiUsages;
     this.databaseItem = databaseItem;
+    this.sourceLocationPrefix = await this.databaseItem.getSourceLocationPrefix(
+      this.cliServer,
+    );
     this.onDidChangeTreeDataEmitter.fire();
   }
 
   getTreeItem(item: ModelDetailsTreeViewItem): TreeItem {
     if (isExternalApiUsage(item)) {
       return {
-        label: item.signature,
+        label: `${item.packageName}.${item.typeName}.${item.methodName}${item.methodParameters}`,
         collapsibleState: TreeItemCollapsibleState.Collapsed,
+        iconPath: new ThemeIcon("symbol-method"),
       };
     } else {
       return {
         label: item.label,
+        description: `${this.relativePathWithinDatabase(item.url.uri)} [${
+          item.url.startLine
+        }, ${item.url.endLine}]`,
         collapsibleState: TreeItemCollapsibleState.None,
         command: {
           title: "Show usage",
           command: "codeQLDataExtensionsEditor.jumpToUsageLocation",
           arguments: [item, this.databaseItem],
         },
+        iconPath: new ThemeIcon("error", new ThemeColor("errorForeground")),
       };
     }
   }
 
+  private relativePathWithinDatabase(uri: string): string {
+    const parsedUri = Uri.parse(uri);
+    if (this.sourceLocationPrefix) {
+      return relative(this.sourceLocationPrefix, parsedUri.fsPath);
+    } else {
+      return parsedUri.fsPath;
+    }
+  }
+
   getChildren(item?: ModelDetailsTreeViewItem): ModelDetailsTreeViewItem[] {
     if (item === undefined) {
       return this.externalApiUsages;