Add Compare Performance command (WIP)

Author: asgerf

extensions/ql-vscode/package.json

@@ -959,6 +959,10 @@
         "command": "codeQLQueryHistory.compareWith",
         "title": "Compare Results"
       },
+      {
+        "command": "codeQLQueryHistory.comparePerformanceWith",
+        "title": "Compare Performance"
+      },
       {
         "command": "codeQLQueryHistory.openOnGithub",
         "title": "View Logs"
@@ -1230,6 +1234,11 @@
           "group": "3_queryHistory@0",
           "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
         },
+        {
+          "command": "codeQLQueryHistory.comparePerformanceWith",
+          "group": "3_queryHistory@1",
+          "when": "viewItem == rawResultsItem || viewItem == interpretedResultsItem"
+        },
         {
           "command": "codeQLQueryHistory.showQueryLog",
           "group": "4_queryHistory@4",
@@ -1733,6 +1742,10 @@
           "command": "codeQLQueryHistory.compareWith",
           "when": "false"
         },
+        {
+          "command": "codeQLQueryHistory.comparePerformanceWith",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.sortByName",
           "when": "false"

extensions/ql-vscode/src/common/commands.ts

@@ -180,6 +180,7 @@ export type QueryHistoryCommands = {
   "codeQLQueryHistory.removeHistoryItemContextInline": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.renameItem": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.compareWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
+  "codeQLQueryHistory.comparePerformanceWith": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLog": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLogSummary": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;
   "codeQLQueryHistory.showEvalLogViewer": TreeViewContextMultiSelectionCommandFunction<QueryHistoryInfo>;

extensions/ql-vscode/src/common/interface-types.ts

@@ -27,6 +27,7 @@ import type {
 } from "./raw-result-types";
 import type { AccessPathSuggestionOptions } from "../model-editor/suggestions";
 import type { ModelEvaluationRunState } from "../model-editor/shared/model-evaluation-run-state";
+import type { PerformanceComparisonDataFromLog } from "../log-insights/performance-comparison";
 
 /**
  * This module contains types and code that are shared between
@@ -396,6 +397,16 @@ export interface SetComparisonsMessage {
   readonly message: string | undefined;
 }
 
+export type ToComparePerformanceViewMessage = SetPerformanceComparisonQueries;
+
+export interface SetPerformanceComparisonQueries {
+  readonly t: "setPerformanceComparison";
+  readonly from: PerformanceComparisonDataFromLog;
+  readonly to: PerformanceComparisonDataFromLog;
+}
+
+export type FromComparePerformanceViewMessage = CommonFromViewMessages;
+
 export type QueryCompareResult =
   | RawQueryCompareResult
   | InterpretedQueryCompareResult;

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -41,6 +41,13 @@ export abstract class AbstractWebview<
 
   constructor(protected readonly app: App) {}
 
+  public hidePanel() {
+    if (this.panel !== undefined) {
+      this.panel.dispose();
+      this.panel = undefined;
+    }
+  }
+
   public async restoreView(panel: WebviewPanel): Promise<void> {
     this.panel = panel;
     const config = await this.getPanelConfig();

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -7,6 +7,7 @@ import type { App } from "../app";
 export type WebviewKind =
   | "results"
   | "compare"
+  | "compare-performance"
   | "variant-analysis"
   | "data-flow-paths"
   | "model-editor"

extensions/ql-vscode/src/compare-performance/compare-performance-view.ts

@@ -0,0 +1,93 @@
+import { ViewColumn } from "vscode";
+
+import type { App } from "../common/app";
+import { redactableError } from "../common/errors";
+import type {
+  FromComparePerformanceViewMessage,
+  ToComparePerformanceViewMessage,
+} from "../common/interface-types";
+import type { Logger } from "../common/logging";
+import { showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
+import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
+import { AbstractWebview } from "../common/vscode/abstract-webview";
+import { telemetryListener } from "../common/vscode/telemetry";
+import type { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
+import { PerformanceOverviewScanner } from "../log-insights/performance-comparison";
+import { scanLog } from "../log-insights/log-scanner";
+import type { ResultsView } from "../local-queries";
+
+export class ComparePerformanceView extends AbstractWebview<
+  ToComparePerformanceViewMessage,
+  FromComparePerformanceViewMessage
+> {
+  constructor(
+    app: App,
+    public logger: Logger,
+    public labelProvider: HistoryItemLabelProvider,
+    private resultsView: ResultsView,
+  ) {
+    super(app);
+  }
+
+  async showResults(fromJsonLog: string, toJsonLog: string) {
+    const panel = await this.getPanel();
+    panel.reveal(undefined, false);
+
+    // Close the results viewer as it will have opened when the user clicked the query in the history view
+    // (which they must do as part of the UI interaction for opening the performance view).
+    // The performance view generally needs a lot of width so it's annoying to have the result viewer open.
+    this.resultsView.hidePanel();
+
+    await this.waitForPanelLoaded();
+
+    // TODO: try processing in (async) parallel once readJsonl is streaming
+    const fromPerf = await scanLog(
+      fromJsonLog,
+      new PerformanceOverviewScanner(),
+    );
+    const toPerf = await scanLog(toJsonLog, new PerformanceOverviewScanner());
+
+    await this.postMessage({
+      t: "setPerformanceComparison",
+      from: fromPerf.getData(),
+      to: toPerf.getData(),
+    });
+  }
+
+  protected getPanelConfig(): WebviewPanelConfig {
+    return {
+      viewId: "comparePerformanceView",
+      title: "Compare CodeQL Performance",
+      viewColumn: ViewColumn.Active,
+      preserveFocus: true,
+      view: "compare-performance",
+    };
+  }
+
+  protected onPanelDispose(): void {}
+
+  protected async onMessage(
+    msg: FromComparePerformanceViewMessage,
+  ): Promise<void> {
+    switch (msg.t) {
+      case "viewLoaded":
+        this.onWebViewLoaded();
+        break;
+
+      case "telemetry":
+        telemetryListener?.sendUIInteraction(msg.action);
+        break;
+
+      case "unhandledError":
+        void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
+          redactableError(
+            msg.error,
+          )`Unhandled error in performance comparison view: ${msg.error.message}`,
+        );
+        break;
+    }
+  }
+}

extensions/ql-vscode/src/extension.ts

@@ -135,6 +135,7 @@ import { LanguageContextStore } from "./language-context-store";
 import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
 import { GitHubDatabasesModule } from "./databases/github-databases";
 import { DatabaseFetcher } from "./databases/database-fetcher";
+import { ComparePerformanceView } from "./compare-performance/compare-performance-view";
 
 /**
  * extension.ts
@@ -924,6 +925,11 @@ async function activateWithInstalledDistribution(
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo,
     ): Promise<void> => showResultsForComparison(compareView, from, to),
+    async (
+      from: CompletedLocalQueryInfo,
+      to: CompletedLocalQueryInfo,
+    ): Promise<void> =>
+      showPerformanceComparison(comparePerformanceView, from, to),
   );
 
   ctx.subscriptions.push(qhm);
@@ -949,6 +955,15 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(compareView);
 
+  void extLogger.log("Initializing performance comparison view.");
+  const comparePerformanceView = new ComparePerformanceView(
+    app,
+    queryServerLogger,
+    labelProvider,
+    localQueryResultsView,
+  );
+  ctx.subscriptions.push(comparePerformanceView);
+
   void extLogger.log("Initializing source archive filesystem provider.");
   archiveFilesystemProvider_activate(ctx, dbm);
 
@@ -1190,6 +1205,25 @@ async function showResultsForComparison(
   }
 }
 
+async function showPerformanceComparison(
+  view: ComparePerformanceView,
+  from: CompletedLocalQueryInfo,
+  to: CompletedLocalQueryInfo,
+): Promise<void> {
+  const fromLog = from.evaluatorLogPaths?.jsonSummary;
+  const toLog = to.evaluatorLogPaths?.jsonSummary;
+  if (fromLog === undefined || toLog === undefined) {
+    return extLogger.showWarningMessage(
+      `Cannot compare performance as the structured logs are missing. Did they queries complete normally?`,
+    );
+  }
+  await extLogger.log(
+    `Comparing performance of ${from.getQueryName()} and ${to.getQueryName()}`,
+  );
+
+  await view.showResults(fromLog, toLog);
+}
+
 function addUnhandledRejectionListener() {
   const handler = (error: unknown) => {
     // This listener will be triggered for errors from other extensions as

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -112,3 +112,20 @@ export class EvaluationLogScannerSet {
     scanners.forEach((scanner) => scanner.onDone());
   }
 }
+
+/**
+ * Scan the evaluator summary log using the given scanner. For conveience, returns the scanner.
+ *
+ * @param jsonSummaryLocation The file path of the JSON summary log.
+ * @param scanner The scanner to process events from the log
+ */
+export async function scanLog<T extends EvaluationLogScanner>(
+  jsonSummaryLocation: string,
+  scanner: T,
+): Promise<T> {
+  await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
+    scanner.onEvent(obj);
+  });
+  scanner.onDone();
+  return scanner;
+}

extensions/ql-vscode/src/log-insights/log-summary.ts

@@ -33,6 +33,7 @@ interface ResultEventBase extends SummaryEventBase {
 export interface ComputeSimple extends ResultEventBase {
   evaluationStrategy: "COMPUTE_SIMPLE";
   ra: Ra;
+  millis: number;
   pipelineRuns?: [PipelineRun];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };
@@ -42,6 +43,7 @@ export interface ComputeRecursive extends ResultEventBase {
   evaluationStrategy: "COMPUTE_RECURSIVE";
   deltaSizes: number[];
   ra: Ra;
+  millis: number;
   pipelineRuns: PipelineRun[];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };