Merge remote-tracking branch 'origin/main' into dbartol/goto-ql

Author: Dave Bartolomeo

extensions/ql-vscode/package-lock.json

@@ -231,7 +231,7 @@
         },
         "codeQL.queryHistory.format": {
           "type": "string",
-          "default": "%q on %d - %s, %r [%t]",
+          "default": "%q on %d - %s %r [%t]",
           "markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
         },
         "codeQL.queryHistory.ttl": {
@@ -1160,8 +1160,10 @@
   },
   "dependencies": {
     "@octokit/rest": "^18.5.6",
+    "@octokit/plugin-retry": "^3.0.9",
     "@primer/octicons-react": "^16.3.0",
     "@primer/react": "^35.0.0",
+    "@vscode/codicons": "^0.0.31",
     "@vscode/webview-ui-toolkit": "^1.0.0",
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
@@ -1177,8 +1179,8 @@
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "semver": "~7.3.2",
-    "source-map-support": "^0.5.21",
     "source-map": "^0.7.4",
+    "source-map-support": "^0.5.21",
     "stream": "^0.0.2",
     "stream-chain": "~2.2.4",
     "stream-json": "~1.7.3",

extensions/ql-vscode/package.json

@@ -1,5 +1,6 @@
 import * as vscode from 'vscode';
 import * as Octokit from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
 
 const GITHUB_AUTH_PROVIDER_ID = 'github';
 
@@ -51,14 +52,15 @@ export class Credentials {
 
   private async createOctokit(createIfNone: boolean, overrideToken?: string): Promise<Octokit.Octokit | undefined> {
     if (overrideToken) {
-      return new Octokit.Octokit({ auth: overrideToken });
+      return new Octokit.Octokit({ auth: overrideToken, retry });
     }
 
     const session = await vscode.authentication.getSession(GITHUB_AUTH_PROVIDER_ID, SCOPES, { createIfNone });
 
     if (session) {
       return new Octokit.Octokit({
-        auth: session.accessToken
+        auth: session.accessToken,
+        retry
       });
     } else {
       return undefined;

extensions/ql-vscode/src/authentication.ts

@@ -690,6 +690,23 @@ export class CodeQLCliServer implements Disposable {
     return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
   }
 
+  /**
+  * Generate a JSON summary of an evaluation log.
+  * @param inputPath The path of an evaluation event log.
+  * @param outputPath The path to write a JSON summary of it to.
+  */
+   async generateJsonLogSummary(
+    inputPath: string,
+    outputPath: string,
+  ): Promise<string> {
+    const subcommandArgs = [
+      '--format=predicates',
+      inputPath,
+      outputPath
+    ];
+    return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating JSON log summary');
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.

extensions/ql-vscode/src/cli.ts

@@ -59,7 +59,7 @@ const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIB
 // Query History configuration
 const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
 const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
-const QUERY_HISTORY_TTL = new Setting('format', QUERY_HISTORY_SETTING);
+const QUERY_HISTORY_TTL = new Setting('ttl', QUERY_HISTORY_SETTING);
 
 /** When these settings change, the distribution should be updated. */
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];

extensions/ql-vscode/src/config.ts

@@ -581,3 +581,11 @@ export async function* walkDirectory(dir: string): AsyncIterableIterator<string>
     }
   }
 }
+
+/**
+ * Pluralizes a word.
+ * Example: Returns "N repository" if N is one, "N repositories" otherwise.
+ */
+export function pluralize(numItems: number | undefined, singular: string, plural: string): string {
+  return numItems ? `${numItems} ${numItems === 1 ? singular : plural}` : '';
+}

extensions/ql-vscode/src/helpers.ts

@@ -3,6 +3,7 @@ import * as path from 'path';
 import { QueryHistoryConfig } from './config';
 import { LocalQueryInfo, QueryHistoryInfo } from './query-results';
 import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
+import { pluralize } from './helpers';
 
 interface InterpolateReplacements {
   t: string; // Start time
@@ -57,23 +58,30 @@ export class HistoryItemLabelProvider {
       t: item.startTime,
       q: item.getQueryName(),
       d: item.initialInfo.databaseInfo.name,
-      r: `${resultCount} results`,
+      r: `(${resultCount} results)`,
       s: statusString,
       f: item.getQueryFileName(),
       '%': '%',
     };
   }
 
+  // Return the number of repositories queried if available. Otherwise, use the controller repository name.
+  private buildRepoLabel(item: RemoteQueryHistoryItem): string {
+    const repositoryCount = item.remoteQuery.repositoryCount;
+
+    if (repositoryCount) {
+      return pluralize(repositoryCount, 'repository', 'repositories');
+    }
+
+    return `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`;
+  }
+
   private getRemoteInterpolateReplacements(item: RemoteQueryHistoryItem): InterpolateReplacements {
     return {
       t: new Date(item.remoteQuery.executionStartTime).toLocaleString(env.language),
-      q: item.remoteQuery.queryName,
-
-      // There is no database name for remote queries. Instead use the controller repository name.
-      d: `${item.remoteQuery.controllerRepository.owner}/${item.remoteQuery.controllerRepository.name}`,
-
-      // There is no synchronous way to get the results count.
-      r: '',
+      q: `${item.remoteQuery.queryName} (${item.remoteQuery.language})`,
+      d: this.buildRepoLabel(item),
+      r: `(${pluralize(item.resultCount, 'result', 'results')})`,
       s: item.status,
       f: path.basename(item.remoteQuery.queryFilePath),
       '%': '%'

extensions/ql-vscode/src/history-item-label-provider.ts

@@ -137,6 +137,8 @@ export function getHtmlForWebview(
     ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
     : `'nonce-${nonce}'`;
 
+  const fontSrc = webview.cspSource;
+
   /*
    * Content security policy:
    * default-src: allow nothing by default.
@@ -149,7 +151,7 @@ export function getHtmlForWebview(
 <html>
   <head>
     <meta http-equiv="Content-Security-Policy"
-          content="default-src 'none'; script-src 'nonce-${nonce}'; style-src ${styleSrc}; connect-src ${webview.cspSource};">
+          content="default-src 'none'; script-src 'nonce-${nonce}'; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${webview.cspSource};">
         ${stylesheetsHtmlLines.join(`    ${os.EOL}`)}
   </head>
   <body>

extensions/ql-vscode/src/interface-utils.ts

@@ -0,0 +1,44 @@
+import * as os from 'os';
+
+// TODO(angelapwen): Only load in necessary information and
+// location in bytes for this log to save memory. 
+export interface EvaluatorLogData {
+    queryCausingWork: string;
+    predicateName: string;
+    millis: number;
+    resultSize: number;
+    ra: Pipelines;
+}
+  
+interface Pipelines {
+    // Key: pipeline identifier; Value: array of pipeline steps 
+    pipelineNamesToSteps: Map<string, string[]>;
+}
+  
+/**
+ * A pure method that parses a string of evaluator log summaries into
+ * an array of EvaluatorLogData objects. 
+ * 
+ */
+ export function parseVisualizerData(logSummary: string): EvaluatorLogData[] {
+    // Remove newline delimiters because summary is in .jsonl format.
+    const jsonSummaryObjects: string[] = logSummary.split(os.EOL + os.EOL);
+    const visualizerData: EvaluatorLogData[] = [];
+  
+    for (const obj of jsonSummaryObjects) {
+      const jsonObj = JSON.parse(obj);
+  
+      // Only convert log items that have an RA and millis field
+      if (jsonObj.ra !== undefined && jsonObj.millis !== undefined) {
+        const newLogData: EvaluatorLogData = {
+          queryCausingWork: jsonObj.queryCausingWork,
+          predicateName: jsonObj.predicateName,
+          millis: jsonObj.millis,
+          resultSize: jsonObj.resultSize,
+          ra: jsonObj.ra
+        };
+        visualizerData.push(newLogData);
+      }
+    } 
+    return visualizerData;
+}

extensions/ql-vscode/src/pure/log-summary-parser.ts

@@ -3,6 +3,7 @@ import * as os from 'os';
 import * as path from 'path';
 import { Disposable, ExtensionContext } from 'vscode';
 import { logger } from './logging';
+import { QueryHistoryManager } from './query-history';
 
 const LAST_SCRUB_TIME_KEY = 'lastScrubTime';
 
@@ -30,12 +31,13 @@ export function registerQueryHistoryScubber(
   throttleTime: number,
   maxQueryTime: number,
   queryDirectory: string,
+  qhm: QueryHistoryManager,
   ctx: ExtensionContext,
 
   // optional counter to keep track of how many times the scrubber has run
   counter?: Counter
 ): Disposable {
-  const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, ctx, counter);
+  const deregister = setInterval(scrubQueries, wakeInterval, throttleTime, maxQueryTime, queryDirectory, qhm, ctx, counter);
 
   return {
     dispose: () => {
@@ -48,6 +50,7 @@ async function scrubQueries(
   throttleTime: number,
   maxQueryTime: number,
   queryDirectory: string,
+  qhm: QueryHistoryManager,
   ctx: ExtensionContext,
   counter?: Counter
 ) {
@@ -89,6 +92,7 @@ async function scrubQueries(
     } finally {
       void logger.log(`Scrubbed ${scrubCount} old queries.`);
     }
+    await qhm.removeDeletedQueries();
   }
 }