Merge branch 'main' into selected-line-column

Author: aeisenberg

.github/codeql/queries/assert-pure.ql

@@ -19,7 +19,7 @@ class PureFile extends File {
       this.getRelativePath().regexpMatch(".*/src/pure/.*") or
       this.getRelativePath().regexpMatch(".*/src/common/.*")
     ) and
-    not this.getRelativePath().regexpMatch(".*/src/common/vscode/.*")
+    not this.getRelativePath().regexpMatch(".*/vscode/.*")
   }
 }
 

extensions/ql-vscode/package.json

@@ -516,6 +516,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+        "title": "Add repositories with GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -961,6 +965,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1297,6 +1306,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -218,7 +218,7 @@ export class CodeQLCliServer implements Disposable {
     private readonly app: App,
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    public readonly logger: Logger,
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -330,6 +330,7 @@ export class CodeQLCliServer implements Disposable {
     commandArgs: string[],
     description: string,
     onLine?: OnLineCallback,
+    silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
@@ -349,7 +350,12 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(" ");
-      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      // If we are running silently, we don't want to print anything to the console.
+      if (!silent) {
+        void this.logger.log(
+          `${description} using CodeQL CLI: ${argsString}...`,
+        );
+      }
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -395,24 +401,30 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        void this.logger.log("CLI command succeeded.");
+        if (!silent) {
+          void this.logger.log("CLI command succeeded.");
+        }
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
+        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
         const newError =
           stderrBuffers.length === 0
-            ? new Error(`${description} failed: ${err}`)
+            ? new Error(
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
+              )
             : new Error(
-                `${description} failed: ${Buffer.concat(stderrBuffers).toString(
-                  "utf8",
-                )}`,
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
+                  stderrBuffers,
+                ).toString("utf8")}`,
               );
         newError.stack += getErrorStack(err);
         throw newError;
       } finally {
-        void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent) {
+          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
         process.stderr.removeAllListeners("data");
@@ -549,9 +561,11 @@ export class CodeQLCliServer implements Disposable {
     {
       progressReporter,
       onLine,
+      silent = false,
     }: {
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<string> {
     if (progressReporter) {
@@ -567,6 +581,7 @@ export class CodeQLCliServer implements Disposable {
             commandArgs,
             description,
             onLine,
+            silent,
           ).then(resolve, reject);
         } catch (err) {
           reject(err);
@@ -600,10 +615,12 @@ export class CodeQLCliServer implements Disposable {
       addFormat = true,
       progressReporter,
       onLine,
+      silent = false,
     }: {
       addFormat?: boolean;
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
@@ -614,6 +631,7 @@ export class CodeQLCliServer implements Disposable {
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
       onLine,
+      silent,
     });
     try {
       return JSON.parse(result) as OutputType;
@@ -739,14 +757,19 @@ export class CodeQLCliServer implements Disposable {
   /**
    * Finds all available queries in a given directory.
    * @param queryDir Root of directory tree to search for queries.
+   * @param silent If true, don't print logs to the CodeQL extension log.
    * @returns The list of queries that were found.
    */
-  public async resolveQueries(queryDir: string): Promise<ResolvedQueries> {
+  public async resolveQueries(
+    queryDir: string,
+    silent?: boolean,
+  ): Promise<ResolvedQueries> {
     const subcommandArgs = [queryDir];
     return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
       ["resolve", "queries"],
       subcommandArgs,
       "Resolving queries",
+      { silent },
     );
   }
 
@@ -1050,13 +1073,15 @@ export class CodeQLCliServer implements Disposable {
     resultsPath: string,
     interpretedResultsPath: string,
     sourceInfo?: SourceInfo,
+    args?: string[],
   ): Promise<sarif.Log> {
     const additionalArgs = [
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
       "--no-group-results",
+      ...(args ?? []),
     ];
 
     await this.runInterpretCommand(

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -6,12 +6,7 @@ import * as semver from "semver";
 import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "../config";
-import {
-  InvocationRateLimiter,
-  InvocationRateLimiterResultKind,
-  showAndLogErrorMessage,
-  showAndLogWarningMessage,
-} from "../helpers";
+import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers";
 import { extLogger } from "../common";
 import { getCodeQlCliVersion } from "./cli-version";
 import {
@@ -24,6 +19,10 @@ import {
   extractZipArchive,
   getRequiredAssetName,
 } from "../pure/distribution";
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+} from "../common/invocation-rate-limiter";
 
 /**
  * distribution.ts
@@ -76,7 +75,7 @@ export class DistributionManager implements DistributionProvider {
         extensionContext,
       );
     this.updateCheckRateLimiter = new InvocationRateLimiter(
-      extensionContext,
+      extensionContext.globalState,
       "extensionSpecificDistributionUpdateCheck",
       () =>
         this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),

extensions/ql-vscode/src/common/commands.ts

@@ -275,6 +275,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {

extensions/ql-vscode/src/common/discovery.ts

@@ -1,25 +1,35 @@
 import { DisposableObject } from "../pure/disposable-object";
-import { extLogger } from "./logging/vscode/loggers";
 import { getErrorMessage } from "../pure/helpers-pure";
+import { Logger } from "./logging";
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
  * files. This class automatically prevents more than one discovery operation from running at the
  * same time.
  */
 export abstract class Discovery<T> extends DisposableObject {
-  private retry = false;
-  private discoveryInProgress = false;
+  private restartWhenFinished = false;
+  private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string) {
+  constructor(private readonly name: string, private readonly logger: Logger) {
     super();
   }
 
+  /**
+   * Returns the promise of the currently running refresh operation, if one is in progress.
+   * Otherwise returns a promise that resolves immediately.
+   */
+  public waitForCurrentRefresh(): Promise<void> {
+    return this.currentDiscoveryPromise ?? Promise.resolve();
+  }
+
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
+   *
+   * Returns a promise that resolves when the refresh is complete, including any retries.
    */
-  public refresh(): void {
+  public refresh(): Promise<void> {
     // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
     // got a storm of refresh requests due to, say, the copying or deletion of a large directory
     // tree, we could potentially spawn a separate simultaneous discovery operation for each
@@ -36,49 +46,48 @@ export abstract class Discovery<T> extends DisposableObject {
     // other change notifications that might be coming along. However, this would create more
     // latency in the common case, in order to save a bit of latency in the uncommon case.
 
-    if (this.discoveryInProgress) {
+    if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
-      this.retry = true;
+      this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
-      this.discoveryInProgress = true;
-      this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery().finally(() => {
+        this.currentDiscoveryPromise = undefined;
+      });
     }
+    return this.currentDiscoveryPromise;
   }
 
   /**
    * Starts the asynchronous discovery operation by invoking the `discover` function. When the
    * discovery operation completes, the `update` function will be invoked with the results of the
    * discovery.
    */
-  private launchDiscovery(): void {
-    const discoveryPromise = this.discover();
-    discoveryPromise
-      .then((results) => {
-        if (!this.retry) {
-          // Update any listeners with the results of the discovery.
-          this.discoveryInProgress = false;
-          this.update(results);
-        }
-      })
-
-      .catch((err: unknown) => {
-        void extLogger.log(
-          `${this.name} failed. Reason: ${getErrorMessage(err)}`,
-        );
-      })
+  private async launchDiscovery(): Promise<void> {
+    let results: T | undefined;
+    try {
+      results = await this.discover();
+    } catch (err) {
+      void this.logger.log(
+        `${this.name} failed. Reason: ${getErrorMessage(err)}`,
+      );
+      results = undefined;
+    }
 
-      .finally(() => {
-        if (this.retry) {
-          // Another refresh request came in while we were still running a previous discovery
-          // operation. Since the discovery results we just computed are now stale, we'll launch
-          // another discovery operation instead of updating.
-          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-          // initial discovery operation failed.
-          this.retry = false;
-          this.launchDiscovery();
-        }
-      });
+    if (this.restartWhenFinished) {
+      // Another refresh request came in while we were still running a previous discovery
+      // operation. Since the discovery results we just computed are now stale, we'll launch
+      // another discovery operation instead of updating.
+      // We want to relaunch discovery regardless of if the initial discovery operation
+      // succeeded or failed.
+      this.restartWhenFinished = false;
+      await this.launchDiscovery();
+    } else {
+      // If the discovery was successful, then update any listeners with the results.
+      if (results !== undefined) {
+        this.update(results);
+      }
+    }
   }
 
   /**