Merge remote-tracking branch 'origin/main' into dbartol/new-test-ui

Author: Dave Bartolomeo

extensions/ql-vscode/CHANGELOG.md

@@ -2,8 +2,12 @@
 
 ## [UNRELEASED]
 
+## 1.8.2 - 12 April 2023
+
+- Fix bug where users could end up with the managed CodeQL CLI getting uninstalled during upgrades and not reinstalled. [#2294](https://github.com/github/vscode-codeql/pull/2294)
 - Fix bug that was causing code flows to not get updated when switching between results. [#2288](https://github.com/github/vscode-codeql/pull/2288)
 - Restart the CodeQL language server whenever the _CodeQL: Restart Query Server_ command is invoked. This avoids bugs where the CLI version changes to support new language features, but the language server is not updated. [#2238](https://github.com/github/vscode-codeql/pull/2238)
+- Avoid requiring a manual restart of the query server when the [external CLI config file](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file) changes. [#2289](https://github.com/github/vscode-codeql/pull/2289)
 
 ## 1.8.1 - 23 March 2023
 

extensions/ql-vscode/package-lock.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.8.2",
+  "version": "1.8.3",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -361,6 +361,10 @@
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
       },
+      {
+        "command": "codeQL.createSkeletonQuery",
+        "title": "CodeQL: Create Query"
+      },
       {
         "command": "codeQL.openDocumentation",
         "title": "CodeQL: Open Documentation"

extensions/ql-vscode/package.json

@@ -71,6 +71,7 @@ export type BaseCommands = {
   "codeQL.restartQueryServer": () => Promise<void>;
   "codeQL.restartQueryServerOnConfigChange": () => Promise<void>;
   "codeQL.restartLegacyQueryServerOnConfigChange": () => Promise<void>;
+  "codeQL.restartQueryServerOnExternalConfigChange": () => Promise<void>;
 };
 
 // Commands used when working with queries in the editor
@@ -98,6 +99,7 @@ export type LocalQueryCommands = {
   "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
   "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
   "codeQL.quickQuery": () => Promise<void>;
+  "codeQL.createSkeletonQuery": () => Promise<void>;
 };
 
 export type ResultsViewCommands = {

extensions/ql-vscode/src/common/commands.ts

@@ -1,7 +1,7 @@
 import fetch, { Response } from "node-fetch";
 import { zip } from "zip-a-folder";
 import { Open } from "unzipper";
-import { Uri, CancellationToken, window } from "vscode";
+import { Uri, CancellationToken, window, InputBoxOptions } from "vscode";
 import { CodeQLCliServer } from "./cli";
 import {
   ensureDir,
@@ -78,6 +78,10 @@ export async function promptImportInternetDatabase(
  *
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
+ * @param credentials the credentials to use to authenticate with GitHub
+ * @param progress the progress callback
+ * @param token the cancellation token
+ * @param cli the CodeQL CLI server
  */
 export async function promptImportGithubDatabase(
   commandManager: AppCommandManager,
@@ -88,21 +92,78 @@ export async function promptImportGithubDatabase(
   token: CancellationToken,
   cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
-  progress({
+  const githubRepo = await askForGitHubRepo(progress);
+  if (!githubRepo) {
+    return;
+  }
+
+  const databaseItem = await downloadGitHubDatabase(
+    githubRepo,
+    databaseManager,
+    storagePath,
+    credentials,
+    progress,
+    token,
+    cli,
+  );
+
+  if (databaseItem) {
+    await commandManager.execute("codeQLDatabases.focus");
+    void showAndLogInformationMessage(
+      "Database downloaded and imported successfully.",
+    );
+    return databaseItem;
+  }
+
+  return;
+}
+
+export async function askForGitHubRepo(
+  progress?: ProgressCallback,
+  suggestedValue?: string,
+): Promise<string | undefined> {
+  progress?.({
     message: "Choose repository",
     step: 1,
     maxStep: 2,
   });
-  const githubRepo = await window.showInputBox({
+
+  const options: InputBoxOptions = {
     title:
       'Enter a GitHub repository URL or "name with owner" (e.g. https://github.com/github/codeql or github/codeql)',
     placeHolder: "https://github.com/<owner>/<repo> or <owner>/<repo>",
     ignoreFocusOut: true,
-  });
-  if (!githubRepo) {
-    return;
+  };
+
+  if (suggestedValue) {
+    options.value = suggestedValue;
   }
 
+  return await window.showInputBox(options);
+}
+
+/**
+ * Downloads a database from GitHub
+ *
+ * @param githubRepo the GitHub repository to download the database from
+ * @param databaseManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ * @param credentials the credentials to use to authenticate with GitHub
+ * @param progress the progress callback
+ * @param token the cancellation token
+ * @param cli the CodeQL CLI server
+ * @param language the language to download. If undefined, the user will be prompted to choose a language.
+ **/
+export async function downloadGitHubDatabase(
+  githubRepo: string,
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  credentials: Credentials | undefined,
+  progress: ProgressCallback,
+  token: CancellationToken,
+  cli?: CodeQLCliServer,
+  language?: string,
+): Promise<DatabaseItem | undefined> {
   const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo;
   if (!isValidGitHubNwo(nwo)) {
     throw new Error(`Invalid GitHub repository: ${githubRepo}`);
@@ -112,7 +173,12 @@ export async function promptImportGithubDatabase(
     ? await credentials.getOctokit()
     : new Octokit.Octokit({ retry });
 
-  const result = await convertGithubNwoToDatabaseUrl(nwo, octokit, progress);
+  const result = await convertGithubNwoToDatabaseUrl(
+    nwo,
+    octokit,
+    progress,
+    language,
+  );
   if (!result) {
     return;
   }
@@ -130,7 +196,7 @@ export async function promptImportGithubDatabase(
    * We only need the actual token string.
    */
   const octokitToken = ((await octokit.auth()) as { token: string })?.token;
-  const item = await databaseArchiveFetcher(
+  return await databaseArchiveFetcher(
     databaseUrl,
     {
       Accept: "application/zip",
@@ -143,14 +209,6 @@ export async function promptImportGithubDatabase(
     token,
     cli,
   );
-  if (item) {
-    await commandManager.execute("codeQLDatabases.focus");
-    void showAndLogInformationMessage(
-      "Database downloaded and imported successfully.",
-    );
-    return item;
-  }
-  return;
 }
 
 /**
@@ -450,6 +508,7 @@ export async function convertGithubNwoToDatabaseUrl(
   nwo: string,
   octokit: Octokit.Octokit,
   progress: ProgressCallback,
+  language?: string,
 ): Promise<
   | {
       databaseUrl: string;
@@ -468,9 +527,11 @@ export async function convertGithubNwoToDatabaseUrl(
 
     const languages = response.data.map((db: any) => db.language);
 
-    const language = await promptForLanguage(languages, progress);
-    if (!language) {
-      return;
+    if (!language || !languages.includes(language)) {
+      language = await promptForLanguage(languages, progress);
+      if (!language) {
+        return;
+      }
     }
 
     return {
@@ -484,7 +545,7 @@ export async function convertGithubNwoToDatabaseUrl(
   }
 }
 
-async function promptForLanguage(
+export async function promptForLanguage(
   languages: string[],
   progress: ProgressCallback,
 ): Promise<string | undefined> {

extensions/ql-vscode/src/databaseFetcher.ts

@@ -13,12 +13,13 @@ import {
   workspace,
 } from "vscode";
 import { LanguageClient } from "vscode-languageclient/node";
-import { arch, platform } from "os";
+import { arch, platform, homedir } from "os";
 import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { dirSync } from "tmp-promise";
 import { testExplorerExtensionId, TestHub } from "vscode-test-adapter-api";
 import { lt, parse } from "semver";
+import { watch } from "chokidar";
 
 import { AstViewer } from "./astViewer";
 import {
@@ -197,6 +198,7 @@ function getCommands(
     "codeQL.restartQueryServer": restartQueryServer,
     "codeQL.restartQueryServerOnConfigChange": restartQueryServer,
     "codeQL.restartLegacyQueryServerOnConfigChange": restartQueryServer,
+    "codeQL.restartQueryServerOnExternalConfigChange": restartQueryServer,
     "codeQL.copyVersion": async () => {
       const text = `CodeQL extension version: ${
         extension?.packageJSON.version
@@ -675,6 +677,7 @@ async function activateWithInstalledDistribution(
     extLogger,
   );
   ctx.subscriptions.push(cliServer);
+  watchExternalConfigFile(app, ctx);
 
   const statusBar = new CodeQlStatusBarHandler(
     cliServer,
@@ -1023,6 +1026,34 @@ async function activateWithInstalledDistribution(
   };
 }
 
+/**
+ * Handle changes to the external config file. This is used to restart the query server
+ * when the user changes options.
+ * See https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file
+ */
+function watchExternalConfigFile(app: ExtensionApp, ctx: ExtensionContext) {
+  const home = homedir();
+  if (home) {
+    const configPath = join(home, ".config", "codeql", "config");
+    const configWatcher = watch(configPath, {
+      // These options avoid firing the event twice.
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: true,
+    });
+    configWatcher.on("all", async () => {
+      await app.commands.execute(
+        "codeQL.restartQueryServerOnExternalConfigChange",
+      );
+    });
+    ctx.subscriptions.push({
+      dispose: () => {
+        void configWatcher.close();
+      },
+    });
+  }
+}
+
 async function showResultsForComparison(
   compareView: CompareView,
   from: CompletedLocalQueryInfo,

extensions/ql-vscode/src/extension.ts

@@ -8,7 +8,7 @@ import {
   window,
 } from "vscode";
 import { BaseLogger, extLogger, Logger, TeeLogger } from "./common";
-import { MAX_QUERIES } from "./config";
+import { isCanary, MAX_QUERIES } from "./config";
 import { gatherQlFiles } from "./pure/files";
 import { basename } from "path";
 import {
@@ -51,6 +51,7 @@ import { App } from "./common/app";
 import { DisposableObject } from "./pure/disposable-object";
 import { QueryResultType } from "./pure/new-messages";
 import { redactableError } from "./pure/errors";
+import { SkeletonQueryWizard } from "./skeleton-query-wizard";
 
 interface DatabaseQuickPickItem extends QuickPickItem {
   databaseItem: DatabaseItem;
@@ -237,6 +238,7 @@ export class LocalQueries extends DisposableObject {
       "codeQL.quickEvalContextEditor": this.quickEval.bind(this),
       "codeQL.codeLensQuickEval": this.codeLensQuickEval.bind(this),
       "codeQL.quickQuery": this.quickQuery.bind(this),
+      "codeQL.createSkeletonQuery": this.createSkeletonQuery.bind(this),
     };
   }
 
@@ -375,6 +377,26 @@ export class LocalQueries extends DisposableObject {
     );
   }
 
+  private async createSkeletonQuery(): Promise<void> {
+    await withProgress(
+      async (progress: ProgressCallback, token: CancellationToken) => {
+        const credentials = isCanary() ? this.app.credentials : undefined;
+        const skeletonQueryWizard = new SkeletonQueryWizard(
+          this.cliServer,
+          progress,
+          credentials,
+          extLogger,
+          this.databaseManager,
+          token,
+        );
+        await skeletonQueryWizard.execute();
+      },
+      {
+        title: "Create Query",
+      },
+    );
+  }
+
   /**
    * Creates a new `LocalQueryRun` object to track a query evaluation. This creates a timestamp
    * file in the query's output directory, creates a `LocalQueryInfo` object, and registers that

extensions/ql-vscode/src/local-queries.ts

@@ -66,8 +66,8 @@ export class QlPackGenerator {
     await writeFile(qlPackFilePath, this.header + dump(qlPackYml), "utf8");
   }
 
-  private async createExampleQlFile() {
-    const exampleQlFilePath = join(this.folderUri.fsPath, "example.ql");
+  public async createExampleQlFile(fileName = "example.ql") {
+    const exampleQlFilePath = join(this.folderUri.fsPath, fileName);
 
     const exampleQl = `
 /**