Merge remote-tracking branch 'origin/main' into koesie10/improve-test-setup

Author: koesie10

.github/workflows/release.yml

@@ -94,6 +94,23 @@ jobs:
           asset_name: ${{ format('vscode-codeql-{0}.vsix', steps.prepare-artifacts.outputs.ref_name) }}
           asset_content_type: application/zip
 
+      - name: Create sourcemap ZIP file
+        run: |
+          cd dist/vscode-codeql/out
+          zip -r ../../vscode-codeql-sourcemaps.zip *.map
+
+      - name: Upload sourcemap ZIP file
+        uses: actions/upload-release-asset@v1.0.1
+        if: success()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # Get the `upload_url` from the `create-release` step above.
+          upload_url: ${{ steps.create-release.outputs.upload_url }}
+          asset_path: dist/vscode-codeql-sourcemaps.zip
+          asset_name: ${{ format('vscode-codeql-sourcemaps-{0}.zip', steps.prepare-artifacts.outputs.ref_name) }}
+          asset_content_type: application/zip
+
       ###
       # Do Post release work: version bump and changelog PR
       # Only do this if we are running from a PR (ie- this is part of the release process)

CONTRIBUTING.md

@@ -229,6 +229,13 @@ Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's poss
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
     * Go through all recent PRs and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
+    * For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
+      * Making substantial new features available to all users. This can include lifting a feature flag.
+      * Breakage in compatibility with recent versions of the CLI.
+      * Minimum required version of VS Code is increased.
+      * New telemetry events are added.
+      * Deprecation or removal of commands.
+      * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
 1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
     * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
     * `.github/workflows/main.yml` - all the "node-version: <version>" settings

extensions/ql-vscode/package.json

@@ -365,7 +365,7 @@
       {
         "command": "codeQLVariantAnalysisRepositories.openConfigFile",
         "title": "Open Database Configuration File",
-        "icon": "$(edit)"
+        "icon": "$(json)"
       },
       {
         "command": "codeQLVariantAnalysisRepositories.addNewDatabase",
@@ -379,7 +379,7 @@
       },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
-        "title": "✓"
+        "title": "Select"
       },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItemContextMenu",
@@ -822,7 +822,7 @@
         },
         {
           "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
-          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeSelected/ && false",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeSelected/",
           "group": "inline"
         },
         {

extensions/ql-vscode/scripts/source-map.ts

@@ -20,6 +20,7 @@ import { spawnSync } from "child_process";
 import { basename, resolve } from "path";
 import { pathExists, readJSON } from "fs-extra";
 import { RawSourceMap, SourceMapConsumer } from "source-map";
+import { Open } from "unzipper";
 
 if (process.argv.length !== 4) {
   console.error(
@@ -36,6 +37,12 @@ const versionNumber = process.argv[2].startsWith("v")
 const stacktrace = process.argv[3];
 
 async function extractSourceMap() {
+  const releaseAssetsDirectory = resolve(
+    __dirname,
+    "..",
+    "release-assets",
+    versionNumber,
+  );
   const sourceMapsDirectory = resolve(
     __dirname,
     "..",
@@ -47,34 +54,64 @@ async function extractSourceMap() {
   if (!(await pathExists(sourceMapsDirectory))) {
     console.log("Downloading source maps...");
 
-    const workflowRuns = runGhJSON<WorkflowRunListItem[]>([
-      "run",
-      "list",
-      "--workflow",
-      "release.yml",
-      "--branch",
+    const release = runGhJSON<Release>([
+      "release",
+      "view",
       versionNumber,
       "--json",
-      "databaseId,number",
+      "id,name,assets",
     ]);
 
-    if (workflowRuns.length !== 1) {
-      throw new Error(
-        `Expected exactly one workflow run for ${versionNumber}, got ${workflowRuns.length}`,
+    const sourcemapAsset = release.assets.find(
+      (asset) => asset.name === `vscode-codeql-sourcemaps-${versionNumber}.zip`,
+    );
+
+    if (sourcemapAsset) {
+      // This downloads a ZIP file of the source maps
+      runGh([
+        "release",
+        "download",
+        versionNumber,
+        "--pattern",
+        sourcemapAsset.name,
+        "--dir",
+        releaseAssetsDirectory,
+      ]);
+
+      const file = await Open.file(
+        resolve(releaseAssetsDirectory, sourcemapAsset.name),
       );
+      await file.extract({ path: sourceMapsDirectory });
+    } else {
+      const workflowRuns = runGhJSON<WorkflowRunListItem[]>([
+        "run",
+        "list",
+        "--workflow",
+        "release.yml",
+        "--branch",
+        versionNumber,
+        "--json",
+        "databaseId,number",
+      ]);
+
+      if (workflowRuns.length !== 1) {
+        throw new Error(
+          `Expected exactly one workflow run for ${versionNumber}, got ${workflowRuns.length}`,
+        );
+      }
+
+      const workflowRun = workflowRuns[0];
+
+      runGh([
+        "run",
+        "download",
+        workflowRun.databaseId.toString(),
+        "--name",
+        "vscode-codeql-sourcemaps",
+        "--dir",
+        sourceMapsDirectory,
+      ]);
     }
-
-    const workflowRun = workflowRuns[0];
-
-    runGh([
-      "run",
-      "download",
-      workflowRun.databaseId.toString(),
-      "--name",
-      "vscode-codeql-sourcemaps",
-      "--dir",
-      sourceMapsDirectory,
-    ]);
   }
 
   if (stacktrace.includes("at")) {
@@ -172,6 +209,17 @@ function runGhJSON<T>(args: readonly string[]): T {
   return JSON.parse(runGh(args));
 }
 
+type ReleaseAsset = {
+  id: string;
+  name: string;
+};
+
+type Release = {
+  id: string;
+  name: string;
+  assets: ReleaseAsset[];
+};
+
 type WorkflowRunListItem = {
   databaseId: number;
   number: number;

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -19,6 +19,7 @@ import { CancellationToken, Uri } from "vscode";
 import { ProgressCallback } from "../commandRunner";
 import { QueryRunner } from "../queryRunner";
 import { redactableError } from "../pure/errors";
+import { QLPACK_FILENAMES } from "../pure/ql";
 
 export async function qlpackOfDatabase(
   cli: CodeQLCliServer,
@@ -113,7 +114,7 @@ async function resolveContextualQuery(
   // Work out the enclosing pack.
   const packContents = await cli.packPacklist(query, false);
   const packFilePath = packContents.find((p) =>
-    ["codeql-pack.yml", "qlpack.yml"].includes(basename(p)),
+    QLPACK_FILENAMES.includes(basename(p)),
   );
   if (packFilePath === undefined) {
     // Should not happen; we already resolved this query.

extensions/ql-vscode/src/extension.ts

@@ -845,6 +845,7 @@ async function activateWithInstalledDistribution(
       documentSelector: [
         { language: "ql", scheme: "file" },
         { language: "yaml", scheme: "file", pattern: "**/qlpack.yml" },
+        { language: "yaml", scheme: "file", pattern: "**/codeql-pack.yml" },
       ],
       synchronize: {
         configurationSection: "codeQL",

extensions/ql-vscode/src/helpers.ts

@@ -23,6 +23,7 @@ import { extLogger, OutputChannelLogger } from "./common";
 import { QueryMetadata } from "./pure/interface-types";
 import { telemetryListener } from "./telemetry";
 import { RedactableError } from "./pure/errors";
+import { getQlPackPath } from "./pure/ql";
 
 // Shared temporary folder for the extension.
 export const tmpDir = dirSync({
@@ -387,17 +388,22 @@ async function findDbschemePack(
 ): Promise<{ name: string; isLibraryPack: boolean }> {
   for (const { packDir, packName } of packs) {
     if (packDir !== undefined) {
-      const qlpack = load(
-        await readFile(join(packDir, "qlpack.yml"), "utf8"),
-      ) as { dbscheme?: string; library?: boolean };
-      if (
-        qlpack.dbscheme !== undefined &&
-        basename(qlpack.dbscheme) === basename(dbschemePath)
-      ) {
-        return {
-          name: packName,
-          isLibraryPack: qlpack.library === true,
+      const qlpackPath = await getQlPackPath(packDir);
+
+      if (qlpackPath !== undefined) {
+        const qlpack = load(await readFile(qlpackPath, "utf8")) as {
+          dbscheme?: string;
+          library?: boolean;
         };
+        if (
+          qlpack.dbscheme !== undefined &&
+          basename(qlpack.dbscheme) === basename(dbschemePath)
+        ) {
+          return {
+            name: packName,
+            isLibraryPack: qlpack.library === true,
+          };
+        }
       }
     }
   }

extensions/ql-vscode/src/pure/ql.ts

@@ -0,0 +1,19 @@
+import { join } from "path";
+import { pathExists } from "fs-extra";
+
+export const QLPACK_FILENAMES = ["qlpack.yml", "codeql-pack.yml"];
+export const FALLBACK_QLPACK_FILENAME = QLPACK_FILENAMES[0];
+
+export async function getQlPackPath(
+  packRoot: string,
+): Promise<string | undefined> {
+  for (const filename of QLPACK_FILENAMES) {
+    const path = join(packRoot, filename);
+
+    if (await pathExists(path)) {
+      return path;
+    }
+  }
+
+  return undefined;
+}

extensions/ql-vscode/src/quick-query.ts

@@ -19,6 +19,7 @@ import {
 } from "./helpers";
 import { ProgressCallback, UserCancellationException } from "./commandRunner";
 import { getErrorMessage } from "./pure/helpers-pure";
+import { FALLBACK_QLPACK_FILENAME, getQlPackPath } from "./pure/ql";
 
 const QUICK_QUERIES_DIR_NAME = "quick-queries";
 const QUICK_QUERY_QUERY_NAME = "quick-query.ql";
@@ -112,7 +113,7 @@ export async function displayQuickQuery(
     const dbscheme = await getPrimaryDbscheme(datasetFolder);
     const qlpack = (await getQlPackForDbscheme(cliServer, dbscheme))
       .dbschemePack;
-    const qlPackFile = join(queriesDir, "qlpack.yml");
+    const qlPackFile = await getQlPackPath(queriesDir);
     const qlFile = join(queriesDir, QUICK_QUERY_QUERY_NAME);
     const shouldRewrite = await checkShouldRewrite(qlPackFile, qlpack);
 
@@ -126,7 +127,7 @@ export async function displayQuickQuery(
         },
       };
       await writeFile(
-        qlPackFile,
+        qlPackFile ?? join(queriesDir, FALLBACK_QLPACK_FILENAME),
         QLPACK_FILE_HEADER + dump(quickQueryQlpackYaml),
         "utf8",
       );
@@ -158,7 +159,13 @@ export async function displayQuickQuery(
   }
 }
 
-async function checkShouldRewrite(qlPackFile: string, newDependency: string) {
+async function checkShouldRewrite(
+  qlPackFile: string | undefined,
+  newDependency: string,
+) {
+  if (!qlPackFile) {
+    return true;
+  }
   if (!(await pathExists(qlPackFile))) {
     return true;
   }