Add extension packs to variant analysis queries

This change allows the `codeQL.runningQueries.useExtensionPacks`
setting to be respected when running variant analysis queries. When
set to `all`, before uploading the generated query pack, all extension
packs in the workspace will be injected as dependencies into the qlpack
file.

Author: aeisenberg

extensions/ql-vscode/src/cli.ts

@@ -1284,11 +1284,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  async packInstall(dir: string, forceUpdate = false) {
+  async packInstall(
+    dir: string,
+    { forceUpdate = false, workspaceFolders = [] as string[] } = {},
+  ) {
     const args = [dir];
     if (forceUpdate) {
       args.push("--mode", "update");
     }
+    if (workspaceFolders?.length > 0) {
+      if (await this.cliConstraints.supportsAdditionalPacksInstall()) {
+        args.push(
+          // Allow prerelease packs from the ql submodule.
+          "--allow-prerelease",
+          // Allow the use of --additional-packs argument without issueing a warning
+          "--no-strict-mode",
+          ...this.getAdditionalPacksArg(workspaceFolders),
+        );
+      }
+    }
     return this.runJsonCodeQlCliCommandWithAuthentication(
       ["pack", "install"],
       args,
@@ -1692,6 +1706,13 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_QLPACKS_KIND = new SemVer("2.12.3");
 
+  /**
+   * CLI version that supports the `--additional-packs` option for the `pack install` command.
+   */
+  public static CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL = new SemVer(
+    "2.12.4",
+  );
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1755,4 +1776,10 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,
     );
   }
+
+  async supportsAdditionalPacksInstall() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
+    );
+  }
 }

extensions/ql-vscode/src/pure/ql.ts

@@ -2,6 +2,10 @@ import { join } from "path";
 import { pathExists } from "fs-extra";
 
 export const QLPACK_FILENAMES = ["qlpack.yml", "codeql-pack.yml"];
+export const QLPACK_LOCK_FILENAMES = [
+  "qlpack.lock.yml",
+  "codeql-pack.lock.yml",
+];
 export const FALLBACK_QLPACK_FILENAME = QLPACK_FILENAMES[0];
 
 export async function getQlPackPath(

extensions/ql-vscode/src/quick-query.ts

@@ -143,7 +143,7 @@ export async function displayQuickQuery(
 
     if (shouldRewrite) {
       await cliServer.clearCache();
-      await cliServer.packInstall(queriesDir, true);
+      await cliServer.packInstall(queriesDir, { forceUpdate: true });
     }
 
     await Window.showTextDocument(await workspace.openTextDocument(qlFile));

extensions/ql-vscode/src/variant-analysis/run-remote-query.ts

@@ -34,6 +34,7 @@ import {
   getQlPackPath,
   FALLBACK_QLPACK_FILENAME,
   QLPACK_FILENAMES,
+  QLPACK_LOCK_FILENAMES,
 } from "../pure/ql";
 
 export interface QlPack {
@@ -86,7 +87,6 @@ async function generateQueryPack(
       queryFile,
       queryPackDir,
       packRelativePath,
-      workspaceFolders,
     );
 
     language = await findLanguage(cliServer, Uri.file(targetQueryFileName));
@@ -102,8 +102,6 @@ async function generateQueryPack(
       targetQueryFileName,
       language,
       packRelativePath,
-      cliServer,
-      workspaceFolders,
     );
   }
   if (!language) {
@@ -125,11 +123,21 @@ async function generateQueryPack(
     precompilationOpts = ["--no-precompile"];
   }
 
+  if (await cliServer.useExtensionPacks()) {
+    await injectExtensionPacks(cliServer, queryPackDir, workspaceFolders);
+  }
+
+  await cliServer.packInstall(queryPackDir, {
+    workspaceFolders,
+  });
+
+  // Clear the CLI cache so that the most recent qlpack lock file is used.
+  await cliServer.clearCache();
+
   const bundlePath = await getPackedBundlePath(queryPackDir);
   void extLogger.log(
     `Compiling and bundling query pack from ${queryPackDir} to ${bundlePath}. (This may take a while.)`,
   );
-  await cliServer.packInstall(queryPackDir);
   await cliServer.packBundle(
     queryPackDir,
     workspaceFolders,
@@ -149,8 +157,6 @@ async function createNewQueryPack(
   targetQueryFileName: string,
   language: string | undefined,
   packRelativePath: string,
-  cliServer: cli.CodeQLCliServer,
-  workspaceFolders: string[],
 ) {
   void extLogger.log(`Copying ${queryFile} to ${queryPackDir}`);
   await copy(queryFile, targetQueryFileName);
@@ -163,9 +169,6 @@ async function createNewQueryPack(
     },
     defaultSuite: generateDefaultSuite(packRelativePath),
   };
-  if (await cliServer.useExtensionPacks()) {
-    injectExtensionPacks(cliServer, syntheticQueryPack, workspaceFolders);
-  }
   await writeFile(
     join(queryPackDir, FALLBACK_QLPACK_FILENAME),
     dump(syntheticQueryPack),
@@ -178,14 +181,12 @@ async function copyExistingQueryPack(
   queryFile: string,
   queryPackDir: string,
   packRelativePath: string,
-  workspaceFolders: string[],
 ) {
   const toCopy = await cliServer.packPacklist(originalPackRoot, false);
 
-  // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist.
   [
-    join(originalPackRoot, "qlpack.lock.yml"),
-    join(originalPackRoot, "codeql-pack.lock.yml"),
+    // also copy the lock file (either new name or old name) and the query file itself. These are not included in the packlist.
+    ...QLPACK_LOCK_FILENAMES.map((f) => join(originalPackRoot, f)),
     queryFile,
   ].forEach((absolutePath) => {
     if (absolutePath) {
@@ -211,12 +212,7 @@ async function copyExistingQueryPack(
 
   void extLogger.log(`Copied ${copiedCount} files to ${queryPackDir}`);
 
-  await fixPackFile(
-    queryPackDir,
-    packRelativePath,
-    cliServer,
-    workspaceFolders,
-  );
+  await fixPackFile(queryPackDir, packRelativePath);
 }
 
 async function findPackRoot(queryFile: string): Promise<string> {
@@ -367,8 +363,6 @@ export async function prepareRemoteQueryRun(
 async function fixPackFile(
   queryPackDir: string,
   packRelativePath: string,
-  cliServer: cli.CodeQLCliServer,
-  workspaceFolders: string[],
 ): Promise<void> {
   const packPath = await getQlPackPath(queryPackDir);
 
@@ -385,19 +379,26 @@ async function fixPackFile(
   qlpack.name = QUERY_PACK_NAME;
   updateDefaultSuite(qlpack, packRelativePath);
   removeWorkspaceRefs(qlpack);
-  if (await cliServer.useExtensionPacks()) {
-    injectExtensionPacks(cliServer, qlpack, workspaceFolders);
-  }
 
   await writeFile(packPath, dump(qlpack));
 }
 
-function injectExtensionPacks(
+async function injectExtensionPacks(
   cliServer: cli.CodeQLCliServer,
-  qlpack: QlPack,
+  queryPackDir: string,
   workspaceFolders: string[],
 ) {
-  const extensionPacks = cliServer.resolveQlpacks(workspaceFolders, true);
+  const qlpackFile = await getQlPackPath(queryPackDir);
+  if (!qlpackFile) {
+    throw new Error(
+      `Could not find ${QLPACK_FILENAMES.join(
+        " or ",
+      )} file in '${queryPackDir}'`,
+    );
+  }
+  const syntheticQueryPack = load(await readFile(qlpackFile, "utf8")) as QlPack;
+
+  const extensionPacks = await cliServer.resolveQlpacks(workspaceFolders, true);
   Object.entries(extensionPacks).forEach(([name, paths]) => {
     // We are guaranteed that there is at least one path found for each extension pack.
     // If there are multiple paths, then we have a problem. This means that there is
@@ -412,8 +413,10 @@ function injectExtensionPacks(
     // Add this extension pack as a dependency. It doesn't matter which
     // version we specify, since we are guaranteed that the extension pack
     // is resolved from source at the given path.
-    qlpack.dependencies[name] = "*";
+    syntheticQueryPack.dependencies[name] = "*";
   });
+  await writeFile(qlpackFile, dump(syntheticQueryPack));
+  await cliServer.clearCache();
 }
 
 function updateDefaultSuite(qlpack: QlPack, packRelativePath: string) {

extensions/ql-vscode/test/vscode-tests/cli-integration/data-extension-pack/extension-file.yml

@@ -0,0 +1,12 @@
+extensions:
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: sourceModel
+    data:
+      - [ "@example/read-write-user-data", "Member[readUserData].ReturnValue.Awaited", "remote" ]
+
+  - addsTo:
+      pack: codeql/javascript-all
+      extensible: sinkModel
+    data:
+      - [ "@example/read-write-user-data", "Member[writeUserData].Argument[0]", "command-line-injection" ]

extensions/ql-vscode/test/vscode-tests/cli-integration/data-extension-pack/qlpack.yml

@@ -0,0 +1,8 @@
+name: github/extension-pack-for-testing
+version: 0.0.0
+library: true
+extensionTargets:
+  github/remote-query-pack: "*"
+
+dataExtensions:
+  - extension-file.yml

extensions/ql-vscode/test/vscode-tests/cli-integration/variant-analysis/variant-analysis-manager.test.ts

@@ -12,7 +12,7 @@ import * as ghApiClient from "../../../../src/variant-analysis/gh-api/gh-api-cli
 import { join } from "path";
 
 import { VariantAnalysisManager } from "../../../../src/variant-analysis/variant-analysis-manager";
-import { CodeQLCliServer } from "../../../../src/cli";
+import { CliVersionConstraint, CodeQLCliServer } from "../../../../src/cli";
 import {
   fixWorkspaceReferences,
   restoreWorkspaceReferences,
@@ -255,18 +255,44 @@ describe("Variant Analysis Manager", () => {
           qlxFilesThatExist: ["subfolder/in-pack.qlx"],
         });
       });
+
+      it("should run a remote query with extension packs inside a qlpack", async () => {
+        if (!(await cli.cliConstraints.supportsQlpacksKind())) {
+          console.log(
+            `Skipping test because qlpacks kind is only suppported in CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND} or later.`,
+          );
+          return;
+        }
+        await cli.setUseExtensionPacks(true);
+        await doVariantAnalysisTest({
+          queryPath: "data-remote-qlpack-nested/subfolder/in-pack.ql",
+          filesThatExist: [
+            "subfolder/in-pack.ql",
+            "otherfolder/lib.qll",
+            ".codeql/libraries/semmle/targets-extension/0.0.0/ext/extension.yml",
+          ],
+          filesThatDoNotExist: ["subfolder/not-in-pack.ql"],
+          qlxFilesThatExist: ["subfolder/in-pack.qlx"],
+          dependenciesToCheck: [
+            "codeql/javascript-all",
+            "semmle/targets-extension",
+          ],
+        });
+      });
     });
 
     async function doVariantAnalysisTest({
       queryPath,
       filesThatExist,
       qlxFilesThatExist,
       filesThatDoNotExist,
+      dependenciesToCheck = ["codeql/javascript-all"],
     }: {
       queryPath: string;
       filesThatExist: string[];
       qlxFilesThatExist: string[];
       filesThatDoNotExist: string[];
+      dependenciesToCheck?: string[];
     }) {
       const fileUri = getFile(queryPath);
       await variantAnalysisManager.runVariantAnalysis(
@@ -328,8 +354,10 @@ describe("Variant Analysis Manager", () => {
 
       const actualLockKeys = Object.keys(qlpackLockContents.dependencies);
 
-      // The lock file should contain at least codeql/javascript-all.
-      expect(actualLockKeys).toContain("codeql/javascript-all");
+      // The lock file should contain at least the specified dependencies.
+      dependenciesToCheck.forEach((dep) =>
+        expect(actualLockKeys).toContain(dep),
+      );
     }
 
     function getFile(file: string): Uri {