Merge pull request #3233 from github/dbartol/mrva-multi-bundle

Use CLI to bundle packs for MRVA

Author: Dave Bartolomeo

extensions/ql-vscode/src/codeql-cli/cli-command.ts

@@ -3,34 +3,38 @@ import { promisify } from "util";
 
 import type { BaseLogger } from "../common/logging";
 import type { ProgressReporter } from "../common/logging/vscode";
-import { getChildProcessErrorMessage } from "../common/helpers-pure";
+import {
+  getChildProcessErrorMessage,
+  getErrorMessage,
+} from "../common/helpers-pure";
 
 /**
  * Flags to pass to all cli commands.
  */
 export const LOGGING_FLAGS = ["-v", "--log-to-stderr"];
 
 /**
- * Runs a CodeQL CLI command without invoking the CLI server, returning the output as a string.
+ * Runs a CodeQL CLI command without invoking the CLI server, deserializing the output as JSON.
  * @param codeQlPath The path to the CLI.
  * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
  * @param commandArgs The arguments to pass to the `codeql` command.
  * @param description Description of the action being run, to be shown in log and error messages.
  * @param logger Logger to write command log messages, e.g. to an output channel.
  * @param progressReporter Used to output progress messages, e.g. to the status bar.
- * @returns The contents of the command's stdout, if the command succeeded.
+ * @returns A JSON object parsed from the contents of the command's stdout, if the command succeeded.
  */
-export async function runCodeQlCliCommand(
+export async function runJsonCodeQlCliCommand<OutputType>(
   codeQlPath: string,
   command: string[],
   commandArgs: string[],
   description: string,
   logger: BaseLogger,
   progressReporter?: ProgressReporter,
-): Promise<string> {
+): Promise<OutputType> {
   // Add logging arguments first, in case commandArgs contains positional parameters.
   const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
   const argsString = args.join(" ");
+  let stdout: string;
   try {
     if (progressReporter !== undefined) {
       progressReporter.report({ message: description });
@@ -41,10 +45,18 @@ export async function runCodeQlCliCommand(
     const result = await promisify(execFile)(codeQlPath, args);
     void logger.log(result.stderr);
     void logger.log("CLI command succeeded.");
-    return result.stdout;
+    stdout = result.stdout;
   } catch (err) {
     throw new Error(
       `${description} failed: ${getChildProcessErrorMessage(err)}`,
     );
   }
+
+  try {
+    return JSON.parse(stdout) as OutputType;
+  } catch (err) {
+    throw new Error(
+      `Parsing output of ${description} failed: ${getErrorMessage(err)}`,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,25 +1,49 @@
 import type { SemVer } from "semver";
 import { parse } from "semver";
-import { runCodeQlCliCommand } from "./cli-command";
+import { runJsonCodeQlCliCommand } from "./cli-command";
 import type { Logger } from "../common/logging";
 import { getErrorMessage } from "../common/helpers-pure";
 
+interface VersionResult {
+  version: string;
+  features: CliFeatures | undefined;
+}
+
+export interface CliFeatures {
+  featuresInVersionResult?: boolean;
+  mrvaPackCreate?: boolean;
+}
+
+export interface VersionAndFeatures {
+  version: SemVer;
+  features: CliFeatures;
+}
+
 /**
  * Get the version of a CodeQL CLI.
  */
 export async function getCodeQlCliVersion(
   codeQlPath: string,
   logger: Logger,
-): Promise<SemVer | undefined> {
+): Promise<VersionAndFeatures | undefined> {
   try {
-    const output: string = await runCodeQlCliCommand(
+    const output: VersionResult = await runJsonCodeQlCliCommand<VersionResult>(
       codeQlPath,
       ["version"],
-      ["--format=terse"],
+      ["--format=json"],
       "Checking CodeQL version",
       logger,
     );
-    return parse(output.trim()) || undefined;
+
+    const version = parse(output.version.trim()) || undefined;
+    if (version === undefined) {
+      return undefined;
+    }
+
+    return {
+      version,
+      features: output.features ?? {},
+    };
   } catch (e) {
     // Failed to run the version command. This might happen if the cli version is _really_ old, or it is corrupted.
     // Either way, we can't determine compatibility.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -34,6 +34,7 @@ import { QueryLanguage } from "../common/query-language";
 import { LINE_ENDINGS, splitStreamAtSeparators } from "../common/split-stream";
 import type { Position } from "../query-server/messages";
 import { LOGGING_FLAGS } from "./cli-command";
+import type { CliFeatures, VersionAndFeatures } from "./cli-version";
 
 /**
  * The version of the SARIF format that we are using.
@@ -203,7 +204,9 @@ type OnLineCallback = (
   line: string,
 ) => Promise<string | undefined> | string | undefined;
 
-type VersionChangedListener = (newVersion: SemVer | undefined) => void;
+type VersionChangedListener = (
+  newVersionAndFeatures: VersionAndFeatures | undefined,
+) => void;
 
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
@@ -221,8 +224,8 @@ export class CodeQLCliServer implements Disposable {
   /**  A buffer with a single null byte. */
   nullBuffer: Buffer;
 
-  /** Version of current cli, lazily computed by the `getVersion()` method */
-  private _version: SemVer | undefined;
+  /** Version of current cli and its supported features, lazily computed by the `getVersion()` method */
+  private _versionAndFeatures: VersionAndFeatures | undefined;
 
   private _versionChangedListeners: VersionChangedListener[] = [];
 
@@ -298,7 +301,7 @@ export class CodeQLCliServer implements Disposable {
     const callback = (): void => {
       try {
         this.killProcessIfRunning();
-        this._version = undefined;
+        this._versionAndFeatures = undefined;
         this._supportedLanguages = undefined;
       } finally {
         this.runNext();
@@ -1427,16 +1430,28 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Compile a CodeQL pack and bundle it into a single file.
+   *
+   * @param sourcePackDir The directory of the input CodeQL pack.
+   * @param workspaceFolders The workspace folders to search for additional packs.
+   * @param outputBundleFile The path to the output bundle file.
+   * @param outputPackDir The directory to contain the unbundled output pack.
+   * @param moreOptions Additional options to be passed to `codeql pack bundle`.
+   */
   async packBundle(
-    dir: string,
+    sourcePackDir: string,
     workspaceFolders: string[],
-    outputPath: string,
+    outputBundleFile: string,
+    outputPackDir: string,
     moreOptions: string[],
   ): Promise<void> {
     const args = [
       "-o",
-      outputPath,
-      dir,
+      outputBundleFile,
+      sourcePackDir,
+      "--pack-path",
+      outputPackDir,
       ...moreOptions,
       ...this.getAdditionalPacksArg(workspaceFolders),
     ];
@@ -1491,27 +1506,35 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
-  public async getVersion() {
-    if (!this._version) {
+  public async getVersion(): Promise<SemVer> {
+    return (await this.getVersionAndFeatures()).version;
+  }
+
+  public async getFeatures(): Promise<CliFeatures> {
+    return (await this.getVersionAndFeatures()).features;
+  }
+
+  private async getVersionAndFeatures(): Promise<VersionAndFeatures> {
+    if (!this._versionAndFeatures) {
       try {
-        const newVersion = await this.refreshVersion();
-        this._version = newVersion;
+        const newVersionAndFeatures = await this.refreshVersion();
+        this._versionAndFeatures = newVersionAndFeatures;
         this._versionChangedListeners.forEach((listener) =>
-          listener(newVersion),
+          listener(newVersionAndFeatures),
         );
 
         // this._version is only undefined upon config change, so we reset CLI-based context key only when necessary.
         await this.app.commands.execute(
           "setContext",
           "codeql.supportsQuickEvalCount",
-          newVersion.compare(
+          newVersionAndFeatures.version.compare(
             CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
           ) >= 0,
         );
         await this.app.commands.execute(
           "setContext",
           "codeql.supportsTrimCache",
-          newVersion.compare(
+          newVersionAndFeatures.version.compare(
             CliVersionConstraint.CLI_VERSION_WITH_TRIM_CACHE,
           ) >= 0,
         );
@@ -1522,23 +1545,23 @@ export class CodeQLCliServer implements Disposable {
         throw e;
       }
     }
-    return this._version;
+    return this._versionAndFeatures;
   }
 
   public addVersionChangedListener(listener: VersionChangedListener) {
-    if (this._version) {
-      listener(this._version);
+    if (this._versionAndFeatures) {
+      listener(this._versionAndFeatures);
     }
     this._versionChangedListeners.push(listener);
   }
 
-  private async refreshVersion() {
+  private async refreshVersion(): Promise<VersionAndFeatures> {
     const distribution = await this.distributionProvider.getDistribution();
     switch (distribution.kind) {
       case FindDistributionResultKind.CompatibleDistribution:
       // eslint-disable-next-line no-fallthrough -- Intentional fallthrough
       case FindDistributionResultKind.IncompatibleDistribution:
-        return distribution.version;
+        return distribution.versionAndFeatures;
 
       default:
         // We should not get here because if no distributions are available, then
@@ -1755,4 +1778,8 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_EXTENSIBLE_PREDICATE_METADATA,
     );
   }
+
+  async supportsMrvaPackCreate(): Promise<boolean> {
+    return (await this.cli.getFeatures()).mrvaPackCreate === true;
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -1,11 +1,11 @@
 import { createWriteStream, mkdtemp, pathExists, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
-import type { SemVer } from "semver";
 import { Range, satisfies } from "semver";
 import type { Event, ExtensionContext } from "vscode";
 import type { DistributionConfig } from "../config";
 import { extLogger } from "../common/logging/vscode";
+import type { VersionAndFeatures } from "./cli-version";
 import { getCodeQlCliVersion } from "./cli-version";
 import type { ProgressCallback } from "../common/vscode/progress";
 import { reportStreamProgress } from "../common/vscode/progress";
@@ -88,11 +88,11 @@ export class DistributionManager implements DistributionProvider {
         kind: FindDistributionResultKind.NoDistribution,
       };
     }
-    const version = await getCodeQlCliVersion(
+    const versionAndFeatures = await getCodeQlCliVersion(
       distribution.codeQlPath,
       extLogger,
     );
-    if (version === undefined) {
+    if (versionAndFeatures === undefined) {
       return {
         distribution,
         kind: FindDistributionResultKind.UnknownCompatibilityDistribution,
@@ -119,17 +119,21 @@ export class DistributionManager implements DistributionProvider {
       distribution.kind !== DistributionKind.ExtensionManaged ||
       this.config.includePrerelease;
 
-    if (!satisfies(version, this.versionRange, { includePrerelease })) {
+    if (
+      !satisfies(versionAndFeatures.version, this.versionRange, {
+        includePrerelease,
+      })
+    ) {
       return {
         distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
-        version,
+        versionAndFeatures,
       };
     }
     return {
       distribution,
       kind: FindDistributionResultKind.CompatibleDistribution,
-      version,
+      versionAndFeatures,
     };
   }
 
@@ -599,7 +603,7 @@ interface DistributionResult {
 
 interface CompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.CompatibleDistribution;
-  version: SemVer;
+  versionAndFeatures: VersionAndFeatures;
 }
 
 interface UnknownCompatibilityDistributionResult extends DistributionResult {
@@ -608,7 +612,7 @@ interface UnknownCompatibilityDistributionResult extends DistributionResult {
 
 interface IncompatibleDistributionResult extends DistributionResult {
   kind: FindDistributionResultKind.IncompatibleDistribution;
-  version: SemVer;
+  versionAndFeatures: VersionAndFeatures;
 }
 
 interface NoDistributionResult {