Fix AST viewer for refactored language packs

Most of the languages have recently been refactored into separate library and query packs, with the contextual queries defined in the query pack. In the near future, these contextual queries will move to the library pack.

Current CLI releases throw an error in `codeql resolve queries` when the extension tries to search the library pack for contextual queries. This change makes two related fixes:

1. If the queries are not found in the library pack, it then scans the corresponding standard query pack as a fallback.
2. It detects the problematic combination of CLI and packs, and avoids scanning the library pack at all in those cases. If no queries are found in the problematic scenario, the error message instructs the user to upgrade to the latest CLI version, instead of claiming that the language simply doesn't support the contextual queries yet.

This change depends on CLI 2.6.1, which is being released soon, adding the `--allow-library-packs` option to `codeql resolve queries`. That PR is already open against the CLI.

Author: Dave Bartolomeo

extensions/ql-vscode/src/cli.ts

@@ -775,11 +775,15 @@ export class CodeQLCliServer implements Disposable {
    *   the default CLI search path is used.
    * @returns A list of query files found.
    */
-  resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
+  async resolveQueriesInSuite(suite: string, additionalPacks: string[], searchPath?: string[]): Promise<string[]> {
     const args = ['--additional-packs', additionalPacks.join(path.delimiter)];
     if (searchPath !== undefined) {
       args.push('--search-path', path.join(...searchPath));
     }
+    if (await this.cliConstraints.supportsAllowLibraryPacksInResolveQueries()) {
+      // All of our usage of `codeql resolve queries` needs to handle library packs.
+      args.push('--allow-library-packs');
+    }
     args.push(suite);
     return this.runJsonCodeQlCliCommand<string[]>(
       ['resolve', 'queries'],
@@ -1064,6 +1068,12 @@ export class CliVersionConstraint {
    */
   public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
 
+  /**
+   * CLI version where the `--allow-library-packs` option to `codeql resolve queries` was
+   * introduced.
+   */
+  public static CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES = new SemVer('2.6.1');
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1088,6 +1098,10 @@ export class CliVersionConstraint {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_QLREF);
   }
 
+  public async supportsAllowLibraryPacksInResolveQueries() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_ALLOW_LIBRARY_PACKS_IN_RESOLVE_QUERIES);
+  }
+
   async supportsDatabaseRegistration() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_WITH_DB_REGISTRATION);
   }

extensions/ql-vscode/src/contextual/queryResolver.ts

@@ -11,8 +11,9 @@ import {
 } from './keyType';
 import { CodeQLCliServer } from '../cli';
 import { DatabaseItem } from '../databases';
+import { QlPacksForLanguage } from '../helpers';
 
-export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<string> {
+export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem): Promise<QlPacksForLanguage> {
   if (db.contents === undefined) {
     throw new Error('Database is invalid and cannot infer QLPack.');
   }
@@ -21,28 +22,87 @@ export async function qlpackOfDatabase(cli: CodeQLCliServer, db: DatabaseItem):
   return await helpers.getQlPackForDbscheme(cli, dbscheme);
 }
 
+/**
+ * Finds the contextual queries with the specified key in a list of CodeQL packs.
+ *
+ * @param cli The CLI instance to use.
+ * @param qlpacks The list of packs to search.
+ * @param keyType The contextual query key of the query to search for.
+ * @returns The found queries from the first pack in which any matching queries were found.
+ */
+async function resolveQueriesFromPacks(cli: CodeQLCliServer, qlpacks: string[], keyType: KeyType): Promise<string[]> {
+  for (const qlpack of qlpacks) {
+    const suiteFile = (await tmp.file({
+      postfix: '.qls'
+    })).path;
+    const suiteYaml = {
+      qlpack,
+      include: {
+        kind: kindOfKeyType(keyType),
+        'tags contain': tagOfKeyType(keyType)
+      }
+    };
+    await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
 
-export async function resolveQueries(cli: CodeQLCliServer, qlpack: string, keyType: KeyType): Promise<string[]> {
-  const suiteFile = (await tmp.file({
-    postfix: '.qls'
-  })).path;
-  const suiteYaml = {
-    qlpack,
-    include: {
-      kind: kindOfKeyType(keyType),
-      'tags contain': tagOfKeyType(keyType)
+    const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
+    if (queries.length > 0) {
+      return queries;
     }
-  };
-  await fs.writeFile(suiteFile, yaml.safeDump(suiteYaml), 'utf8');
-
-  const queries = await cli.resolveQueriesInSuite(suiteFile, helpers.getOnDiskWorkspaceFolders());
-  if (queries.length === 0) {
-    void helpers.showAndLogErrorMessage(
-      `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
-      Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
-      for this language.`
-    );
-    throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} for qlpack ${qlpack}`);
   }
-  return queries;
+
+  return [];
+}
+
+export async function resolveQueries(cli: CodeQLCliServer, qlpacks: QlPacksForLanguage, keyType: KeyType): Promise<string[]> {
+  const cliCanHandleLibraryPack = await cli.cliConstraints.supportsAllowLibraryPacksInResolveQueries();
+  const packsToSearch: string[] = [];
+  let blameCli: boolean;
+
+  if (cliCanHandleLibraryPack) {
+    // The CLI can handle both library packs and query packs, so search both packs in order.
+    packsToSearch.push(qlpacks.dbschemePack);
+    if (qlpacks.queryPack !== undefined) {
+      packsToSearch.push(qlpacks.queryPack);
+    }
+    // If we don't find the query, it's because it's not there, not because the CLI was unable to
+    // search the pack.
+    blameCli = false;
+  } else {
+    // Older CLIs can't handle `codeql resolve queries` with a suite that references a library pack.
+    if (qlpacks.dbschemePackIsLibraryPack) {
+      if (qlpacks.queryPack !== undefined) {
+        // Just search the query pack, because some older library/query releases still had the
+        // contextual queries in the query pack.
+        packsToSearch.push(qlpacks.queryPack);
+      }
+      // If we don't find it, it's because the CLI was unable to search the library pack that
+      // actually contains the query. Blame any failure on the CLI, not the packs.
+      blameCli = true;
+    } else {
+      // We have an old CLI, but the dbscheme pack is old enough that it's still a unified pack with
+      // both libraries and queries. Just search that pack.
+      packsToSearch.push(qlpacks.dbschemePack);
+      // Any CLI should be able to search the single query pack, so if we don't find it, it's
+      // because the language doesn't support it.
+      blameCli = false;
+    }
+  }
+
+  const queries = await resolveQueriesFromPacks(cli, packsToSearch, keyType);
+  if (queries.length > 0) {
+    return queries;
+  }
+
+  // No queries found. Determine the correct error message for the various scenarios.
+  const errorMessage = blameCli ?
+    `Your current version of the CodeQL CLI, '${(await cli.getVersion()).version}', \
+    is unable to use contextual queries from recent versions of the standard CodeQL libraries. \
+    Please upgrade to the latest version of the CodeQL CLI.`
+    :
+    `No ${nameOfKeyType(keyType)} queries (tagged "${tagOfKeyType(keyType)}") could be found in the current library path. \
+    Try upgrading the CodeQL libraries. If that doesn't work, then ${nameOfKeyType(keyType)} queries are not yet available \
+    for this language.`;
+
+  void helpers.showAndLogErrorMessage(errorMessage);
+  throw new Error(`Couldn't find any queries tagged ${tagOfKeyType(keyType)} in any of the following packs: ${packsToSearch.join(', ')}.`);
 }

extensions/ql-vscode/src/contextual/templateProvider.ts

@@ -175,8 +175,8 @@ export class TemplatePrintAstProvider {
       throw new Error('Can\'t infer database from the provided source.');
     }
 
-    const qlpack = await qlpackOfDatabase(this.cli, db);
-    const queries = await resolveQueries(this.cli, qlpack, KeyType.PrintAstQuery);
+    const qlpacks = await qlpackOfDatabase(this.cli, db);
+    const queries = await resolveQueries(this.cli, qlpacks, KeyType.PrintAstQuery);
     if (queries.length > 1) {
       throw new Error('Found multiple Print AST queries. Can\'t continue');
     }

extensions/ql-vscode/src/helpers.ts

@@ -9,7 +9,7 @@ import {
   workspace,
   env
 } from 'vscode';
-import { CodeQLCliServer } from './cli';
+import { CodeQLCliServer, QlpacksInfo } from './cli';
 import { logger } from './logging';
 
 /**
@@ -254,9 +254,55 @@ function createRateLimitedResult(): RateLimitedResult {
   };
 }
 
-export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<string> {
+export interface QlPacksForLanguage {
+  /** The name of the pack containing the dbscheme. */
+  dbschemePack: string;
+  /** `true` if `dbschemePack` is a library pack. */
+  dbschemePackIsLibraryPack: boolean;
+  /**
+   * The name of the corresponding standard query pack.
+   * Only defined if `dbschemePack` is a library pack.
+   */
+  queryPack?: string;
+}
+
+interface QlPackWithPath {
+  packName: string;
+  packDir: string | undefined;
+}
+
+async function findDbschemePack(packs: QlPackWithPath[], dbschemePath: string): Promise<{ name: string; isLibraryPack: boolean; }> {
+  for (const { packDir, packName } of packs) {
+    if (packDir !== undefined) {
+      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme: string; library?: boolean; };
+      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
+        return {
+          name: packName,
+          isLibraryPack: qlpack.library === true
+        };
+      }
+    }
+  }
+  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+}
+
+function findStandardQueryPack(qlpacks: QlpacksInfo, dbschemePackName: string): string | undefined {
+  const matches = dbschemePackName.match(/^codeql\/(?<language>[a-z]+)-all$/);
+  if (matches) {
+    const queryPackName = `codeql/${matches.groups!.language}-queries`;
+    if (qlpacks[queryPackName] !== undefined) {
+      return queryPackName;
+    }
+  }
+
+  // Either the dbscheme pack didn't look like one where the queries might be in the query pack, or
+  // no query pack was found in the search path. Either is OK.
+  return undefined;
+}
+
+export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemePath: string): Promise<QlPacksForLanguage> {
   const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
-  const packs: { packDir: string | undefined; packName: string }[] =
+  const packs: QlPackWithPath[] =
     Object.entries(qlpacks).map(([packName, dirs]) => {
       if (dirs.length < 1) {
         void logger.log(`In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`);
@@ -270,15 +316,13 @@ export async function getQlPackForDbscheme(cliServer: CodeQLCliServer, dbschemeP
         packDir: dirs[0]
       };
     });
-  for (const { packDir, packName } of packs) {
-    if (packDir !== undefined) {
-      const qlpack = yaml.safeLoad(await fs.readFile(path.join(packDir, 'qlpack.yml'), 'utf8')) as { dbscheme: string };
-      if (qlpack.dbscheme !== undefined && path.basename(qlpack.dbscheme) === path.basename(dbschemePath)) {
-        return packName;
-      }
-    }
-  }
-  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+  const dbschemePack = await findDbschemePack(packs, dbschemePath);
+  const queryPack = dbschemePack.isLibraryPack ? findStandardQueryPack(qlpacks, dbschemePack.name) : undefined;
+  return {
+    dbschemePack: dbschemePack.name,
+    dbschemePackIsLibraryPack: dbschemePack.isLibraryPack,
+    queryPack
+  };
 }
 
 export async function getPrimaryDbscheme(datasetFolder: string): Promise<string> {

extensions/ql-vscode/src/quick-query.ts

@@ -110,7 +110,7 @@ export async function displayQuickQuery(
 
     const datasetFolder = await dbItem.getDatasetFolder(cliServer);
     const dbscheme = await getPrimaryDbscheme(datasetFolder);
-    const qlpack = await getQlPackForDbscheme(cliServer, dbscheme);
+    const qlpack = (await getQlPackForDbscheme(cliServer, dbscheme)).dbschemePack;
     const qlPackFile = path.join(queriesDir, 'qlpack.yml');
     const qlFile = path.join(queriesDir, QUICK_QUERY_QUERY_NAME);
     const shouldRewrite = await checkShouldRewrite(qlPackFile, qlpack);