Merge pull request #2504 from github/koesie10/split-helpers

Split helpers.ts file

Author: koesie10

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -22,11 +22,11 @@ import {
   getErrorMessage,
   getErrorStack,
 } from "../pure/helpers-pure";
+import { walkDirectory } from "../pure/files";
 import { QueryMetadata, SortDirection } from "../pure/interface-types";
 import { BaseLogger, Logger, ProgressReporter } from "../common";
 import { CompilationMessage } from "../pure/legacy-messages";
 import { sarifParser } from "../common/sarif-parser";
-import { walkDirectory } from "../helpers";
 import { App } from "../common/app";
 import { QueryLanguage } from "../common/query-language";
 

extensions/ql-vscode/src/common/query-language.ts

@@ -35,3 +35,7 @@ export const dbSchemeToLanguage = {
   "ruby.dbscheme": "ruby",
   "swift.dbscheme": "swift",
 };
+
+export function isQueryLanguage(language: string): language is QueryLanguage {
+  return Object.values(QueryLanguage).includes(language as QueryLanguage);
+}

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts

@@ -4,10 +4,10 @@ import { writeFile } from "fs-extra";
 import { dump as dumpYaml } from "js-yaml";
 import {
   getOnDiskWorkspaceFolders,
-  isQueryLanguage,
   showAndLogExceptionWithTelemetry,
 } from "../helpers";
 import { TeeLogger } from "../common";
+import { isQueryLanguage } from "../common/query-language";
 import { CancellationToken } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseItem } from "../databases/local-databases";

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -29,11 +29,13 @@ import {
   withProgress,
 } from "../common/vscode/progress";
 import {
-  isLikelyDatabaseRoot,
-  isLikelyDbLanguageFolder,
   showAndLogErrorMessage,
   showAndLogExceptionWithTelemetry,
 } from "../helpers";
+import {
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder,
+} from "./local-databases/db-contents-heuristics";
 import { extLogger } from "../common";
 import {
   importArchiveDatabase,

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -10,7 +10,7 @@ import {
   zipArchiveScheme,
 } from "../../common/vscode/archive-filesystem-provider";
 import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
-import { isLikelyDatabaseRoot } from "../../helpers";
+import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
 import { stat } from "fs-extra";
 import { pathsEqual } from "../../pure/files";
 import { DatabaseContents } from "./database-contents";

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -16,10 +16,10 @@ import { DatabaseItemImpl } from "./database-item-impl";
 import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
-  isQueryLanguage,
   showAndLogExceptionWithTelemetry,
   showNeverAskAgainDialog,
 } from "../../helpers";
+import { isQueryLanguage } from "../../common/query-language";
 import { existsSync } from "fs";
 import { QlPackGenerator } from "../../qlpack-generator";
 import { asError, getErrorMessage } from "../../pure/helpers-pure";

extensions/ql-vscode/src/databases/local-databases/db-contents-heuristics.ts

@@ -0,0 +1,35 @@
+import { pathExists } from "fs-extra";
+import { basename, join } from "path";
+import { glob } from "glob";
+
+/**
+ * The following functions al heuristically determine metadata about databases.
+ */
+
+/**
+ * Heuristically determines if the directory passed in corresponds
+ * to a database root. A database root is a directory that contains
+ * a codeql-database.yml or (historically) a .dbinfo file. It also
+ * contains a folder starting with `db-`.
+ */
+export async function isLikelyDatabaseRoot(maybeRoot: string) {
+  const [a, b, c] = await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    pathExists(join(maybeRoot, ".dbinfo")),
+    pathExists(join(maybeRoot, "codeql-database.yml")),
+
+    // they *must* have a db-{language} folder
+    glob("db-*/", { cwd: maybeRoot }),
+  ]);
+
+  return (a || b) && c.length > 0;
+}
+
+/**
+ * A language folder is any folder starting with `db-` that is itself not a database root.
+ */
+export async function isLikelyDbLanguageFolder(dbPath: string) {
+  return (
+    basename(dbPath).startsWith("db-") && !(await isLikelyDatabaseRoot(dbPath))
+  );
+}

extensions/ql-vscode/src/databases/qlpack.ts

@@ -0,0 +1,130 @@
+import { window } from "vscode";
+import { glob } from "glob";
+import { basename } from "path";
+import { load } from "js-yaml";
+import { readFile } from "fs-extra";
+import { getQlPackPath } from "../pure/ql";
+import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
+import { extLogger } from "../common";
+import { getOnDiskWorkspaceFolders } from "../helpers";
+
+export interface QlPacksForLanguage {
+  /** The name of the pack containing the dbscheme. */
+  dbschemePack: string;
+  /** `true` if `dbschemePack` is a library pack. */
+  dbschemePackIsLibraryPack: boolean;
+  /**
+   * The name of the corresponding standard query pack.
+   * Only defined if `dbschemePack` is a library pack.
+   */
+  queryPack?: string;
+}
+
+interface QlPackWithPath {
+  packName: string;
+  packDir: string | undefined;
+}
+
+async function findDbschemePack(
+  packs: QlPackWithPath[],
+  dbschemePath: string,
+): Promise<{ name: string; isLibraryPack: boolean }> {
+  for (const { packDir, packName } of packs) {
+    if (packDir !== undefined) {
+      const qlpackPath = await getQlPackPath(packDir);
+
+      if (qlpackPath !== undefined) {
+        const qlpack = load(await readFile(qlpackPath, "utf8")) as {
+          dbscheme?: string;
+          library?: boolean;
+        };
+        if (
+          qlpack.dbscheme !== undefined &&
+          basename(qlpack.dbscheme) === basename(dbschemePath)
+        ) {
+          return {
+            name: packName,
+            isLibraryPack: qlpack.library === true,
+          };
+        }
+      }
+    }
+  }
+  throw new Error(`Could not find qlpack file for dbscheme ${dbschemePath}`);
+}
+
+function findStandardQueryPack(
+  qlpacks: QlpacksInfo,
+  dbschemePackName: string,
+): string | undefined {
+  const matches = dbschemePackName.match(/^codeql\/(?<language>[a-z]+)-all$/);
+  if (matches) {
+    const queryPackName = `codeql/${matches.groups!.language}-queries`;
+    if (qlpacks[queryPackName] !== undefined) {
+      return queryPackName;
+    }
+  }
+
+  // Either the dbscheme pack didn't look like one where the queries might be in the query pack, or
+  // no query pack was found in the search path. Either is OK.
+  return undefined;
+}
+
+export async function getQlPackForDbscheme(
+  cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
+  dbschemePath: string,
+): Promise<QlPacksForLanguage> {
+  const qlpacks = await cliServer.resolveQlpacks(getOnDiskWorkspaceFolders());
+  const packs: QlPackWithPath[] = Object.entries(qlpacks).map(
+    ([packName, dirs]) => {
+      if (dirs.length < 1) {
+        void extLogger.log(
+          `In getQlPackFor ${dbschemePath}, qlpack ${packName} has no directories`,
+        );
+        return { packName, packDir: undefined };
+      }
+      if (dirs.length > 1) {
+        void extLogger.log(
+          `In getQlPackFor ${dbschemePath}, qlpack ${packName} has more than one directory; arbitrarily choosing the first`,
+        );
+      }
+      return {
+        packName,
+        packDir: dirs[0],
+      };
+    },
+  );
+  const dbschemePack = await findDbschemePack(packs, dbschemePath);
+  const queryPack = dbschemePack.isLibraryPack
+    ? findStandardQueryPack(qlpacks, dbschemePack.name)
+    : undefined;
+  return {
+    dbschemePack: dbschemePack.name,
+    dbschemePackIsLibraryPack: dbschemePack.isLibraryPack,
+    queryPack,
+  };
+}
+
+export async function getPrimaryDbscheme(
+  datasetFolder: string,
+): Promise<string> {
+  const dbschemes = await glob("*.dbscheme", {
+    cwd: datasetFolder,
+  });
+
+  if (dbschemes.length < 1) {
+    throw new Error(
+      `Can't find dbscheme for current database in ${datasetFolder}`,
+    );
+  }
+
+  dbschemes.sort();
+  const dbscheme = dbschemes[0];
+
+  if (dbschemes.length > 1) {
+    void window.showErrorMessage(
+      `Found multiple dbschemes in ${datasetFolder} during quick query; arbitrarily choosing the first, ${dbscheme}, to decide what library to use.`,
+    );
+  }
+  return dbscheme;
+}