Support overlay database creation

Author: cklin

src/codeql.ts

@@ -24,6 +24,11 @@ import {
 import { isAnalyzingDefaultBranch } from "./git-utils";
 import { Language } from "./languages";
 import { Logger } from "./logging";
+import {
+  OverlayDatabaseMode,
+  writeBaseDatabaseOidsFile,
+  writeOverlayChangedFilesFile,
+} from "./overlay-database-utils";
 import * as setupCodeql from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
 import { ToolsDownloadStatusReport } from "./tools-download";
@@ -82,6 +87,7 @@ export interface CodeQL {
     sourceRoot: string,
     processName: string | undefined,
     qlconfigFile: string | undefined,
+    overlayDatabaseMode: OverlayDatabaseMode,
     logger: Logger,
   ): Promise<void>;
   /**
@@ -552,6 +558,7 @@ export async function getCodeQLForCmd(
       sourceRoot: string,
       processName: string | undefined,
       qlconfigFile: string | undefined,
+      overlayDatabaseMode: OverlayDatabaseMode,
       logger: Logger,
     ) {
       const extraArgs = config.languages.map(
@@ -606,12 +613,21 @@ export async function getCodeQLForCmd(
         ? "--force-overwrite"
         : "--overwrite";
 
+      if (overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
+        await writeOverlayChangedFilesFile(config, sourceRoot, logger);
+        extraArgs.push("--overlay");
+      } else if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
+        extraArgs.push("--overlay-base");
+      }
+
       await runCli(
         cmd,
         [
           "database",
           "init",
-          overwriteFlag,
+          ...(overlayDatabaseMode === OverlayDatabaseMode.Overlay
+            ? []
+            : [overwriteFlag]),
           "--db-cluster",
           config.dbLocation,
           `--source-root=${sourceRoot}`,
@@ -625,6 +641,10 @@ export async function getCodeQLForCmd(
         ],
         { stdin: externalRepositoryToken },
       );
+
+      if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
+        await writeBaseDatabaseOidsFile(config, sourceRoot);
+      }
     },
     async runAutobuild(config: Config, language: Language) {
       applyAutobuildAzurePipelinesTimeoutFix();

src/git-utils.ts

@@ -300,6 +300,61 @@ export const decodeGitFilePath = function (filePath: string): string {
   return filePath;
 };
 
+/**
+ * Get the oids of all files in HEAD.
+ *
+ * @param checkoutPath A path into the Git repository.
+ * @returns a map from file paths to the corresponding oid.
+ * @throws {Error} if "git ls-tree" produces unexpected output.
+ */
+export const getAllFileOids = async function (
+  checkoutPath: string,
+): Promise<{ [key: string]: string }> {
+  const stdout = await runGitCommand(
+    checkoutPath,
+    ["ls-tree", "--format=%(objectname)_%(path)", "-r", "HEAD"],
+    "Cannot list file OIDs in HEAD.",
+  );
+
+  const fileOidMap: { [key: string]: string } = {};
+  const regex = /^([0-9a-f]{40})_(.+)$/;
+  for (const line of stdout.split("\n")) {
+    if (line) {
+      const match = line.match(regex);
+      if (match) {
+        const oid = match[1];
+        const path = decodeGitFilePath(match[2]);
+        fileOidMap[path] = oid;
+      } else {
+        throw new Error(`Unexpected "git ls-tree" output: ${line}`);
+      }
+    }
+  }
+  return fileOidMap;
+};
+
+/**
+ * Get the root of the Git repository.
+ *
+ * @param sourceRoot The source root of the code being analyzed.
+ * @returns The root of the Git repository.
+ */
+export const getGitRoot = async function (
+  sourceRoot: string,
+): Promise<string | undefined> {
+  try {
+    const stdout = await runGitCommand(
+      sourceRoot,
+      ["rev-parse", "--show-toplevel"],
+      "Cannot find Git repository root from the source root ${sourceRoot}.",
+    );
+    return stdout.trim();
+  } catch {
+    // Errors are already logged by runGitCommand()
+    return undefined;
+  }
+};
+
 function getRefFromEnv(): string {
   // To workaround a limitation of Actions dynamic workflows not setting
   // the GITHUB_REF in some cases, we accept also the ref within the

src/init-action.ts

@@ -36,12 +36,14 @@ import { Feature, featureConfig, Features } from "./feature-flags";
 import {
   checkInstallPython311,
   cleanupDatabaseClusterDirectory,
+  getOverlayDatabaseMode,
   initCodeQL,
   initConfig,
   runInit,
 } from "./init";
 import { Language } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
+import { OverlayDatabaseMode } from "./overlay-database-utils";
 import { parseRepositoryNwo } from "./repository";
 import { ToolsSource } from "./setup-codeql";
 import {
@@ -395,7 +397,21 @@ async function run() {
   }
 
   try {
-    cleanupDatabaseClusterDirectory(config, logger);
+    const sourceRoot = path.resolve(
+      getRequiredEnvParam("GITHUB_WORKSPACE"),
+      getOptionalInput("source-root") || "",
+    );
+
+    const overlayDatabaseMode = await getOverlayDatabaseMode(
+      (await codeql.getVersion()).version,
+      config,
+      sourceRoot,
+      logger,
+    );
+
+    if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
+      cleanupDatabaseClusterDirectory(config, logger);
+    }
 
     if (zstdAvailability) {
       await recordZstdAvailability(config, zstdAvailability);
@@ -675,18 +691,14 @@ async function run() {
       }
     }
 
-    const sourceRoot = path.resolve(
-      getRequiredEnvParam("GITHUB_WORKSPACE"),
-      getOptionalInput("source-root") || "",
-    );
-
     const tracerConfig = await runInit(
       codeql,
       config,
       sourceRoot,
       "Runner.Worker.exe",
       getOptionalInput("registries"),
       apiDetails,
+      overlayDatabaseMode,
       logger,
     );
     if (tracerConfig !== undefined) {

src/init.ts

@@ -3,14 +3,20 @@ import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
 import * as io from "@actions/io";
+import * as semver from "semver";
 
 import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
 import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
 import { CodeQL, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
+import { getGitRoot } from "./git-utils";
 import { Language, isScannedLanguage } from "./languages";
 import { Logger } from "./logging";
+import {
+  CODEQL_OVERLAY_MINIMUM_VERSION,
+  OverlayDatabaseMode,
+} from "./overlay-database-utils";
 import { ToolsSource } from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
 import { ToolsDownloadStatusReport } from "./tools-download";
@@ -79,13 +85,55 @@ export async function initConfig(
   return config;
 }
 
+export async function getOverlayDatabaseMode(
+  codeqlVersion: string,
+  config: configUtils.Config,
+  sourceRoot: string,
+  logger: Logger,
+): Promise<OverlayDatabaseMode> {
+  const overlayDatabaseMode = process.env.CODEQL_OVERLAY_DATABASE_MODE;
+
+  if (
+    overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
+    overlayDatabaseMode === OverlayDatabaseMode.OverlayBase
+  ) {
+    if (config.buildMode !== util.BuildMode.None) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `build-mode is set to "${config.buildMode}" instead of "none". ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    if (semver.lt(codeqlVersion, CODEQL_OVERLAY_MINIMUM_VERSION)) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `the CodeQL CLI is older than ${CODEQL_OVERLAY_MINIMUM_VERSION}. ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    if ((await getGitRoot(sourceRoot)) === undefined) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `the source root "${sourceRoot}" is not inside a git repository. ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    return overlayDatabaseMode as OverlayDatabaseMode;
+  }
+  return OverlayDatabaseMode.None;
+}
+
 export async function runInit(
   codeql: CodeQL,
   config: configUtils.Config,
   sourceRoot: string,
   processName: string | undefined,
   registriesInput: string | undefined,
   apiDetails: GitHubApiCombinedDetails,
+  overlayDatabaseMode: OverlayDatabaseMode,
   logger: Logger,
 ): Promise<TracerConfig | undefined> {
   fs.mkdirSync(config.dbLocation, { recursive: true });
@@ -109,6 +157,7 @@ export async function runInit(
         sourceRoot,
         processName,
         qlconfigFile,
+        overlayDatabaseMode,
         logger,
       ),
   );

src/overlay-database-utils.ts

@@ -0,0 +1,89 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import { getTemporaryDirectory } from "./actions-util";
+import { type Config } from "./config-utils";
+import { getAllFileOids } from "./git-utils";
+import { Logger } from "./logging";
+
+export enum OverlayDatabaseMode {
+  Overlay = "overlay",
+  OverlayBase = "overlay-base",
+  None = "none",
+}
+
+export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";
+
+export async function writeBaseDatabaseOidsFile(
+  config: Config,
+  sourceRoot: string,
+): Promise<void> {
+  const gitFileOids = await getAllFileOids(sourceRoot);
+  const gitFileOidsJson = JSON.stringify(gitFileOids);
+  const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
+  await fs.promises.writeFile(baseDatabaseOidsFilePath, gitFileOidsJson);
+}
+
+async function readBaseDatabaseOidsFile(
+  config: Config,
+  logger: Logger,
+): Promise<{ [key: string]: string }> {
+  const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
+  try {
+    const contents = await fs.promises.readFile(
+      baseDatabaseOidsFilePath,
+      "utf-8",
+    );
+    return JSON.parse(contents) as { [key: string]: string };
+  } catch (e) {
+    logger.error(
+      "Failed to read overlay-base file OIDs from " +
+        `${baseDatabaseOidsFilePath}: ${(e as any).message || e}`,
+    );
+    throw e;
+  }
+}
+
+export async function writeOverlayChangedFilesFile(
+  config: Config,
+  sourceRoot: string,
+  logger: Logger,
+): Promise<string> {
+  const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
+  const overlayFileOids = await getAllFileOids(sourceRoot);
+  const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
+  const changedFilesJson = JSON.stringify(changedFiles);
+
+  const overlayChangedFilesFilePath = path.join(
+    getTemporaryDirectory(),
+    "overlay-changed-files.json",
+  );
+  logger.debug(
+    "Writing overlay changed files to " +
+      `${overlayChangedFilesFilePath}: ${changedFilesJson}`,
+  );
+  await fs.promises.writeFile(overlayChangedFilesFilePath, changedFilesJson);
+  return overlayChangedFilesFilePath;
+}
+
+function getBaseDatabaseOidsFilePath(config: Config): string {
+  return path.join(config.dbLocation, "base-database-oids.json");
+}
+
+function computeChangedFiles(
+  baseFileOids: { [key: string]: string },
+  overlayFileOids: { [key: string]: string },
+): string[] {
+  const changes: string[] = [];
+  for (const [file, oid] of Object.entries(overlayFileOids)) {
+    if (!(file in baseFileOids) || baseFileOids[file] !== oid) {
+      changes.push(file);
+    }
+  }
+  for (const file of Object.keys(baseFileOids)) {
+    if (!(file in overlayFileOids)) {
+      changes.push(file);
+    }
+  }
+  return changes;
+}