Ask user if they want to re-import outdated testproj dbs

Before running a query now, do the following:

1. Check if the selected database is imported from a testproj
2. If so, check the last modified time of the `codeql-datase.yml` file
   of the imported database with that of its origin.
3. If the origin database has a file that is newer, assume that the
   database has been recreated since the last time it was imported.
4. If newer, then ask the user if they want to re-import before running
   the query.

Also, this change appends the `(test)` label to all test databases in
the database list.

Author: aeisenberg

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -146,7 +146,8 @@ class DatabaseTreeDataProvider
       item.iconPath = new ThemeIcon("error", new ThemeColor("errorForeground"));
     }
     item.tooltip = element.databaseUri.fsPath;
-    item.description = element.language;
+    item.description =
+      element.language + (element.origin?.type === "testproj" ? " (test)" : "");
     return item;
   }
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -18,7 +18,10 @@ import {
 import { join } from "path";
 import type { FullDatabaseOptions } from "./database-options";
 import { DatabaseItemImpl } from "./database-item-impl";
-import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
+import {
+  showBinaryChoiceDialog,
+  showNeverAskAgainDialog,
+} from "../../common/vscode/dialog";
 import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
@@ -32,7 +35,7 @@ import { QlPackGenerator } from "../../local-queries/qlpack-generator";
 import { asError, getErrorMessage } from "../../common/helpers-pure";
 import type { DatabaseItem, PersistedDatabaseItem } from "./database-item";
 import { redactableError } from "../../common/errors";
-import { remove } from "fs-extra";
+import { copy, remove, stat } from "fs-extra";
 import { containsPath } from "../../common/files";
 import type { DatabaseChangedEvent } from "./database-events";
 import { DatabaseEventKind } from "./database-events";
@@ -116,6 +119,7 @@ export class DatabaseManager extends DisposableObject {
     super();
 
     qs.onStart(this.reregisterDatabases.bind(this));
+    qs.onQueryRunStarting(this.maybeReimportTestDatabase.bind(this));
 
     this.push(
       this.languageContext.onLanguageContextChanged(async () => {
@@ -170,12 +174,82 @@ export class DatabaseManager extends DisposableObject {
     const originPath = uri.fsPath;
     for (const item of this._databaseItems) {
       if (item.origin?.type === "testproj" && item.origin.path === originPath) {
-        return item
+        return item;
       }
     }
     return undefined;
   }
 
+  public async maybeReimportTestDatabase(
+    databaseUri: vscode.Uri,
+    forceImport = false,
+  ): Promise<void> {
+    const res = await this.isTestDatabaseOutdated(databaseUri);
+    if (!res) {
+      return;
+    }
+    const doit =
+      forceImport ||
+      (await showBinaryChoiceDialog(
+        "This test database is outdated. Do you want to reimport it?",
+      ));
+
+    if (doit) {
+      await this.reimportTestDatabase(databaseUri);
+    }
+  }
+
+  /**
+   * Checks if the origin of the imported database is newer.
+   * The imported database must be a test database.
+   * @param databaseUri the URI of the imported database to check
+   * @returns true if both databases exist and the origin database is newer.
+   */
+  private async isTestDatabaseOutdated(
+    databaseUri: vscode.Uri,
+  ): Promise<boolean> {
+    const dbItem = this.findDatabaseItem(databaseUri);
+    if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
+      return false;
+    }
+
+    // Compare timestmps of the codeql-database.yml files of the original and the
+    // imported databases.
+    const originDbYml = join(dbItem.origin.path, "codeql-database.yml");
+    const importedDbYml = join(
+      dbItem.databaseUri.fsPath,
+      "codeql-database.yml",
+    );
+
+    // TODO add error handling if one does not exist.
+    const originStat = await stat(originDbYml);
+    const importedStat = await stat(importedDbYml);
+    return originStat.mtimeMs > importedStat.mtimeMs;
+  }
+
+  /**
+   * Reimport the specified imported database from its origin.
+   * The imported databsae must be a testproj database.
+   *
+   * @param databaseUri the URI of the imported database to reimport
+   */
+  private async reimportTestDatabase(databaseUri: vscode.Uri): Promise<void> {
+    const dbItem = this.findDatabaseItem(databaseUri);
+    if (dbItem === undefined || dbItem.origin?.type !== "testproj") {
+      throw new Error(`Database ${databaseUri} is not a testproj.`);
+    }
+
+    await this.removeDatabaseItem(dbItem);
+    await copy(dbItem.origin.path, databaseUri.fsPath);
+    const newDbItem = new DatabaseItemImpl(databaseUri, dbItem.contents, {
+      dateAdded: Date.now(),
+      language: dbItem.language,
+      origin: dbItem.origin,
+    });
+    await this.addDatabaseItem(newDbItem);
+    await this.setCurrentDatabaseItem(newDbItem);
+  }
+
   /**
    * Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
    * the list.

extensions/ql-vscode/src/query-server/query-runner.ts

@@ -1,4 +1,4 @@
-import { window } from "vscode";
+import { window, Uri } from "vscode";
 import type { CancellationToken, MessageItem } from "vscode";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
 import type { ProgressCallback } from "../common/vscode/progress";
@@ -63,9 +63,22 @@ export interface CoreQueryRun {
 export type CoreCompletedQuery = CoreQueryResults &
   Omit<CoreQueryRun, "evaluate">;
 
+type OnQueryRunStargingListener = (dbPath: Uri) => Promise<void>;
 export class QueryRunner {
   constructor(public readonly qs: QueryServerClient) {}
 
+  // Event handlers that get notified whenever a query is about to start running.
+  // Can't use vscode EventEmitters since they are not asynchronous.
+  private readonly onQueryRunStartingListeners: OnQueryRunStargingListener[] =
+    [];
+  public onQueryRunStarting(listener: OnQueryRunStargingListener) {
+    this.onQueryRunStartingListeners.push(listener);
+  }
+
+  private async fireQueryRunStarting(dbPath: Uri) {
+    await Promise.all(this.onQueryRunStartingListeners.map((l) => l(dbPath)));
+  }
+
   get cliServer(): CodeQLCliServer {
     return this.qs.cliServer;
   }
@@ -138,6 +151,8 @@ export class QueryRunner {
     templates: Record<string, string> | undefined,
     logger: BaseLogger,
   ): Promise<CoreQueryResults> {
+    await this.fireQueryRunStarting(Uri.file(dbPath));
+
     return await compileAndRunQueryAgainstDatabaseCore(
       this.qs,
       dbPath,

extensions/ql-vscode/test/vscode-tests/cli-integration/databases/database-fetcher.test.ts

@@ -16,7 +16,8 @@ import {
   testprojLoc,
 } from "../../global.helper";
 import { createMockCommandManager } from "../../../__mocks__/commandsMock";
-import { remove } from "fs-extra";
+import { utimesSync } from "fs";
+import { remove, existsSync } from "fs-extra";
 
 /**
  * Run various integration tests for databases
@@ -80,7 +81,26 @@ describe("database-fetcher", () => {
       expect(dbItem).toBeDefined();
       dbItem = dbItem!;
       expect(dbItem.name).toBe("db");
-      expect(dbItem.databaseUri.fsPath).toBe(join(storagePath, "db", "db"));
+      expect(dbItem.databaseUri.fsPath).toBe(join(storagePath, "db"));
+
+      // Now that we have fetched it. Check for re-importing
+      // Delete a file in the imported database and we can check if the file is recreated
+      const srczip = join(dbItem.databaseUri.fsPath, "src.zip");
+      await remove(srczip);
+
+      // Attempt 1: re-import database should be a no-op since timestamp of imported database is newer
+      await databaseManager.maybeReimportTestDatabase(dbItem.databaseUri);
+      expect(existsSync(srczip)).toBeFalsy();
+
+      // Attempt 3: re-import database should re-import the database after updating modified time
+      utimesSync(
+        join(testprojLoc, "codeql-database.yml"),
+        new Date(),
+        new Date(),
+      );
+
+      await databaseManager.maybeReimportTestDatabase(dbItem.databaseUri, true);
+      expect(existsSync(srczip)).toBeTruthy();
     });
   });
 

extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts

@@ -6,15 +6,16 @@ import {
   beforeEachAction,
 } from "../jest.activated-extension.setup";
 import { createWriteStream, existsSync, mkdirpSync } from "fs-extra";
-import { dirname } from "path";
+import { dirname, join } from "path";
 import { DB_URL, dbLoc, testprojLoc } from "../global.helper";
 import fetch from "node-fetch";
 import { createReadStream, renameSync } from "fs";
 import { Extract } from "unzipper";
 
 beforeAll(async () => {
   // ensure the test database is downloaded
-  mkdirpSync(dirname(dbLoc));
+  const dbParentDir = dirname(dbLoc);
+  mkdirpSync(dbParentDir);
   if (!existsSync(dbLoc)) {
     console.log(`Downloading test database to ${dbLoc}`);
 
@@ -30,18 +31,23 @@ beforeAll(async () => {
         });
       });
     });
+  }
+
+  // unzip the database from dbLoc to testprojLoc
+  if (!existsSync(testprojLoc)) {
+    console.log(`Unzipping test database to ${testprojLoc}`);
 
-    // unzip the database from dbLoc to testprojLoc
-    if (!existsSync(testprojLoc)) {
-      console.log(`Unzipping test database to ${testprojLoc}`);
-      const dbDir = dirname(testprojLoc);
-      mkdirpSync(dbDir);
-      console.log(`Unzipping test database to ${testprojLoc}`);
+    await new Promise((resolve, reject) => {
       createReadStream(dbLoc)
-        .pipe(Extract({ path: dirname(dbDir) }))
-        .on("close", () => console.log("Unzip completed."));
-    }
-    renameSync(dbLoc, testprojLoc);
+        .pipe(Extract({ path: dbParentDir }))
+        .on("close", () => {
+          console.log("Unzip completed.");
+          resolve(undefined);
+        })
+        .on("error", (e) => reject(e));
+    });
+
+    renameSync(join(dbParentDir, "db"), testprojLoc);
   }
 
   await beforeAllAction();

extensions/ql-vscode/test/vscode-tests/global.helper.ts

@@ -24,7 +24,7 @@ export const dbLoc = join(
 
 export const testprojLoc = join(
   realpathSync(join(__dirname, "../../../")),
-  "build/tests/db.zip",
+  "build/tests/db.testproj",
 );
 
 // eslint-disable-next-line import/no-mutable-exports