Merge pull request #3071 from github/koesie10/download-github-database

Prompt user for database download on startup

Author: koesie10

extensions/ql-vscode/package.json

@@ -421,8 +421,33 @@
       },
       {
         "type": "object",
-        "title": "Log insights",
+        "title": "GitHub Databases",
         "order": 8,
+        "properties": {
+          "codeQL.githubDatabase.enable": {
+            "type": "boolean",
+            "default": false,
+            "markdownDescription": "Enable automatic detection of GitHub databases."
+          },
+          "codeQL.githubDatabase.download": {
+            "type": "string",
+            "default": "ask",
+            "enum": [
+              "ask",
+              "never"
+            ],
+            "enumDescriptions": [
+              "Ask to download a GitHub database when a workspace is opened.",
+              "Never download a GitHub databases when a workspace is opened."
+            ],
+            "description": "Ask to download a GitHub database when a workspace is opened."
+          }
+        }
+      },
+      {
+        "type": "object",
+        "title": "Log insights",
+        "order": 9,
         "properties": {
           "codeQL.logInsights.joinOrderWarningThreshold": {
             "type": "number",
@@ -436,7 +461,7 @@
       {
         "type": "object",
         "title": "Telemetry",
-        "order": 9,
+        "order": 10,
         "properties": {
           "codeQL.telemetry.enableTelemetry": {
             "type": "boolean",

extensions/ql-vscode/src/config.ts

@@ -1,19 +1,19 @@
 import { DisposableObject } from "./common/disposable-object";
 import {
-  workspace,
-  Event,
-  EventEmitter,
   ConfigurationChangeEvent,
-  ConfigurationTarget,
   ConfigurationScope,
+  ConfigurationTarget,
+  Event,
+  EventEmitter,
+  workspace,
 } from "vscode";
 import { DistributionManager } from "./codeql-cli/distribution";
 import { extLogger } from "./common/logging/vscode";
 import { ONE_DAY_IN_MS } from "./common/time";
 import {
+  defaultFilterSortState,
   FilterKey,
   SortKey,
-  defaultFilterSortState,
 } from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
@@ -775,3 +775,52 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return !!ENABLE_RUBY.getValue<boolean>();
   }
 }
+
+const GITHUB_DATABASE_SETTING = new Setting("githubDatabase", ROOT_SETTING);
+
+// Feature flag for the GitHub database downnload.
+const GITHUB_DATABASE_ENABLE = new Setting("enable", GITHUB_DATABASE_SETTING);
+const GITHUB_DATABASE_DOWNLOAD = new Setting(
+  "download",
+  GITHUB_DATABASE_SETTING,
+);
+
+const GitHubDatabaseDownloadValues = ["ask", "never"] as const;
+type GitHubDatabaseDownload = (typeof GitHubDatabaseDownloadValues)[number];
+
+export interface GitHubDatabaseConfig {
+  enable: boolean;
+  download: GitHubDatabaseDownload;
+  setDownload(
+    value: GitHubDatabaseDownload,
+    target?: ConfigurationTarget,
+  ): Promise<void>;
+}
+
+export class GitHubDatabaseConfigListener
+  extends ConfigListener
+  implements GitHubDatabaseConfig
+{
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings(
+      [GITHUB_DATABASE_SETTING],
+      e,
+    );
+  }
+
+  public get enable() {
+    return !!GITHUB_DATABASE_ENABLE.getValue<boolean>();
+  }
+
+  public get download(): GitHubDatabaseDownload {
+    const value = GITHUB_DATABASE_DOWNLOAD.getValue<GitHubDatabaseDownload>();
+    return GitHubDatabaseDownloadValues.includes(value) ? value : "ask";
+  }
+
+  public async setDownload(
+    value: GitHubDatabaseDownload,
+    target: ConfigurationTarget = ConfigurationTarget.Workspace,
+  ): Promise<void> {
+    await GITHUB_DATABASE_DOWNLOAD.updateValue(value, target);
+  }
+}

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -207,6 +207,38 @@ export async function downloadGitHubDatabase(
   const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
     result;
 
+  return downloadGitHubDatabaseFromUrl(
+    databaseUrl,
+    databaseId,
+    databaseCreatedAt,
+    commitOid,
+    owner,
+    name,
+    octokit,
+    progress,
+    databaseManager,
+    storagePath,
+    cli,
+    makeSelected,
+    addSourceArchiveFolder,
+  );
+}
+
+export async function downloadGitHubDatabaseFromUrl(
+  databaseUrl: string,
+  databaseId: number,
+  databaseCreatedAt: string,
+  commitOid: string | null,
+  owner: string,
+  name: string,
+  octokit: Octokit.Octokit,
+  progress: ProgressCallback,
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  cli?: CodeQLCliServer,
+  makeSelected = true,
+  addSourceArchiveFolder = true,
+): Promise<DatabaseItem | undefined> {
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
    * The object is undocumented, but looks something like this:
@@ -229,7 +261,7 @@ export async function downloadGitHubDatabase(
     `${owner}/${name}`,
     {
       type: "github",
-      repository: nwo,
+      repository: `${owner}/${name}`,
       databaseId,
       databaseCreatedAt,
       commitOid,
@@ -577,7 +609,7 @@ export async function convertGithubNwoToDatabaseUrl(
     }
 
     const databaseForLanguage = response.data.find(
-      (db: any) => db.language === language,
+      (db) => db.language === language,
     );
     if (!databaseForLanguage) {
       throw new Error(`No database found for language '${language}'`);
@@ -599,9 +631,9 @@ export async function convertGithubNwoToDatabaseUrl(
 
 export async function promptForLanguage(
   languages: string[],
-  progress: ProgressCallback,
+  progress: ProgressCallback | undefined,
 ): Promise<string | undefined> {
-  progress({
+  progress?.({
     message: "Choose language",
     step: 2,
     maxStep: 2,

extensions/ql-vscode/src/databases/github-database-module.ts

@@ -2,22 +2,58 @@ import { DisposableObject } from "../common/disposable-object";
 import { App } from "../common/app";
 import { findGitHubRepositoryForWorkspace } from "./github-repository-finder";
 import { redactableError } from "../common/errors";
-import { asError } from "../common/helpers-pure";
+import { asError, getErrorMessage } from "../common/helpers-pure";
+import {
+  CodeqlDatabase,
+  findGitHubDatabasesForRepository,
+  promptAndDownloadGitHubDatabase,
+} from "./github-database-prompt";
+import {
+  GitHubDatabaseConfig,
+  GitHubDatabaseConfigListener,
+  isCanary,
+} from "../config";
+import { AppOctokit } from "../common/octokit";
+import { DatabaseManager } from "./local-databases";
+import { CodeQLCliServer } from "../codeql-cli/cli";
 
 export class GithubDatabaseModule extends DisposableObject {
-  private constructor(private readonly app: App) {
+  private readonly config: GitHubDatabaseConfig;
+
+  private constructor(
+    private readonly app: App,
+    private readonly databaseManager: DatabaseManager,
+    private readonly databaseStoragePath: string,
+    private readonly cliServer: CodeQLCliServer,
+  ) {
     super();
+
+    this.config = this.push(new GitHubDatabaseConfigListener());
   }
 
-  public static async initialize(app: App): Promise<GithubDatabaseModule> {
-    const githubDatabaseModule = new GithubDatabaseModule(app);
+  public static async initialize(
+    app: App,
+    databaseManager: DatabaseManager,
+    databaseStoragePath: string,
+    cliServer: CodeQLCliServer,
+  ): Promise<GithubDatabaseModule> {
+    const githubDatabaseModule = new GithubDatabaseModule(
+      app,
+      databaseManager,
+      databaseStoragePath,
+      cliServer,
+    );
     app.subscriptions.push(githubDatabaseModule);
 
     await githubDatabaseModule.initialize();
     return githubDatabaseModule;
   }
 
   private async initialize(): Promise<void> {
+    if (!this.config.enable) {
+      return;
+    }
+
     // Start the check and downloading the database asynchronously. We don't want to block on this
     // in extension activation since this makes network requests and waits for user input.
     void this.promptGitHubRepositoryDownload().catch((e: unknown) => {
@@ -31,6 +67,10 @@ export class GithubDatabaseModule extends DisposableObject {
   }
 
   private async promptGitHubRepositoryDownload(): Promise<void> {
+    if (this.config.download === "never") {
+      return;
+    }
+
     const githubRepositoryResult = await findGitHubRepositoryForWorkspace();
     if (githubRepositoryResult.isFailure) {
       void this.app.logger.log(
@@ -42,8 +82,58 @@ export class GithubDatabaseModule extends DisposableObject {
     }
 
     const githubRepository = githubRepositoryResult.value;
-    void this.app.logger.log(
-      `Found GitHub repository for workspace: '${githubRepository.owner}/${githubRepository.name}'`,
+
+    const hasExistingDatabase = this.databaseManager.databaseItems.some(
+      (db) =>
+        db.origin?.type === "github" &&
+        db.origin.repository ===
+          `${githubRepository.owner}/${githubRepository.name}`,
+    );
+    if (hasExistingDatabase) {
+      return;
+    }
+
+    const credentials = isCanary() ? this.app.credentials : undefined;
+
+    const octokit = credentials
+      ? await credentials.getOctokit()
+      : new AppOctokit();
+
+    let databases: CodeqlDatabase[];
+    try {
+      databases = await findGitHubDatabasesForRepository(
+        octokit,
+        githubRepository.owner,
+        githubRepository.name,
+      );
+    } catch (e) {
+      this.app.telemetry?.sendError(
+        redactableError(
+          asError(e),
+        )`Failed to prompt for GitHub database download`,
+      );
+
+      void this.app.logger.log(
+        `Failed to find GitHub databases for repository: ${getErrorMessage(e)}`,
+      );
+
+      return;
+    }
+
+    if (databases.length === 0) {
+      return;
+    }
+
+    await promptAndDownloadGitHubDatabase(
+      octokit,
+      githubRepository.owner,
+      githubRepository.name,
+      databases,
+      this.config,
+      this.databaseManager,
+      this.databaseStoragePath,
+      this.cliServer,
+      this.app.commands,
     );
   }
 }