Add source to database item

This adds a source property to the database item to store the source of
the database, for example GitHub or an internet URL. This will be used
to automatically check for updates to GitHub-downloaded databases in the
future.

Author: koesie10

extensions/ql-vscode/src/databases/config/db-config.ts

@@ -1,6 +1,8 @@
 // Contains models and consts for the data we want to store in the database config.
 // Changes to these models should be done carefully and account for backwards compatibility of data.
 
+import { DatabaseSource } from "../local-databases/database-source";
+
 export const DB_CONFIG_VERSION = 1;
 
 export interface DbConfig {
@@ -88,6 +90,7 @@ export interface LocalDatabase {
   name: string;
   dateAdded: number;
   language: string;
+  source: DatabaseSource;
   storagePath: string;
 }
 

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -33,6 +33,7 @@ import { allowHttp } from "../config";
 import { showAndLogInformationMessage } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
 import { getLanguageDisplayName } from "../common/query-language";
+import { DatabaseSource } from "./local-databases/database-source";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -62,6 +63,10 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     undefined,
+    {
+      type: "url",
+      url: databaseUrl,
+    },
     progress,
     cli,
   );
@@ -199,7 +204,7 @@ export async function downloadGitHubDatabase(
     return;
   }
 
-  const { databaseUrl, name, owner } = result;
+  const { databaseUrl, name, owner, commitOid } = result;
 
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
@@ -221,6 +226,11 @@ export async function downloadGitHubDatabase(
     databaseManager,
     storagePath,
     `${owner}/${name}`,
+    {
+      type: "github",
+      repository: nwo,
+      commitOid,
+    },
     progress,
     cli,
     makeSelected,
@@ -250,6 +260,10 @@ export async function importArchiveDatabase(
       databaseManager,
       storagePath,
       undefined,
+      {
+        type: "archive",
+        path: databaseUrl,
+      },
       progress,
       cli,
     );
@@ -282,6 +296,7 @@ export async function importArchiveDatabase(
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  * @param nameOverride a name for the database that overrides the default
+ * @param source the source of the database
  * @param progress callback to send progress messages to
  * @param makeSelected make the new database selected in the databases panel (default: true)
  * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -292,6 +307,7 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   nameOverride: string | undefined,
+  source: DatabaseSource,
   progress: ProgressCallback,
   cli?: CodeQLCliServer,
   makeSelected = true,
@@ -336,6 +352,7 @@ async function databaseArchiveFetcher(
 
     const item = await databaseManager.openDatabase(
       Uri.file(dbPath),
+      source,
       makeSelected,
       nameOverride,
       {
@@ -533,6 +550,7 @@ export async function convertGithubNwoToDatabaseUrl(
       databaseUrl: string;
       owner: string;
       name: string;
+      commitOid: string | null;
     }
   | undefined
 > {
@@ -553,10 +571,18 @@ export async function convertGithubNwoToDatabaseUrl(
       }
     }
 
+    const databaseForLanguage = response.data.find(
+      (db: any) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
     return {
       databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
       owner,
       name: repo,
+      commitOid: databaseForLanguage.commit_oid,
     };
   } catch (e) {
     void extLogger.log(`Error: ${getErrorMessage(e)}`);

extensions/ql-vscode/src/databases/db-item.ts

@@ -1,5 +1,7 @@
 // This file contains models that are used to represent the databases.
 
+import { DatabaseSource } from "./local-databases/database-source";
+
 export enum DbItemKind {
   RootLocal = "RootLocal",
   LocalList = "LocalList",
@@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem {
   databaseName: string;
   dateAdded: number;
   language: string;
+  source: DatabaseSource;
   storagePath: string;
   parentListName?: string;
 }

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -197,6 +197,7 @@ function createLocalDb(
     databaseName: db.name,
     dateAdded: db.dateAdded,
     language: db.language,
+    source: db.source,
     storagePath: db.storagePath,
     selected: !!selected,
     parentListName: listName,

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -367,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
 
               await this.databaseManager.openDatabase(
                 uri,
+                {
+                  type: "folder",
+                },
                 makeSelected,
                 nameOverride,
                 {
@@ -704,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
               this.queryServer?.cliServer,
             );
           } else {
-            await this.databaseManager.openDatabase(uri);
+            await this.databaseManager.openDatabase(uri, {
+              type: "folder",
+            });
           }
         } catch (e) {
           // rethrow and let this be handled by default error handling.
@@ -819,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
         if (byFolder) {
           const fixedUri = await this.fixDbUri(uri);
           // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri);
+          return await this.databaseManager.openDatabase(fixedUri, {
+            type: "folder",
+          });
         } else {
           // we are selecting a database archive. Must unzip into a workspace-controlled area
           // before importing.

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
 import { stat } from "fs-extra";
 import { containsPath, pathsEqual } from "../../common/files";
 import { DatabaseContents } from "./database-contents";
+import { DatabaseSource } from "./database-source";
 
 export class DatabaseItemImpl implements DatabaseItem {
   // These are only public in the implementation, they are readonly in the interface
@@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return this.options.dateAdded;
   }
 
+  public get source(): DatabaseSource | undefined {
+    return this.options.source;
+  }
+
   public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -2,6 +2,7 @@ import vscode from "vscode";
 import * as cli from "../../codeql-cli/cli";
 import { DatabaseContents } from "./database-contents";
 import { DatabaseOptions } from "./database-options";
+import { DatabaseSource } from "./database-source";
 
 /** An item in the list of available databases */
 export interface DatabaseItem {
@@ -25,6 +26,11 @@ export interface DatabaseItem {
    */
   readonly dateAdded: number | undefined;
 
+  /**
+   * The source this database item was added from or undefined if unknown.
+   */
+  readonly source: DatabaseSource | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -34,6 +34,7 @@ import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 import { DatabaseResolver } from "./database-resolver";
 import { telemetryListener } from "../../common/vscode/telemetry";
 import { LanguageContextStore } from "../../language-context-store";
+import { DatabaseSource } from "./database-source";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -131,14 +132,19 @@ export class DatabaseManager extends DisposableObject {
    */
   public async openDatabase(
     uri: vscode.Uri,
+    source: DatabaseSource | undefined,
     makeSelected = true,
     displayName?: string,
     {
       isTutorialDatabase = false,
       addSourceArchiveFolder = true,
     }: OpenDatabaseOptions = {},
   ): Promise<DatabaseItem> {
-    const databaseItem = await this.createDatabaseItem(uri, displayName);
+    const databaseItem = await this.createDatabaseItem(
+      uri,
+      source,
+      displayName,
+    );
 
     return await this.addExistingDatabaseItem(
       databaseItem,
@@ -189,6 +195,7 @@ export class DatabaseManager extends DisposableObject {
    */
   private async createDatabaseItem(
     uri: vscode.Uri,
+    source: DatabaseSource | undefined,
     displayName: string | undefined,
   ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
@@ -197,6 +204,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
+      source,
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
@@ -212,6 +220,7 @@ export class DatabaseManager extends DisposableObject {
    */
   public async createOrOpenDatabaseItem(
     uri: vscode.Uri,
+    source: DatabaseSource | undefined,
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(uri);
     if (existingItem !== undefined) {
@@ -220,7 +229,7 @@ export class DatabaseManager extends DisposableObject {
     }
 
     // We don't add this to the list automatically, but the user can add it later.
-    return this.createDatabaseItem(uri, undefined);
+    return this.createDatabaseItem(uri, source, undefined);
   }
 
   public async createSkeletonPacks(databaseItem: DatabaseItem) {
@@ -355,6 +364,7 @@ export class DatabaseManager extends DisposableObject {
     let displayName: string | undefined = undefined;
     let dateAdded = undefined;
     let language = undefined;
+    let source = undefined;
     if (state.options) {
       if (typeof state.options.displayName === "string") {
         displayName = state.options.displayName;
@@ -363,6 +373,7 @@ export class DatabaseManager extends DisposableObject {
         dateAdded = state.options.dateAdded;
       }
       language = state.options.language;
+      source = state.options.source;
     }
 
     const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -375,6 +386,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded,
       language,
+      source,
     };
     const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -1,10 +1,14 @@
+import { DatabaseSource } from "./database-source";
+
 export interface DatabaseOptions {
   displayName?: string;
   dateAdded?: number | undefined;
   language?: string;
+  source?: DatabaseSource;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   dateAdded: number | undefined;
   language: string | undefined;
+  source: DatabaseSource | undefined;
 }

extensions/ql-vscode/src/databases/local-databases/database-source.ts

@@ -0,0 +1,30 @@
+interface DatabaseSourceFolder {
+  type: "folder";
+}
+
+interface DatabaseSourceArchive {
+  type: "archive";
+  path: string;
+}
+
+interface DatabaseSourceGitHub {
+  type: "github";
+  repository: string;
+  commitOid: string | null;
+}
+
+interface DatabaseSourceInternet {
+  type: "url";
+  url: string;
+}
+
+interface DatabaseSourceDebugger {
+  type: "debugger";
+}
+
+export type DatabaseSource =
+  | DatabaseSourceFolder
+  | DatabaseSourceArchive
+  | DatabaseSourceGitHub
+  | DatabaseSourceInternet
+  | DatabaseSourceDebugger;