Merge branch 'master' into jcreed/jump-to-def-release

Author: jcreedcmu

extensions/ql-vscode/CHANGELOG.md

@@ -5,7 +5,7 @@
 - Enable 'Go to Definition' and 'Go to References' on source archive
   files in CodeQL databases. This is handled by a CodeQL query.
 
-## 1.1.5
+## 1.1.5 - 15 May 2020
 
 - Links in results are no longer underlined and monospaced.
 - Add the ability to choose a database either from an archive, a folder, or from the internet.

extensions/ql-vscode/README.md

@@ -66,12 +66,11 @@ While you can use the [CodeQL CLI to create your own databases](https://help.sem
 1. Find a project you're interested in and display the **Integrations** tab (for example, [Apache Kafka](https://lgtm.com/projects/g/apache/kafka/ci/)).
 1. Scroll to the **CodeQL databases for local analysis** section at the bottom of the page.
 1. Download databases for the languages that you want to explore.
-1. Unzip the databases.
 1. For each database that you want to import:
-    1. In the VS Code sidebar, go to **CodeQL** > **Databases** and click **+**.
-    1. Browse to the unzipped database folder (the parent folder that contains `db-<language>` and `src`) and select **Choose database** to add it.
+    1. Go to the CodeQL Databases view in the sidebar. Hover over the Databases title bar and click the icon to **Choose Database from Archive**.
+    1. Browse to the zipped database that you downloaded from LGTM.
 
-When the import is complete, each CodeQL database is displayed in the CodeQL sidebar under **Databases**.
+Once you've added a CodeQL database, it is displayed in the Databases view.
 
 ### Running a query
 

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.1.5",
+  "version": "1.1.6",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -27,11 +27,13 @@
     "onView:codeQLQueryHistory",
     "onView:test-explorer",
     "onCommand:codeQL.checkForUpdatesToCLI",
+    "onCommand:codeQLDatabases.chooseDatabaseFolder",
+    "onCommand:codeQLDatabases.chooseDatabaseArchive",
+    "onCommand:codeQLDatabases.chooseDatabaseInternet",
+    "onCommand:codeQL.setCurrentDatabase",
     "onCommand:codeQL.chooseDatabaseFolder",
     "onCommand:codeQL.chooseDatabaseArchive",
     "onCommand:codeQL.chooseDatabaseInternet",
-    "onCommand:codeQL.setCurrentDatabase",
-    "onCommand:codeQL.downloadDatabase",
     "onCommand:codeQLDatabases.chooseDatabase",
     "onCommand:codeQLDatabases.setCurrentDatabase",
     "onCommand:codeQL.quickQuery",
@@ -175,24 +177,24 @@
         "title": "CodeQL: Quick Query"
       },
       {
-        "command": "codeQL.chooseDatabaseFolder",
+        "command": "codeQLDatabases.chooseDatabaseFolder",
         "title": "Choose Database from Folder",
         "icon": {
           "light": "media/light/folder-opened-plus.svg",
           "dark": "media/dark/folder-opened-plus.svg"
         }
       },
       {
-        "command": "codeQL.chooseDatabaseArchive",
+        "command": "codeQLDatabases.chooseDatabaseArchive",
         "title": "Choose Database from Archive",
         "icon": {
           "light": "media/light/archive-plus.svg",
           "dark": "media/dark/archive-plus.svg"
         }
       },
       {
-        "command": "codeQL.chooseDatabaseInternet",
-        "title": "Download database",
+        "command": "codeQLDatabases.chooseDatabaseInternet",
+        "title": "Download Database",
         "icon": {
           "light": "media/light/cloud-download.svg",
           "dark": "media/dark/cloud-download.svg"
@@ -231,8 +233,16 @@
         "title": "Show Database Directory"
       },
       {
-        "command": "codeQL.downloadDatabase",
-        "title": "CodeQL: Download database"
+        "command": "codeQL.chooseDatabaseFolder",
+        "title": "CodeQL: Choose Database from Folder"
+      },
+      {
+        "command": "codeQL.chooseDatabaseArchive",
+        "title": "CodeQL: Choose Database from Archive"
+      },
+      {
+        "command": "codeQL.chooseDatabaseInternet",
+        "title": "CodeQL: Download Database"
       },
       {
         "command": "codeQLDatabases.sortByName",
@@ -312,17 +322,17 @@
           "group": "navigation"
         },
         {
-          "command": "codeQL.chooseDatabaseFolder",
+          "command": "codeQLDatabases.chooseDatabaseFolder",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         },
         {
-          "command": "codeQL.chooseDatabaseArchive",
+          "command": "codeQLDatabases.chooseDatabaseArchive",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         },
         {
-          "command": "codeQL.chooseDatabaseInternet",
+          "command": "codeQLDatabases.chooseDatabaseInternet",
           "when": "view == codeQLDatabases",
           "group": "navigation"
         }
@@ -406,10 +416,6 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
-        {
-          "command": "codeQL.downloadDatabase",
-          "when": "true"
-        },
         {
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
@@ -442,6 +448,22 @@
           "command": "codeQLDatabases.removeDatabase",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.chooseDatabaseFolder",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseArchive",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.chooseDatabaseInternet",
+          "when": "false"
+        },
+        {
+          "command": "codeQLDatabases.upgradeDatabase",
+          "when": "false"
+        },
         {
           "command": "codeQLQueryHistory.openQuery",
           "when": "false"

extensions/ql-vscode/src/databaseFetcher.ts

@@ -4,23 +4,23 @@ import { Uri, ProgressOptions, ProgressLocation, commands, window } from "vscode
 import * as fs from "fs-extra";
 import * as path from "path";
 import { DatabaseManager, DatabaseItem } from "./databases";
-import { ProgressCallback, showAndLogErrorMessage, withProgress } from "./helpers";
+import { ProgressCallback, showAndLogErrorMessage, withProgress, showAndLogInformationMessage } from "./helpers";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
  *
  * @param databasesManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  */
-export default async function promptFetchDatabase(databasesManager: DatabaseManager, storagePath: string): Promise<DatabaseItem | undefined>  {
+export async function promptImportInternetDatabase(databasesManager: DatabaseManager, storagePath: string): Promise<DatabaseItem | undefined> {
   let item: DatabaseItem | undefined = undefined;
 
   try {
     const databaseUrl = await window.showInputBox({
       prompt: 'Enter URL of zipfile of database to download'
     });
     if (databaseUrl) {
-      validateUrl(databaseUrl);
+      validateHttpsUrl(databaseUrl);
 
       const progressOptions: ProgressOptions = {
         location: ProgressLocation.Notification,
@@ -30,13 +30,41 @@ export default async function promptFetchDatabase(databasesManager: DatabaseMana
       await withProgress(progressOptions, async progress => (item = await databaseArchiveFetcher(databaseUrl, databasesManager, storagePath, progress)));
       commands.executeCommand('codeQLDatabases.focus');
     }
+    showAndLogInformationMessage('Database downloaded and imported successfully.');
   } catch (e) {
     showAndLogErrorMessage(e.message);
   }
 
   return item;
 }
 
+
+/**
+ * Imports a database from a local archive.
+ *
+ * @param databaseUrl the file url of the archive to import
+ * @param databasesManager the DatabaseManager
+ * @param storagePath where to store the unzipped database.
+ */
+export async function importArchiveDatabase(databaseUrl: string, databasesManager: DatabaseManager, storagePath: string): Promise<DatabaseItem | undefined> {
+  let item: DatabaseItem | undefined = undefined;
+  try {
+    const progressOptions: ProgressOptions = {
+      location: ProgressLocation.Notification,
+      title: 'Importing database from archive',
+      cancellable: false,
+    };
+    await withProgress(progressOptions, async progress => (item = await databaseArchiveFetcher(databaseUrl, databasesManager, storagePath, progress)));
+    commands.executeCommand('codeQLDatabases.focus');
+
+    showAndLogInformationMessage('Database unzipped and imported successfully.');
+  } catch (e) {
+    showAndLogErrorMessage(e.message);
+  }
+  return item;
+}
+
+
 /**
  * Fetches an archive database. The database might be on the internet
  * or in the local filesystem.
@@ -46,15 +74,15 @@ export default async function promptFetchDatabase(databasesManager: DatabaseMana
  * @param storagePath where to store the unzipped database.
  * @param progressCallback optional callback to send progress messages to
  */
-export async function databaseArchiveFetcher(
+async function databaseArchiveFetcher(
   databaseUrl: string,
   databasesManager: DatabaseManager,
   storagePath: string,
   progressCallback?: ProgressCallback
 ): Promise<DatabaseItem> {
   progressCallback?.({
     maxStep: 3,
-    message: 'Downloading database',
+    message: 'Getting database',
     step: 1
   });
   if (!storagePath) {
@@ -75,9 +103,10 @@ export async function databaseArchiveFetcher(
     step: 3
   });
 
+  // find the path to the database. The actual database might be in a sub-folder
   const dbPath = await findDirWithFile(unzipPath, '.dbinfo', 'codeql-database.yml');
   if (dbPath) {
-    const item = await databasesManager.openDatabase(Uri.parse(dbPath));
+    const item = await databasesManager.openDatabase(Uri.parse(`file:${dbPath}`));
     databasesManager.setCurrentDatabaseItem(item);
     return item;
   } else {
@@ -86,14 +115,21 @@ export async function databaseArchiveFetcher(
 }
 
 async function getStorageFolder(storagePath: string, urlStr: string) {
+  // we need to generate a folder name for the unzipped archive,
+  // this needs to be human readable since we may use this name as the initial
+  // name for the database
   const url = Uri.parse(urlStr);
-  let lastName = path.basename(url.path).substring(0, 255);
+  // MacOS has a max filename length of 255
+  // and remove a few extra chars in case we need to add a counter at the end.
+  let lastName = path.basename(url.path).substring(0, 250);
   if (lastName.endsWith(".zip")) {
     lastName = lastName.substring(0, lastName.length - 4);
   }
 
   const realpath = await fs.realpath(storagePath);
   let folderName = path.join(realpath, lastName);
+
+  // avoid overwriting existing folders
   let counter = 0;
   while (await fs.pathExists(folderName)) {
     counter++;
@@ -106,7 +142,7 @@ async function getStorageFolder(storagePath: string, urlStr: string) {
 }
 
 
-function validateUrl(databaseUrl: string) {
+function validateHttpsUrl(databaseUrl: string) {
   let uri;
   try {
     uri = Uri.parse(databaseUrl, true);
@@ -126,7 +162,7 @@ async function readAndUnzip(databaseUrl: string, unzipPath: string) {
 
   await new Promise((resolve, reject) => {
     // we already know this is a file scheme
-    const databaseFile = Uri.parse(databaseUrl).path;
+    const databaseFile = Uri.parse(databaseUrl).fsPath;
     const stream = fs.createReadStream(databaseFile);
     stream.on('error', reject);
     unzipStream.on('error', reject);

extensions/ql-vscode/src/databases-ui.ts

@@ -8,7 +8,7 @@ import { logger } from './logging';
 import { clearCacheInDatabase, UserCancellationException } from './run-queries';
 import * as qsClient from './queryserver-client';
 import { upgradeDatabase } from './upgrades';
-import promptFetchDatabase, { databaseArchiveFetcher } from './databaseFetcher';
+import { importArchiveDatabase, promptImportInternetDatabase } from './databaseFetcher';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -174,9 +174,9 @@ export class DatabaseUI extends DisposableObject {
     this.treeDataProvider = this.push(new DatabaseTreeDataProvider(ctx, databaseManager));
     this.push(window.createTreeView('codeQLDatabases', { treeDataProvider: this.treeDataProvider }));
 
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseFolder', this.handleChooseDatabaseFolder));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseArchive', this.handleChooseDatabaseArchive));
-    ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseInternet', this.handleChooseDatabaseInternet));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseFolder', this.handleChooseDatabaseFolder));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseArchive', this.handleChooseDatabaseArchive));
+    ctx.subscriptions.push(commands.registerCommand('codeQLDatabases.chooseDatabaseInternet', this.handleChooseDatabaseInternet));
     ctx.subscriptions.push(commands.registerCommand('codeQL.setCurrentDatabase', this.handleSetCurrentDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQL.upgradeCurrentDatabase', this.handleUpgradeCurrentDatabase));
     ctx.subscriptions.push(commands.registerCommand('codeQL.clearCache', this.handleClearCache));
@@ -193,7 +193,7 @@ export class DatabaseUI extends DisposableObject {
     await this.databaseManager.setCurrentDatabaseItem(databaseItem);
   }
 
-  private handleChooseDatabaseFolder = async (): Promise<DatabaseItem | undefined> => {
+  handleChooseDatabaseFolder = async (): Promise<DatabaseItem | undefined> => {
     try {
       return await this.chooseAndSetDatabase(true);
     } catch (e) {
@@ -202,7 +202,7 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
-  private handleChooseDatabaseArchive = async (): Promise<DatabaseItem | undefined> => {
+  handleChooseDatabaseArchive = async (): Promise<DatabaseItem | undefined> => {
     try {
       return await this.chooseAndSetDatabase(false);
     } catch (e) {
@@ -211,8 +211,8 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
-  private handleChooseDatabaseInternet = async (): Promise<DatabaseItem | undefined> => {
-    return await promptFetchDatabase(this.databaseManager, this.storagePath);
+  handleChooseDatabaseInternet = async (): Promise<DatabaseItem | undefined> => {
+    return await promptImportInternetDatabase(this.databaseManager, this.storagePath);
   }
 
   private handleSortByName = async () => {
@@ -292,7 +292,7 @@ export class DatabaseUI extends DisposableObject {
   private handleSetCurrentDatabase = async (uri: Uri): Promise<DatabaseItem | undefined> => {
     // Assume user has selected an archive if the file has a .zip extension
     if (uri.path.endsWith('.zip')) {
-      return await databaseArchiveFetcher(uri.toString(), this.databaseManager, this.storagePath);
+      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
     }
 
     return await this.setCurrentDatabase(uri);
@@ -366,7 +366,7 @@ export class DatabaseUI extends DisposableObject {
     else {
       // we are selecting a database archive. Must unzip into a workspace-controlled area
       // before importing.
-      return await databaseArchiveFetcher(uri.toString(), this.databaseManager, this.storagePath);
+      return await importArchiveDatabase(uri.toString(true), this.databaseManager, this.storagePath);
     }
   }
 }

extensions/ql-vscode/src/extension.ts

@@ -20,7 +20,6 @@ import { displayQuickQuery } from './quick-query';
 import { compileAndRunQueryAgainstDatabase, tmpDirDisposal, UserCancellationException } from './run-queries';
 import { QLTestAdapterFactory } from './test-adapter';
 import { TestUIService } from './test-ui';
-import promptFetchDatabase from './databaseFetcher';
 
 /**
  * extension.ts
@@ -335,7 +334,9 @@ async function activateWithInstalledDistribution(ctx: ExtensionContext, distribu
     await qs.restartQueryServer();
     helpers.showAndLogInformationMessage('CodeQL Query Server restarted.', { outputLogger: queryServerLogger });
   }));
-  ctx.subscriptions.push(commands.registerCommand('codeQL.downloadDatabase', () => promptFetchDatabase(dbm, getContextStoragePath(ctx))));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseFolder', () => databaseUI.handleChooseDatabaseFolder()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseArchive', () => databaseUI.handleChooseDatabaseArchive()));
+  ctx.subscriptions.push(commands.registerCommand('codeQL.chooseDatabaseInternet', () => databaseUI.handleChooseDatabaseInternet()));
 
   ctx.subscriptions.push(client.start());