Implement viewing of query results as a CSV

Author: edoardopirovano

extensions/ql-vscode/CHANGELOG.md

@@ -8,6 +8,7 @@
 - Better error messages when BQRS interpretation fails to produce SARIF. [#770](https://github.com/github/vscode-codeql/pull/770)
 - Implement sorting of the query history view by name, date, and results count. [#777](https://github.com/github/vscode-codeql/pull/777)
 - Add a configuration option to pass additional arguments to the CLI when running tests. [#785](https://github.com/github/vscode-codeql/pull/785)
+- Introduce option to view query results as CSV. [#784](https://github.com/github/vscode-codeql/pull/784)
 
 ## 1.4.3 - 22 February 2021
 

extensions/ql-vscode/package.json

@@ -396,8 +396,12 @@
         "title": "Show Query Text"
       },
       {
-        "command": "codeQLQueryHistory.viewSarif",
-        "title": "View SARIF"
+        "command": "codeQLQueryHistory.viewCsvResults",
+        "title": "View Results (CSV)"
+      },
+      {
+        "command": "codeQLQueryHistory.viewSarifResults",
+        "title": "View Results (SARIF)"
       },
       {
         "command": "codeQLQueryHistory.viewDil",
@@ -569,7 +573,12 @@
           "when": "view == codeQLQueryHistory"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "group": "9_qlCommands",
+          "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifResults",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory && viewItem == interpretedResultsItem"
         },
@@ -696,7 +705,11 @@
           "when": "false"
         },
         {
-          "command": "codeQLQueryHistory.viewSarif",
+          "command": "codeQLQueryHistory.viewCsvResults",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueryHistory.viewSarifResults",
           "when": "false"
         },
         {

extensions/ql-vscode/src/cli.ts

@@ -25,6 +25,11 @@ import { CompilationMessage } from './pure/messages';
  */
 const SARIF_FORMAT = 'sarifv2.1.0';
 
+/**
+ * The string used to specify CSV format.
+ */
+const CSV_FORMAT = 'csv';
+
 /**
  * Flags to pass to all cli commands.
  */
@@ -582,18 +587,20 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, 'Reading bqrs data');
   }
 
-  async interpretBqrs(metadata: { kind: string; id: string; scored?: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+  async runInterpretCommand(format: string, metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo) {
     const args = [
       `-t=kind=${metadata.kind}`,
       `-t=id=${metadata.id}`,
       '--output', interpretedResultsPath,
-      '--format', SARIF_FORMAT,
+      '--format', format,
+    ];
+    if (format == SARIF_FORMAT) {
       // TODO: This flag means that we don't group interpreted results
       // by primary location. We may want to revisit whether we call
       // interpretation with and without this flag, or do some
       // grouping client-side.
-      '--no-group-results',
-    ];
+      args.push('--no-group-results');
+    }
     if (config.isCanary() && metadata.scored !== undefined) {
       args.push(`-t=scored=${metadata.scored}`);
     }
@@ -611,6 +618,10 @@ export class CodeQLCliServer implements Disposable {
 
     args.push(resultsPath);
     await this.runCodeQlCliCommand(['bqrs', 'interpret'], args, 'Interpreting query results');
+  }
+
+  async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
+    await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
 
     let output: string;
     try {
@@ -629,6 +640,9 @@ export class CodeQLCliServer implements Disposable {
     }
   }
 
+  async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
+    await this.runInterpretCommand(CSV_FORMAT, metadata, resultsPath, csvPath, sourceInfo);
+  }
 
   async sortBqrs(resultsPath: string, sortedResultsPath: string, resultSet: string, sortKeys: number[], sortDirections: SortDirection[]): Promise<void> {
     const sortDirectionStrings = sortDirections.map(direction => {

extensions/ql-vscode/src/query-history.ts

@@ -306,8 +306,14 @@ export class QueryHistoryManager extends DisposableObject {
     );
     this.push(
       commandRunner(
-        'codeQLQueryHistory.viewSarif',
-        this.handleViewSarif.bind(this)
+        'codeQLQueryHistory.viewCsvResults',
+        this.handleViewCsvResults.bind(this)
+      )
+    );
+    this.push(
+      commandRunner(
+        'codeQLQueryHistory.viewSarifResults',
+        this.handleViewSarifResults.bind(this)
       )
     );
     this.push(
@@ -544,7 +550,7 @@ export class QueryHistoryManager extends DisposableObject {
     await vscode.window.showTextDocument(doc, { preview: false });
   }
 
-  async handleViewSarif(
+  async handleViewSarifResults(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[]
   ) {
@@ -565,6 +571,19 @@ export class QueryHistoryManager extends DisposableObject {
     }
   }
 
+  async handleViewCsvResults(
+    singleItem: CompletedQuery,
+    multiSelect: CompletedQuery[]
+  ) {
+    if (!this.assertSingleQuery(multiSelect)) {
+      return;
+    }
+
+    await this.tryOpenExternalFile(
+      await singleItem.query.ensureCsvProduced(this.qs)
+    );
+  }
+
   async handleViewDil(
     singleItem: CompletedQuery,
     multiSelect: CompletedQuery[],
@@ -778,3 +797,4 @@ the file in the file explorer and dragging it into the workspace.`
     this.treeDataProvider.refresh(completedQuery);
   }
 }
+

extensions/ql-vscode/src/query-results.ts

@@ -179,6 +179,10 @@ export async function interpretResults(
   if (await fs.pathExists(interpretedResultsPath)) {
     return JSON.parse(await fs.readFile(interpretedResultsPath, 'utf8'));
   }
+  return await server.interpretBqrs(ensureMetadataIsComplete(metadata), resultsPath, interpretedResultsPath, sourceInfo);
+}
+
+export function ensureMetadataIsComplete(metadata: QueryMetadata | undefined) {
   if (metadata === undefined) {
     throw new Error('Can\'t interpret results without query metadata');
   }
@@ -191,5 +195,5 @@ export async function interpretResults(
     // SARIF format does, so in the absence of one, we use a dummy id.
     id = 'dummy-id';
   }
-  return await server.interpretBqrs({ kind, id, scored }, resultsPath, interpretedResultsPath, sourceInfo);
+  return { kind, id, scored };
 }

extensions/ql-vscode/src/run-queries.ts

@@ -24,6 +24,7 @@ import { QueryHistoryItemOptions } from './query-history';
 import * as qsClient from './queryserver-client';
 import { isQuickQueryPath } from './quick-query';
 import { compileDatabaseUpgradeSequence, hasNondestructiveUpgradeCapabilities, upgradeDatabaseExplicit } from './upgrades';
+import { ensureMetadataIsComplete } from './query-results';
 
 /**
  * run-queries.ts
@@ -53,6 +54,7 @@ export class QueryInfo {
 
   readonly compiledQueryPath: string;
   readonly dilPath: string;
+  readonly csvPath: string;
   readonly resultsPaths: ResultsPaths;
   readonly dataset: Uri; // guarantee the existence of a well-defined dataset dir at this point
   readonly queryID: number;
@@ -68,6 +70,7 @@ export class QueryInfo {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
     this.dilPath = path.join(tmpDir.name, `results${this.queryID}.dil`);
+    this.csvPath = path.join(tmpDir.name, `results${this.queryID}.csv`);
     this.resultsPaths = {
       resultsPath: path.join(tmpDir.name, `results${this.queryID}.bqrs`),
       interpretedResultsPath: path.join(tmpDir.name, `interpretedResults${this.queryID}.sarif`)
@@ -183,6 +186,13 @@ export class QueryInfo {
     return fs.pathExists(this.dilPath);
   }
 
+  /**
+   * Holds if this query already has CSV results produced
+   */
+  async hasCsv(): Promise<boolean> {
+    return fs.pathExists(this.csvPath);
+  }
+
   async ensureDilPath(qs: qsClient.QueryServerClient): Promise<string> {
     if (await this.hasDil()) {
       return this.dilPath;
@@ -198,8 +208,27 @@ export class QueryInfo {
     return this.dilPath;
   }
 
+  async ensureCsvProduced(qs: qsClient.QueryServerClient): Promise<string> {
+    if (await this.hasCsv()) {
+      return this.csvPath;
+    }
+
+    let sourceInfo;
+    if (this.dbItem.sourceArchive !== undefined) {
+      sourceInfo = {
+        sourceArchive: this.dbItem.sourceArchive.fsPath,
+        sourceLocationPrefix: await this.dbItem.getSourceLocationPrefix(
+          qs.cliServer
+        ),
+      };
+    }
+
+    await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
+    return this.csvPath;
+  }
 }
 
+
 export interface QueryWithResults {
   readonly query: QueryInfo;
   readonly result: messages.EvaluationResult;