Merge pull request #350 from jcreedcmu/jcreed/cli-bqrs-parsing

Experimental: Enable parsing bqrs with the cli instead of in the webview

Author: jcreedcmu

extensions/ql-vscode/src/adapt.ts

@@ -0,0 +1,103 @@
+import { DecodedBqrsChunk, ResultSetSchema, ColumnKind, Column, ColumnValue } from "./bqrs-cli-types";
+import { LocationValue, ResultSetSchema as AdaptedSchema, ColumnSchema, ColumnType, LocationStyle } from 'semmle-bqrs';
+
+// FIXME: This is a temporary bit of impedance matching to convert
+// from the types provided by ./bqrs-cli-types, to the types used by
+// the view layer.
+//
+// The reason that it is benign for now is that it is only used by
+// feature-flag-guarded codepaths that won't be encountered by normal
+// users. It is not yet guaranteed to produce correct output for raw
+// results.
+//
+// Eventually, the view layer should be refactored to directly accept data
+// of types coming from bqrs-cli-types, and this file can be deleted.
+
+export type ResultRow = ResultValue[];
+
+export interface ResultElement {
+  label: string;
+  location?: LocationValue;
+}
+
+export interface ResultUri {
+  uri: string;
+}
+
+export type ResultValue = ResultElement | ResultUri | string;
+
+export interface RawResultSet {
+  readonly schema: AdaptedSchema;
+  readonly rows: readonly ResultRow[];
+}
+
+function adaptKind(kind: ColumnKind): ColumnType {
+  // XXX what about 'u'?
+  if (kind === 'e') {
+    return { type: 'e', primitiveType: 's', locationStyle: LocationStyle.FivePart, hasLabel: true }
+  }
+  else {
+    return { type: kind };
+  }
+}
+
+function adaptColumn(col: Column): ColumnSchema {
+  return { name: col.name!, type: adaptKind(col.kind) };
+}
+
+export function adaptSchema(schema: ResultSetSchema): AdaptedSchema {
+  return {
+    columns: schema.columns.map(adaptColumn),
+    name: schema.name,
+    tupleCount: schema.rows,
+    version: 0,
+  }
+}
+
+export function adaptValue(val: ColumnValue): ResultValue {
+  // XXX taking a lot of incorrect shortcuts here
+
+  if (typeof val === 'string') {
+    return val;
+  }
+
+  if (typeof val === 'number' || typeof val === 'boolean') {
+    return val + '';
+  }
+
+  const url = val.url;
+
+  if (typeof url === 'string') {
+    return url;
+  }
+
+  if (url === undefined) {
+    return 'none';
+  }
+
+  return {
+    label: val.label || '',
+    location: {
+      t: LocationStyle.FivePart,
+      lineStart: url.startLine,
+      lineEnd: url.endLine,
+      colStart: url.startColumn,
+      colEnd: url.endColumn,
+      // FIXME: This seems definitely wrong. Should we be using
+      // something like the code in sarif-utils.ts?
+      file: url.uri.replace(/file:/, ''),
+    }
+  }
+
+}
+
+export function adaptRow(row: ColumnValue[]): ResultRow {
+  return row.map(adaptValue);
+}
+
+export function adaptBqrs(schema: AdaptedSchema, page: DecodedBqrsChunk): RawResultSet {
+  return {
+    schema,
+    rows: page.tuples.map(adaptRow),
+  }
+}

extensions/ql-vscode/src/config.ts

@@ -50,6 +50,9 @@ const ROOT_SETTING = new Setting('codeQL');
  */
 export const EXPERIMENTAL_FEATURES_SETTING = new Setting('experimentalFeatures', ROOT_SETTING);
 
+/* Advanced setting: used to enable bqrs parsing in the cli instead of in the webview. */
+export const EXPERIMENTAL_BQRS_SETTING = new Setting('experimentalBqrsParsing', ROOT_SETTING);
+
 // Distribution configuration
 
 const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);

extensions/ql-vscode/src/interface-types.ts

@@ -1,5 +1,6 @@
 import * as sarif from 'sarif';
 import { ResolvableLocationValue } from 'semmle-bqrs';
+import { RawResultSet } from './adapt';
 
 /**
  * Only ever show this many results per run in interpreted results.
@@ -77,6 +78,12 @@ export interface SetStateMsg {
    * This is useful to prevent properties like scroll state being lost when rendering the sorted results after sorting a column.
    */
   shouldKeepOldResultsWhileRendering: boolean;
+
+  /**
+   * An experimental way of providing results from the extension.
+   * Should be undefined unless config.EXPERIMENTAL_BQRS_SETTING is set to true.
+   */
+  resultSets?: RawResultSet[];
 }
 
 /** Advance to the next or previous path no in the path viewer */

extensions/ql-vscode/src/interface.ts

@@ -16,6 +16,8 @@ import * as messages from './messages';
 import { CompletedQuery, interpretResults } from './query-results';
 import { QueryInfo, tmpDir } from './run-queries';
 import { parseSarifLocation, parseSarifPlainTextMessage } from './sarif-utils';
+import { adaptSchema, adaptBqrs, RawResultSet } from './adapt';
+import { EXPERIMENTAL_BQRS_SETTING } from './config';
 
 /**
  * interface.ts
@@ -349,9 +351,7 @@ export class InterfaceManager extends DisposableObject {
       const showButton = "View Results";
       const queryName = results.queryName;
       const resultPromise = vscode.window.showInformationMessage(
-        `Finished running query ${
-          queryName.length > 0 ? ` “${queryName}”` : ""
-        }.`,
+        `Finished running query ${queryName.length > 0 ? ` "${queryName}"` : ""}.`,
         showButton
       );
       // Address this click asynchronously so we still update the
@@ -363,13 +363,30 @@ export class InterfaceManager extends DisposableObject {
       });
     }
 
+    let resultSets: RawResultSet[] | undefined;
+
+    if (EXPERIMENTAL_BQRS_SETTING.getValue()) {
+      resultSets = [];
+      // Setting pageSize very large for now for the sake of
+      // performance testing of vscode on containers.
+      const pageSize = 1e20;
+      const schemas = await this.cliServer.bqrsInfo(results.query.resultsPaths.resultsPath, pageSize);
+      for (const schema of schemas["result-sets"]) {
+        const chunk = await this.cliServer.bqrsDecode(results.query.resultsPaths.resultsPath, schema.name, pageSize, 0)
+        const adaptedSchema = adaptSchema(schema);
+        const resultSet = adaptBqrs(adaptedSchema, chunk);
+        resultSets.push(resultSet);
+      }
+    }
+
     await this.postMessage({
       t: "setState",
       interpretation,
       origResultsPaths: results.query.resultsPaths,
       resultsPath: this.convertPathToWebviewUri(
         results.query.resultsPaths.resultsPath
       ),
+      resultSets,
       sortedResultsMap,
       database: results.database,
       shouldKeepOldResultsWhileRendering,

extensions/ql-vscode/src/view/raw-results-table.tsx

@@ -1,6 +1,7 @@
 import * as React from "react";
 import { renderLocation, ResultTableProps, zebraStripe, className, nextSortDirection } from "./result-table-utils";
-import { RawTableResultSet, ResultValue, vscode } from "./results";
+import { RawTableResultSet, vscode } from "./results";
+import { ResultValue } from "../adapt";
 import { SortDirection, RAW_RESULTS_LIMIT, RawResultsSortState } from "../interface-types";
 
 export type RawTableProps = ResultTableProps & {

extensions/ql-vscode/src/view/results.tsx

@@ -1,11 +1,12 @@
 import * as React from 'react';
 import * as Rdom from 'react-dom';
 import * as bqrs from 'semmle-bqrs';
-import { ElementBase, LocationValue, PrimitiveColumnValue, PrimitiveTypeKind, ResultSetSchema, tryGetResolvableLocation } from 'semmle-bqrs';
+import { ElementBase, PrimitiveColumnValue, PrimitiveTypeKind, ResultSetSchema, tryGetResolvableLocation } from 'semmle-bqrs';
 import { assertNever } from '../helpers-pure';
 import { DatabaseInfo, FromResultsViewMsg, Interpretation, IntoResultsViewMsg, SortedResultSetInfo, RawResultsSortState, NavigatePathMsg, QueryMetadata, ResultsPaths } from '../interface-types';
 import { EventHandlers as EventHandlerList } from './event-handler-list';
 import { ResultTables } from './result-tables';
+import { RawResultSet, ResultValue, ResultRow } from '../adapt';
 
 /**
  * results.tsx
@@ -23,31 +24,13 @@ interface VsCodeApi {
 declare const acquireVsCodeApi: () => VsCodeApi;
 export const vscode = acquireVsCodeApi();
 
-export interface ResultElement {
-  label: string;
-  location?: LocationValue;
-}
-
-export interface ResultUri {
-  uri: string;
-}
-
-export type ResultValue = ResultElement | ResultUri | string;
-
-export type ResultRow = ResultValue[];
-
 export type RawTableResultSet = { t: 'RawResultSet' } & RawResultSet;
 export type PathTableResultSet = { t: 'SarifResultSet'; readonly schema: ResultSetSchema; name: string } & Interpretation;
 
 export type ResultSet =
   | RawTableResultSet
   | PathTableResultSet;
 
-export interface RawResultSet {
-  readonly schema: ResultSetSchema;
-  readonly rows: readonly ResultRow[];
-}
-
 async function* getChunkIterator(response: Response): AsyncIterableIterator<Uint8Array> {
   if (!response.ok) {
     throw new Error(`Failed to load results: (${response.status}) ${response.statusText}`);
@@ -62,9 +45,7 @@ async function* getChunkIterator(response: Response): AsyncIterableIterator<Uint
   }
 }
 
-function translatePrimitiveValue(value: PrimitiveColumnValue, type: PrimitiveTypeKind):
-ResultValue {
-
+function translatePrimitiveValue(value: PrimitiveColumnValue, type: PrimitiveTypeKind): ResultValue {
   switch (type) {
     case 'i':
     case 'f':
@@ -127,6 +108,7 @@ async function parseResultSets(response: Response): Promise<readonly ResultSet[]
 
 interface ResultsInfo {
   resultsPath: string;
+  resultSets: ResultSet[] | undefined;
   origResultsPaths: ResultsPaths;
   database: DatabaseInfo;
   interpretation: Interpretation | undefined;
@@ -187,6 +169,7 @@ class App extends React.Component<{}, ResultsViewState> {
       case 'setState':
         this.updateStateWithNewResultsInfo({
           resultsPath: msg.resultsPath,
+          resultSets: msg.resultSets?.map(x => ({ t: 'RawResultSet', ...x })),
           origResultsPaths: msg.origResultsPaths,
           sortedResultsMap: new Map(Object.entries(msg.sortedResultsMap)),
           database: msg.database,
@@ -247,8 +230,9 @@ class App extends React.Component<{}, ResultsViewState> {
     let results: Results | null = null;
     let statusText = '';
     try {
+      const resultSets = resultsInfo.resultSets || await this.getResultSets(resultsInfo);
       results = {
-        resultSets: await this.getResultSets(resultsInfo),
+        resultSets,
         database: resultsInfo.database,
         sortStates: this.getSortStates(resultsInfo)
       };