Merge pull request #276 from jcreedcmu/jcreed/prep-jump-to-def

Add bqrs types and ability to run template queries.

Author: jcreedcmu

extensions/ql-vscode/src/bqrs-cli-types.ts

@@ -0,0 +1,79 @@
+
+export const PAGE_SIZE = 1000;
+
+export type ColumnKind = "f" | "i" | "s" | "b" | "d" | "e";
+
+export interface Column {
+  name?: string,
+  kind: ColumnKind,
+}
+
+
+export interface ResultSetSchema {
+  name: string,
+  rows: number,
+  columns: Column[],
+  pagination?: PaginationInfo,
+}
+
+export function getResultSetSchema(resultSetName: string, resultSets: BQRSInfo): ResultSetSchema | undefined {
+  for (const schema of resultSets["result-sets"]) {
+    if (schema.name === resultSetName) {
+      return schema;
+    }
+  }
+  return undefined;
+}
+export interface PaginationInfo {
+  "step-size": number,
+  offsets: number[],
+}
+
+export interface BQRSInfo {
+  "result-sets": ResultSetSchema[]
+}
+
+export interface EntityValue {
+  url?: UrlValue,
+  label?: string
+}
+
+export interface LineColumnLocation {
+  uri: string
+  startLine: number,
+  startColumn: number,
+  endLine: number,
+  endColumn: number,
+  charOffset: never
+  charLength: never
+}
+
+export interface OffsetLengthLocation {
+  uri: string,
+  startLine: never,
+  startColumn: never,
+  endLine: never,
+  endColumn: never,
+  charOffset: number,
+  charLength: number,
+}
+
+export interface WholeFileLocation {
+  uri: string,
+  startLine: never,
+  startColumn: never,
+  endLine: never,
+  endColumn: never,
+  charOffset: never,
+  charLength: never,
+}
+
+export type UrlValue = LineColumnLocation | OffsetLengthLocation | WholeFileLocation | string;
+
+
+export type ColumnValue = EntityValue | number | string | boolean;
+
+export interface DecodedBqrsChunk {
+  tuples: ColumnValue[][],
+  next?: number
+}

extensions/ql-vscode/src/cli.ts

@@ -1,17 +1,18 @@
-import * as child_process from 'child_process';
 import * as cpp from 'child-process-promise';
+import * as child_process from 'child_process';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
+import { Readable } from 'stream';
+import { StringDecoder } from 'string_decoder';
 import * as tk from 'tree-kill';
 import * as util from 'util';
-import { SortDirection, QueryMetadata } from './interface-types';
-import { Logger, ProgressReporter } from './logging';
-import { Disposable, CancellationToken } from 'vscode';
+import { CancellationToken, Disposable } from 'vscode';
+import { BQRSInfo, DecodedBqrsChunk } from "./bqrs-cli-types";
 import { DistributionProvider } from './distribution';
 import { assertNever } from './helpers-pure';
-import { Readable } from 'stream';
-import { StringDecoder } from 'string_decoder';
+import { QueryMetadata, SortDirection } from './interface-types';
+import { Logger, ProgressReporter } from './logging';
 
 /**
  * The version of the SARIF format that we are using.
@@ -461,6 +462,7 @@ export class CodeQLCliServer implements Disposable {
    * Gets the RAM setting for the query server.
    * @param queryMemoryMb The maximum amount of RAM to use, in MB.
    * Leave `undefined` for CodeQL to choose a limit based on the available system memory.
+   * @param progressReporter The progress reporter to send progress information to.
    * @returns String arguments that can be passed to the CodeQL query server,
    * indicating how to split the given RAM limit between heap and off-heap memory.
    */
@@ -471,6 +473,38 @@ export class CodeQLCliServer implements Disposable {
     }
     return await this.runJsonCodeQlCliCommand<string[]>(['resolve', 'ram'], args, "Resolving RAM settings", progressReporter);
   }
+  /**
+   * Gets the headers (and optionally pagination info) of a bqrs.
+   * @param bqrsPath The path to the bqrs.
+   * @param pageSize The page size to precompute offsets into the binary file for.
+   */
+  async bqrsInfo(bqrsPath: string, pageSize?: number): Promise<BQRSInfo> {
+    const subcommandArgs = (
+      pageSize ? ["--paginate-rows", pageSize.toString()] : []
+    ).concat(
+      bqrsPath
+    );
+    return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, "Reading bqrs header");
+  }
+
+  /**
+  * Gets the results from a bqrs.
+  * @param bqrsPath The path to the bqrs.
+  * @param resultSet The result set to get.
+  * @param pageSize How many results to get.
+  * @param offset The 0-based index of the first result to get.
+  */
+  async bqrsDecode(bqrsPath: string, resultSet: string, pageSize?: number, offset?: number): Promise<DecodedBqrsChunk> {
+    const subcommandArgs = [
+      "--entities=url,string",
+      "--result-set", resultSet,
+    ].concat(
+      pageSize ? ["--rows", pageSize.toString()] : []
+    ).concat(
+      offset ? ["--start-at", offset.toString()] : []
+    ).concat([bqrsPath]);
+    return await this.runJsonCodeQlCliCommand<DecodedBqrsChunk>(['bqrs', 'decode'], subcommandArgs, "Reading bqrs data");
+  }
 
 
   async interpretBqrs(metadata: { kind: string, id: string }, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {

extensions/ql-vscode/src/messages.ts

@@ -635,6 +635,8 @@ export interface EvaluateQueriesParams {
   useSequenceHint: boolean;
 }
 
+export type TemplateDefinitions = { [key: string]: TemplateSource; }
+
 /**
  * A single query that should be run
  */
@@ -658,7 +660,7 @@ export interface QueryToRun {
   /**
    * Values to set for each template
    */
-  templateValues?: { [key: string]: TemplateSource; };
+  templateValues?: TemplateDefinitions;
   /**
    * Whether templates without values in the templateValues
    * map should be set to the empty set or give an error.

extensions/ql-vscode/src/run-queries.ts

@@ -56,6 +56,7 @@ export class QueryInfo {
     public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
     public readonly quickEvalPosition?: messages.Position,
     public readonly metadata?: QueryMetadata,
+    public readonly templates?: messages.TemplateDefinitions,
   ) {
     this.queryID = QueryInfo.nextQueryId++;
     this.compiledQueryPath = path.join(tmpDir.name, `compiledQuery${this.queryID}.qlo`);
@@ -80,6 +81,7 @@ export class QueryInfo {
       resultsPath: this.resultsPaths.resultsPath,
       qlo: vscode.Uri.file(this.compiledQueryPath).toString(),
       allowUnknownTemplates: true,
+      templateValues: this.templates,
       id: callbackId,
       timeoutSecs: qs.config.timeoutSecs,
     }
@@ -362,7 +364,8 @@ export async function compileAndRunQueryAgainstDatabase(
   qs: qsClient.QueryServerClient,
   db: DatabaseItem,
   quickEval: boolean,
-  selectedQueryUri: vscode.Uri | undefined
+  selectedQueryUri: vscode.Uri | undefined,
+  templates?: messages.TemplateDefinitions,
 ): Promise<QueryWithResults> {
 
   if (!db.contents || !db.contents.dbSchemeUri) {
@@ -415,7 +418,7 @@ export async function compileAndRunQueryAgainstDatabase(
     logger.log(`Couldn't resolve metadata for ${qlProgram.queryPath}: ${e}`);
   }
 
-  const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata);
+  const query = new QueryInfo(qlProgram, db, packConfig.dbscheme, quickEvalPosition, metadata, templates);
   await checkDbschemeCompatibility(cliServer, qs, query);
 
   let errors;