Make QLTest use codeql test run instead of odasa qltest

Author: Dave Bartolomeo

extensions/ql-vscode/package.json

@@ -127,21 +127,13 @@
           "default": "[%t] %q on %d - %s",
           "description": "Default string for how to label query history items. %t is the time of the query, %q is the query name, %d is the database name, and %s is a status string."
         },
-        "codeQL.tests.semmleCoreDistributionPath": {
-          "scope": "window",
-          "type": "string",
-          "default": "",
-          "description": "Location of the Semmle Core distribution"
-        },
-        "codeQL.tests.semmleCoreLicensePath": {
-          "scope": "window",
-          "type": "string",
-          "description": "Location of the directory containing the Semmle Core license file"
-        },
-        "codeQL.tests.numberOfThreads": {
+        "codeQL.runningTests.numberOfThreads": {
           "scope": "window",
           "type": "integer",
-          "description": "Number of threads to use for CodeQL test execution"
+          "default": 1,
+          "minimum": 1,
+          "maximum": 1024,
+          "description": "Number of threads for running CodeQL tests."
         }
       }
     },

extensions/ql-vscode/src/cli.ts

@@ -1,13 +1,16 @@
-import * as child_process from "child_process";
+import * as child_process from 'child_process';
+import * as cpp from 'child-process-promise';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import * as sarif from 'sarif';
 import * as util from 'util';
-import { Logger, ProgressReporter } from "./logging";
-import { Disposable } from "vscode";
-import { DistributionProvider } from "./distribution";
-import { SortDirection } from "./interface-types";
-import { assertNever } from "./helpers-pure";
+import { Logger, ProgressReporter } from './logging';
+import { Disposable } from 'vscode';
+import { DistributionProvider } from './distribution';
+import { SortDirection } from './interface-types';
+import { assertNever } from './helpers-pure';
+import { Readable } from 'stream';
+import { StringDecoder } from 'string_decoder';
 
 /**
  * The version of the SARIF format that we are using.
@@ -74,6 +77,30 @@ export interface ResolvedQLPacks {
   [index: string]: string[];
 }
 
+/**
+ * The expected output of `codeql resolve tests`.
+ */
+export type ResolvedTests = string[];
+
+/**
+ * Options for `codeql test run`.
+ */
+export interface TestRunOptions {
+  logger?: Logger;
+}
+
+/**
+ * Event fired by `codeql test run`.
+ */
+export interface TestCompleted {
+  test: string;
+  pass: boolean;
+  messages: string[];
+  compilationMs: number;
+  evaluationMs: number;
+  expected: string;
+}
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -153,14 +180,22 @@ export class CodeQLCliServer implements Disposable {
 
   }
 
+  /**
+   * Get the path to the CodeQL CLI distribution, or throw an exception if not found.
+   */
+  private async getCodeQlPath(): Promise<string> {
+    const codeqlPath = await this.config.getCodeQlPathWithoutVersionCheck();
+    if (!codeqlPath) {
+      throw new Error('Failed to find CodeQL distribution.');
+    }
+    return codeqlPath;
+  }
+
   /**
    * Launch the cli server
    */
   private async launchProcess(): Promise<child_process.ChildProcessWithoutNullStreams> {
-    const config = await this.config.getCodeQlPathWithoutVersionCheck();
-    if (!config) {
-      throw new Error("Failed to find codeql distribution")
-    }
+    const config = await this.getCodeQlPath();
     return spawnServer(config, "CodeQL CLI Server", ["execute", "cli-server"], [], this.logger, _data => {})
   }
 
@@ -245,6 +280,69 @@ export class CodeQLCliServer implements Disposable {
     }
   }
 
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  private async* runAsyncCodeQlCliCommandInternal(
+    command: string[],
+    commandArgs: string[],
+    logger?: Logger
+  ): AsyncGenerator<string, void, unknown> {
+    // Add format argument first, in case commandArgs contains positional parameters.
+    const args = [
+      ...command,
+      '--format', 'jsonz',
+      ...commandArgs
+    ];
+
+    // Spawn the CodeQL process
+    const codeqlPath = await this.getCodeQlPath();
+    const childPromise = cpp.spawn(codeqlPath, args);
+    const child = childPromise.childProcess;
+
+    if (logger !== undefined) {
+      // The human-readable output goes to stderr.
+      logStream(child.stderr!, logger);
+    }
+
+    for await (const event of await splitStreamAtSeparators(child.stdout!, ['\0'])) {
+      yield event;
+    }
+
+    await childPromise;
+  }
+
+  /**
+   * Runs an asynchronous CodeQL CLI command without invoking the CLI server, returning any events
+   * fired by the command as an asynchronous generator.
+   *
+   * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
+   * @param commandArgs The arguments to pass to the `codeql` command.
+   * @param description Description of the action being run, to be shown in log and error messages.
+   * @param logger Logger to write text output from the command.
+   * @returns The sequence of async events produced by the command.
+   */
+  public async* runAsyncCodeQlCliCommand<EventType>(
+    command: string[],
+    commandArgs: string[],
+    description: string,
+    logger?: Logger
+  ): AsyncGenerator<EventType, void, unknown> {
+    for await (const event of await this.runAsyncCodeQlCliCommandInternal(command, commandArgs, logger)) {
+      try {
+        yield JSON.parse(event) as EventType;
+      } catch (err) {
+        throw new Error(`Parsing output of ${description} failed: ${err.stderr || err}`)
+      }
+    }
+  }
+
   /**
    * Runs a CodeQL CLI command on the server, returning the output as a string.
    * @param command The `codeql` command to be run, provided as an array of command/subcommand names.
@@ -325,6 +423,39 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<ResolvedQLPacks>(['resolve', 'qlpacks'], subcommandArgs, 'Resolving QL packs');
   }
 
+  /**
+   * Finds all available QL tests in a given directory.
+   * @param testPath Root of directory tree to search for tests.
+   * @returns The list of tests that were found.
+   */
+  public async resolveTests(testPath: string): Promise<ResolvedTests> {
+    const subcommandArgs = [
+      testPath
+    ];
+    return await this.runJsonCodeQlCliCommand<ResolvedTests>(['resolve', 'tests'], subcommandArgs, 'Resolving tests');
+  }
+
+  /**
+   * Runs QL tests.
+   * @param testPaths Full paths of the tests to run.
+   * @param workspaces Workspace paths to use as search paths for QL packs.
+   * @param options Additional options.
+   */
+  public async* runTests(testPaths: string[], workspaces: string[], options: TestRunOptions):
+    AsyncGenerator<TestCompleted, void, unknown> {
+
+    const subcommandArgs = [
+      '--additional-packs', workspaces.join(path.delimiter),
+      '--threads', '8',
+      ...testPaths
+    ];
+
+    for await (const event of await this.runAsyncCodeQlCliCommand<TestCompleted>(['test', 'run'],
+      subcommandArgs, 'Run CodeQL Tests', options.logger)) {
+      yield event;
+    }
+  }
+
   /**
    * Gets the metadata for a query.
    * @param queryPath The path to the query.
@@ -516,3 +647,100 @@ export async function runCodeQlCliCommand(codeQlPath: string, command: string[],
     throw new Error(`${description} failed: ${err.stderr || err}`)
   }
 }
+
+/**
+ * Buffer to hold state used when splitting a text stream into lines.
+ */
+class SplitBuffer {
+  private readonly decoder = new StringDecoder('utf8');
+  private readonly maxSeparatorLength: number;
+  private buffer = '';
+  private searchIndex = 0;
+
+  constructor(private readonly separators: readonly string[]) {
+    this.maxSeparatorLength = separators.map(s => s.length).reduce((a, b) => Math.max(a, b), 0);
+  }
+
+  /**
+   * Append new text data to the buffer.
+   * @param chunk The chunk of data to append.
+   */
+  public addChunk(chunk: Buffer): void {
+    this.buffer += this.decoder.write(chunk);
+  }
+
+  /**
+   * Signal that the end of the input stream has been reached.
+   */
+  public end(): void {
+    this.buffer += this.decoder.end();
+    this.buffer += this.separators[0];  // Append a separator to the end to ensure the last line is returned.
+  }
+
+  /**
+   * Extract the next full line from the buffer, if one is available.
+   * @returns The text of the next available full line (without the separator), or `undefined` if no
+   * line is available.
+   */
+  public getNextLine(): string | undefined {
+    while (this.searchIndex <= (this.buffer.length - this.maxSeparatorLength)) {
+      for (const separator of this.separators) {
+        if (this.buffer.startsWith(separator, this.searchIndex)) {
+          const line = this.buffer.substr(0, this.searchIndex);
+          this.buffer = this.buffer.substr(this.searchIndex + separator.length);
+          this.searchIndex = 0;
+          return line;
+        }
+      }
+      this.searchIndex++;
+    }
+
+    return undefined;
+  }
+}
+
+/**
+ * Splits a text stream into lines based on a list of valid line separators.
+ * @param stream The text stream to split. This stream will be fully consumed.
+ * @param separators The list of strings that act as line separators.
+ * @returns A sequence of lines (not including separators).
+ */
+async function* splitStreamAtSeparators(stream: Readable, separators: string[]):
+  AsyncGenerator<string, void, unknown> {
+
+  const buffer = new SplitBuffer(separators);
+  for await (const chunk of stream) {
+    buffer.addChunk(chunk);
+    let line: string | undefined;
+    do {
+      line = buffer.getNextLine();
+      if (line !== undefined) {
+        yield line;
+      }
+    } while (line !== undefined);
+  }
+  buffer.end();
+  let line: string | undefined;
+  do {
+    line = buffer.getNextLine();
+    if (line !== undefined) {
+      yield line;
+    }
+  } while (line !== undefined);
+}
+
+/**
+ *  Standard line endings for splitting human-readable text.
+ */
+const lineEndings = ['\r\n', '\r', '\n'];
+
+/**
+ * Log a text stream to a `Logger` interface.
+ * @param stream The stream to log.
+ * @param logger The logger that will consume the stream output.
+ */
+async function logStream(stream: Readable, logger: Logger): Promise<void> {
+  for await (const line of await splitStreamAtSeparators(stream, lineEndings)) {
+    logger.log(line);
+  }
+}

extensions/ql-vscode/src/logging.ts

@@ -12,7 +12,7 @@ export type ProgressReporter = Progress<{ message: string }>;
 
 /** A logger that writes messages to an output channel in the Output tab. */
 export class OutputChannelLogger extends DisposableObject implements Logger {
-  outputChannel: OutputChannel;
+  public readonly outputChannel: OutputChannel;
 
   constructor(title: string) {
     super();
@@ -27,7 +27,6 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   logWithoutTrailingNewline(message: string) {
     this.outputChannel.append(message);
   }
-
 }
 
 /** The global logger for the extension. */
@@ -38,3 +37,6 @@ export const queryServerLogger = new OutputChannelLogger('CodeQL Query Server');
 
 /** The logger for messages from the language server. */
 export const ideServerLogger = new OutputChannelLogger('CodeQL Language Server');
+
+/** The logger for messages from tests. */
+export const testLogger = new OutputChannelLogger('CodeQL Tests');