Implement QL Test support (using odasa for now)

The PR contains the initial implementing of QL Test support in CodeQL for Visual Studio Code. Because QL Test support isn't quite ready in the CLI yet, this PR uses `odasa` to run the tests for now. As CLI support comes online, it should be straightforward to swap out the implementation to use the CLI.

The treeview UI for the tests is implemented via the `hbenl.vscode-test-explorer` extension. This extension is open source, and appears to be actively maintained. It's used by a couple dozen existing extensions for tests for various languages. The extension doesn't really do anything on its own, so taking it as a dependency isn't introducing any unwanted UI clutter. Note that I did have to remove the `--disable-extensions` argument from `launch.json`, because otherwise the test explorer extension gets disabled, preventing our own extension from loading.

The UI will display a root node for each QL pack that contains tests, with the actual test directories and files as descendants of that root node. We consider only those QL packs in the workspace; QL packs on the default CodeQL search path are ignored. We use `codeql resolve qlpacks` to find the packs, and then watch all `qlpack.yml` files in the workspace for changes in order to refresh the pack discovery when necessary. Ideally, we'd have the CLI return a set of path patterns to watch, but for now the current implementation works fine.

To discover the tests within a given pack, we walk the pack's directory tree manually for now, until the relevant CLI command is available. Because we do not yet have a mechanism in `qlpack.yml` to specify whether or not the pack contains tests, we assume that any pack whose name ends with "-tests" to contain nothing but tests, and any other pack to contain no tests. This is sufficient for the tests in the QL repo. As with QL pack discovery, we watch the file system for changes in `.ql` and `.qlref` files in order to refresh the tree of tests if anything changes.

To actually run the tests, we just invoke `odasa qltest` with the appropriate arguments. This code is pretty much a straight copy-and-paste from the repo where I've had a private version of QL Test support for several months. Once we can run tests via the CLI, this will all be deleted.

The `test-ui.ts` file implements a couple of additional commands for the context menu of the test treeview. You can accept the output of a failing test (copying the `.actual` file to the `.expected` file), and you can bring up a diff view of the `.expected` and `.actual` files).

This PR includes a couple of related utility classes. `UIService` makes it a little easier to implement a service that handles VS Code commands. `Discovery` is a base class that handles most of the work that is shared between the different kinds of discovery that we do, like avoiding running multiple discovery operations simultaneously if we get a storm of file change notifications.

Author: Dave Bartolomeo

.vscode/launch.json

@@ -8,8 +8,7 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--disable-extensions"
+        "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -29,8 +28,7 @@
       "runtimeExecutable": "${execPath}",
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/test",
-        "--disable-extensions"
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/test"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -51,8 +49,7 @@
       "runtimeExecutable": "${execPath}",
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
-        "--disable-extensions"
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,
@@ -70,7 +67,7 @@
       "args": [
         "--extensionDevelopmentPath=${workspaceRoot}/dist/vscode-codeql",
         "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
-        "${workspaceRoot}/extensions/ql-vscode/test/data",
+        "${workspaceRoot}/extensions/ql-vscode/test/data"
       ],
       "stopOnEntry": false,
       "sourceMaps": true,

common/config/rush/shrinkwrap.yaml

@@ -67,6 +67,8 @@ dependencies:
   vscode-jsonrpc: 4.0.0
   vscode-languageclient: 5.2.1
   vscode-test: 1.2.0
+  vscode-test-adapter-api: 1.7.0
+  vscode-test-adapter-util: 0.7.0
   webpack: 4.39.1
   webpack-cli: 3.3.6
 packages:
@@ -6122,6 +6124,21 @@ packages:
     dev: false
     resolution:
       integrity: sha512-lTmS6AlAlMHOvPQemVwo3CezxBp0sNB95KNPkqp3Nxd5VFEnuG1ByM0zlRWos0zjO3ZWtkvhal0COgiV1xIA4A==
+  /vscode-test-adapter-api/1.7.0:
+    dev: false
+    engines:
+      vscode: ^1.23.0
+    resolution:
+      integrity: sha512-X0rTcoDhDBmpmJuev2C5+GHGZD41nmcRYoSe7iw5e9/aIPTOFve1T1F5x9gb+zXoNQnkXSDibyMkeHDKtIkqCg==
+  /vscode-test-adapter-util/0.7.0:
+    dependencies:
+      tslib: 1.10.0
+      vscode-test-adapter-api: 1.7.0
+    dev: false
+    engines:
+      vscode: ^1.24.0
+    resolution:
+      integrity: sha512-eAsB8koXct5JytvUcV62wLEBCQfsoclauzMLEFT6H0qBr1h8LyRc+dGDcs48pO28yFOo6VV+5AwCRLxTKh7TzQ==
   /vscode-test/1.2.0:
     dependencies:
       http-proxy-agent: 2.1.0
@@ -6433,7 +6450,7 @@ packages:
       child-process-promise: 2.2.1
       fs-extra: 8.1.0
       glob: 7.1.4
-      glob-promise: 3.4.0
+      glob-promise: /glob-promise/3.4.0/glob@7.1.4
       gulp: 4.0.2
       gulp-sourcemaps: 2.6.5
       gulp-typescript: /gulp-typescript/5.0.1/typescript@3.7.2
@@ -6449,7 +6466,7 @@ packages:
     dev: false
     name: '@rush-temp/build-tasks'
     resolution:
-      integrity: sha512-ta2kXnX7phnKrO7rxdJl5A9Vtd8B4RDyoae3vhdI1d+COeITaXDd9xdPxo8lvduPSJTw2+HnzOgOu2pMAKSjTw==
+      integrity: sha512-WBog5Gepo348LYeu7FAZMORJpgx0CNFM+IjgLcKNFYnN9YOCRQogUkRL7ZmBt8WXoew1S5OyVltmEM8XPy8i+w==
       tarball: 'file:projects/build-tasks.tgz'
     version: 0.0.0
   'file:projects/semmle-bqrs.tgz':
@@ -6461,7 +6478,7 @@ packages:
     dev: false
     name: '@rush-temp/semmle-bqrs'
     resolution:
-      integrity: sha512-24GdnvMbGfQIWMfgDhift+kYJDnG7dX03NrpX4ajZ2rckteysvq2/K7XI1OXGvUuqrt3m0/+GRDHpSI9XKDJJA==
+      integrity: sha512-ufPu8zLXf9JvPCRycWLiFTDb5rZ7bqxQZuiFjy1DAxnatEG5VJITPSXwDFVc11qpjJpaFd4hI+4QtOda7d5zww==
       tarball: 'file:projects/semmle-bqrs.tgz'
     version: 0.0.0
   'file:projects/semmle-io-node.tgz':
@@ -6473,7 +6490,7 @@ packages:
     dev: false
     name: '@rush-temp/semmle-io-node'
     resolution:
-      integrity: sha512-Bj0ax/bASrHV7tamOuXZZdd3UOB4NBKdjdszIRaDvDRTu8RlEst+TVoUhkfy30qb2/6ePp3/juOJyyiBJN7u8Q==
+      integrity: sha512-jB3C3WWEI991Kr3knPKUwqqNi040WmYCubLJJG7AK1nz3V1YjmyLHIAdjqwOgDNXYKCQPC6tlaEgljbs2Q/kIQ==
       tarball: 'file:projects/semmle-io-node.tgz'
     version: 0.0.0
   'file:projects/semmle-io.tgz':
@@ -6484,7 +6501,7 @@ packages:
     dev: false
     name: '@rush-temp/semmle-io'
     resolution:
-      integrity: sha512-NtyviDSevxbd+hj4J66LucOzo8LU2hJ1Jh0eHw0Qu3tRZPUT8HcQlseyy29AvZR8n8eppfEZiAm/JdiHfmRPMA==
+      integrity: sha512-6DFvjDclWTihDToSf31Hh+wQNhLGkA37l4QajeW/w6gS4NHjSrFL1qBlS7dLUY80VC/8nQJH9foe3r6dfEfQYw==
       tarball: 'file:projects/semmle-io.tgz'
     version: 0.0.0
   'file:projects/semmle-vscode-utils.tgz':
@@ -6496,19 +6513,20 @@ packages:
     dev: false
     name: '@rush-temp/semmle-vscode-utils'
     resolution:
-      integrity: sha512-5y5r8SDoN9Fp44naC9gUe8rOexeckXg2T0h9QCJAIcEgnFqOxzRc6Rv9gbMUStFKNh+rFlvmYmgPAdg5QkfgUg==
+      integrity: sha512-yE5S1wsnrsJ8lTt9O9ALedlvH37M9sWQha7sL5iQ3P6dn2KsyUItGsJDnFoh2f0wy3TpZuj3p/KTEDBbgjXBGg==
       tarball: 'file:projects/semmle-vscode-utils.tgz'
     version: 0.0.0
   'file:projects/typescript-config.tgz':
     dev: false
     name: '@rush-temp/typescript-config'
     resolution:
-      integrity: sha512-XuUIySaNoooIduvehnlKYaHqZJmmQoCqB1RtKhNszjCYZaSSJAnKVucViWBf5oNLKSNP7NchrD7gcoBlQ3xYvw==
+      integrity: sha512-kSFyvKy63jUHFVXQEzALiYfsTdn7J+Y7PcqtUVo9GndU5b5Xh3rBpVbZD1QN8+y8GfT0m/sdZZQVyH0h+On11Q==
       tarball: 'file:projects/typescript-config.tgz'
     version: 0.0.0
   'file:projects/vscode-codeql.tgz':
     dependencies:
       '@types/chai': 4.1.7
+      '@types/child-process-promise': 2.2.1
       '@types/classnames': 2.2.9
       '@types/fs-extra': 8.0.0
       '@types/glob': 7.1.1
@@ -6559,13 +6577,15 @@ packages:
       vscode-jsonrpc: 4.0.0
       vscode-languageclient: 5.2.1
       vscode-test: 1.2.0
+      vscode-test-adapter-api: 1.7.0
+      vscode-test-adapter-util: 0.7.0
       webpack: 4.39.1
       webpack-cli: /webpack-cli/3.3.6/webpack@4.39.1
       xml2js: 0.4.19
     dev: false
     name: '@rush-temp/vscode-codeql'
     resolution:
-      integrity: sha512-DE97bdxda65gVLZne73QzBpj2hyCbyzvQiRZxrJqDP1rkF62EGNohBSmlEQs8H2Jp8hxh5RhPhm/yUx70G7KEA==
+      integrity: sha512-1zj1CO19sanU2olFV3Kwf9OrG+92KO25WR6K1vnVHmScMj8H7aegRrjwyNeHi/IzxXDCAp68CZK+d4vEWBEa+g==
       tarball: 'file:projects/vscode-codeql.tgz'
     version: 0.0.0
 registry: 'https://registry.npmjs.org/'
@@ -6640,5 +6660,7 @@ specifiers:
   vscode-jsonrpc: ^4.0.0
   vscode-languageclient: ^5.2.1
   vscode-test: ^1.0.0
+  vscode-test-adapter-api: ~1.7.0
+  vscode-test-adapter-util: ~0.7.0
   webpack: ^4.38.0
   webpack-cli: ^3.3.2

extensions/ql-vscode/package.json

@@ -18,6 +18,9 @@
   "categories": [
     "Programming Languages"
   ],
+  "extensionDependencies": [
+    "hbenl.vscode-test-explorer"
+  ],
   "activationEvents": [
     "onLanguage:ql",
     "onView:codeQLDatabases",
@@ -95,14 +98,20 @@
           "description": "Number of threads for running queries."
         },
         "codeQL.runningQueries.timeout": {
-          "type": ["integer", "null"],
+          "type": [
+            "integer",
+            "null"
+          ],
           "default": null,
           "minimum": 0,
           "maximum": 2147483647,
           "description": "Timeout (in seconds) for running queries. Leave blank or set to zero for no timeout."
         },
         "codeQL.runningQueries.memory": {
-          "type": ["integer", "null"],
+          "type": [
+            "integer",
+            "null"
+          ],
           "default": null,
           "minimum": 1024,
           "description": "Memory (in MB) to use for running queries. Leave blank for CodeQL to choose a suitable value based on your system's available memory."
@@ -111,6 +120,22 @@
           "type": "boolean",
           "default": false,
           "description": "Enable debug logging and tuple counting when running CodeQL queries. This information is useful for debugging query performance."
+        },
+        "codeQL.tests.odasaDistributionPath": {
+          "scope": "window",
+          "type": "string",
+          "default": "",
+          "description": "Location of the ODASA distribution"
+        },
+        "codeQL.tests.odasaLicensePath": {
+          "scope": "window",
+          "type": "string",
+          "description": "Location of the directory containing the ODASA license file"
+        },
+        "codeQL.tests.numberOfThreads": {
+          "scope": "window",
+          "type": "integer",
+          "description": "Number of threads to use for CodeQL test execution"
         }
       }
     },
@@ -170,6 +195,14 @@
       {
         "command": "codeQLQueryHistory.itemClicked",
         "title": "Query History Item"
+      },
+      {
+        "command": "codeQLTests.showOutputDifferences",
+        "title": "CodeQL: Show Test Output Differences"
+      },
+      {
+        "command": "codeQLTests.acceptOutput",
+        "title": "CodeQL: Accept Test Output"
       }
     ],
     "menus": {
@@ -205,6 +238,16 @@
           "command": "codeQLQueryHistory.removeHistoryItem",
           "group": "9_qlCommands",
           "when": "view == codeQLQueryHistory"
+        },
+        {
+          "command": "codeQLTests.showOutputDifferences",
+          "group": "qltest@1",
+          "when": "view == test-explorer && viewItem == testWithSource"
+        },
+        {
+          "command": "codeQLTests.acceptOutput",
+          "group": "qltest@2",
+          "when": "view == test-explorer && viewItem == testWithSource"
         }
       ],
       "explorer/context": [
@@ -298,6 +341,7 @@
     "format": "tsfmt -r"
   },
   "dependencies": {
+    "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
     "fs-extra": "^8.1.0",
     "glob-promise": "^3.4.0",
@@ -310,10 +354,13 @@
     "tmp": "^0.1.0",
     "unzipper": "~0.10.5",
     "vscode-jsonrpc": "^4.0.0",
-    "vscode-languageclient": "^5.2.1"
+    "vscode-languageclient": "^5.2.1",
+    "vscode-test-adapter-api": "~1.7.0",
+    "vscode-test-adapter-util": "~0.7.0"
   },
   "devDependencies": {
     "@types/chai": "^4.1.7",
+    "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
     "@types/fs-extra": "^8.0.0",
     "@types/glob": "^7.1.1",
@@ -333,7 +380,6 @@
     "@types/xml2js": "~0.4.4",
     "build-tasks": "^0.0.1",
     "chai": "^4.2.0",
-    "child-process-promise": "^2.2.1",
     "css-loader": "~3.1.0",
     "glob": "^7.1.4",
     "gulp": "^4.0.2",

extensions/ql-vscode/src/cli.ts

@@ -67,6 +67,13 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve qlpacks`.
+ */
+export interface ResolvedQLPacks {
+  [index: string]: string[];
+}
+
 /**
  * This class manages a cli server started by `codeql execute cli-server` to
  * run commands without the overhead of starting a new java
@@ -303,6 +310,21 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<QuerySetup>(['resolve', 'library-path'], subcommandArgs, "Resolving library paths");
   }
 
+  /**
+   * Finds all available QL packs.
+   * @param workspaces The current open workspaces
+   * @param searchPath Overrides the default QL pack search path
+   */
+  async resolveQLPacks(workspaces: string[], searchPath?: string[]): Promise<ResolvedQLPacks> {
+    const subcommandArgs = [
+      '--additional-packs', workspaces.join(path.delimiter)
+    ];
+    if (searchPath !== undefined) {
+      subcommandArgs.push('--search-path', searchPath.join(path.delimiter));
+    }
+    return await this.runJsonCodeQlCliCommand<ResolvedQLPacks>(['resolve', 'qlpacks'], subcommandArgs, 'Resolving QL packs');
+  }
+
   /**
    * Gets the metadata for a query.
    * @param queryPath The path to the query.