Run tests with nightly CLI

Author: edoardopirovano

.github/workflows/main.yml

@@ -51,12 +51,30 @@ jobs:
           name: vscode-codeql-extension
           path: artifacts
 
+  find-nightly:
+    name: Find Nightly Release
+    runs-on: ubuntu-latest
+    outputs:
+      url: ${{ steps.get-url.outputs.nightly-url }}
+    steps:
+      - name: Get Nightly Release URL
+        id: get-url
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        shell: bash
+        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+        run: |
+          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
+          echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST"
+    
   test:
     name: Test
     runs-on: ${{ matrix.os }}
+    needs: [find-nightly]
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
+        version: [stable, nightly]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -89,7 +107,12 @@ jobs:
       - name: Install CodeQL
         run: |
           mkdir codeql-home
-          curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
+          if [ ${{ matrix.version }} = "stable" ]
+          then
+            curl -L --silent https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql.zip -o codeql-home/codeql.zip
+          else
+            curl -L --silent ${{ needs.find-nightly.outputs.url }}/codeql.zip -o codeql-home/codeql.zip
+          fi
           unzip -q -o codeql-home/codeql.zip -d codeql-home
           unzip -q -o codeql-home/codeql.zip codeql/codeql.exe -d codeql-home
           rm codeql-home/codeql.zip
@@ -124,12 +147,14 @@ jobs:
   cli-test:
     name: CLI Test
     runs-on: ${{ matrix.os }}
+    needs: [find-nightly]
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.2.6', 'v2.3.3', 'v2.4.6', 'v2.5.7']
+        version: ['v2.2.6', 'v2.3.3', 'v2.4.6', 'v2.5.7', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
+      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
       TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'
 
     steps:

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -24,13 +24,15 @@ describe('Use cli', function() {
     }
   });
 
-  it('should have the correct version of the cli', async () => {
-    expect(
-      (await cli.getVersion()).toString()
-    ).to.eq(
-      new SemVer(process.env.CLI_VERSION || '').toString()
-    );
-  });
+  if (process.env.CLI_VERSION !== 'nightly') {
+    it('should have the correct version of the cli', async () => {
+      expect(
+        (await cli.getVersion()).toString()
+      ).to.eq(
+        new SemVer(process.env.CLI_VERSION || '').toString()
+      );
+    });
+  }
 
   it('should resolve ram', async () => {
     const result = await (cli as any).resolveRam(8192);

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -6,10 +6,15 @@ import { workspace } from 'vscode';
 
 /**
  * This module ensures that the proper CLI is available for tests of the extension.
- * There are two environment variables to control this module:
+ * There are three environment variables to control this module:
  *
  *     - CLI_VERSION: The version of the CLI to install. Defaults to the most recent
  *       version. Note that for now, we must maintain the default version by hand.
+ *       This may be set to `nightly`, in which case the `NIGHTLY_URL` variable must
+ *       also be set.
+ * 
+ *     - NIGHTLY_URL: The URL for a nightly release of the CodeQL CLI that will be
+ *       used if `CLI_VERSION` is set to `nightly`.
  *
  *     - CLI_BASE_DIR: The base dir where the CLI will be downloaded and unzipped.
  *       The download location is `${CLI_BASE_DIR}/assets` and the unzip loction is
@@ -133,6 +138,11 @@ export function skipIfNoCodeQL(context: Mocha.Context) {
  * Url to download from
  */
 function getCliDownloadUrl(assetName: string) {
+  if (CLI_VERSION === 'nightly') {
+    if (!process.env.NIGHTLY_URL)
+      throw new Error('Nightly CLI was specified but no URL to download it from was given!');
+    return process.env.NIGHTLY_URL + `/${assetName}`;
+  }
   return `https://github.com/github/codeql-cli-binaries/releases/download/${CLI_VERSION}/${assetName}`;
 }