Merge pull request #1559 from github/elenatanasoiu/download-variant-analysis-results

Download variant analysis results

Author: elenatanasoiu

extensions/ql-vscode/src/extension.ts

@@ -105,8 +105,12 @@ import { createInitialQueryInfo } from './run-queries-shared';
 import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
 import { QueryRunner } from './queryRunner';
 import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
-import { VariantAnalysisMonitor } from './remote-queries/variant-analysis-monitor';
 import { VariantAnalysis } from './remote-queries/shared/variant-analysis';
+import {
+  VariantAnalysis as VariantAnalysisApiResponse,
+  VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository
+} from './remote-queries/gh-api/variant-analysis';
+import { VariantAnalysisManager } from './remote-queries/variant-analysis-manager';
 
 /**
  * extension.ts
@@ -896,13 +900,23 @@ async function activateWithInstalledDistribution(
     })
   );
 
-  const variantAnalysisMonitor = new VariantAnalysisMonitor(ctx, logger);
+  const variantAnalysisManager = new VariantAnalysisManager(ctx, logger);
   ctx.subscriptions.push(
     commandRunner('codeQL.monitorVariantAnalysis', async (
       variantAnalysis: VariantAnalysis,
       token: CancellationToken
     ) => {
-      await variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis, token);
+      await variantAnalysisManager.monitorVariantAnalysis(variantAnalysis, token);
+    })
+  );
+
+  ctx.subscriptions.push(
+    commandRunner('codeQL.autoDownloadVariantAnalysisResult', async (
+      scannedRepo: ApiVariantAnalysisScannedRepository,
+      variantAnalysisSummary: VariantAnalysisApiResponse,
+      token: CancellationToken
+    ) => {
+      await variantAnalysisManager.autoDownloadVariantAnalysisResult(scannedRepo, variantAnalysisSummary, token);
     })
   );
 

extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts

@@ -74,6 +74,19 @@ export async function getVariantAnalysisRepo(
   return response.data;
 }
 
+export async function getVariantAnalysisRepoResult(
+  credentials: Credentials,
+  downloadUrl: string,
+): Promise<unknown> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<VariantAnalysisRepoTask> = await octokit.request(
+    `GET ${downloadUrl}`
+  );
+
+  return response.data;
+}
+
 export async function getRepositoryFromNwo(
   credentials: Credentials,
   owner: string,

extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts

@@ -0,0 +1,81 @@
+import * as ghApiClient from './gh-api/gh-api-client';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { CancellationToken, ExtensionContext } from 'vscode';
+import { DisposableObject } from '../pure/disposable-object';
+import { Logger } from '../logging';
+import { Credentials } from '../authentication';
+import { VariantAnalysisMonitor } from './variant-analysis-monitor';
+import {
+  VariantAnalysis as VariantAnalysisApiResponse,
+  VariantAnalysisRepoTask,
+  VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository
+} from './gh-api/variant-analysis';
+import { VariantAnalysis } from './shared/variant-analysis';
+import { getErrorMessage } from '../pure/helpers-pure';
+
+export class VariantAnalysisManager extends DisposableObject {
+  private readonly variantAnalysisMonitor: VariantAnalysisMonitor;
+
+  constructor(
+    private readonly ctx: ExtensionContext,
+    logger: Logger,
+  ) {
+    super();
+    this.variantAnalysisMonitor = new VariantAnalysisMonitor(ctx, logger);
+  }
+
+  public async monitorVariantAnalysis(
+    variantAnalysis: VariantAnalysis,
+    cancellationToken: CancellationToken
+  ): Promise<void> {
+    await this.variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis, cancellationToken);
+  }
+
+  public async autoDownloadVariantAnalysisResult(
+    scannedRepo: ApiVariantAnalysisScannedRepository,
+    variantAnalysisSummary: VariantAnalysisApiResponse,
+    cancellationToken: CancellationToken
+  ): Promise<void> {
+
+    const credentials = await Credentials.initialize(this.ctx);
+    if (!credentials) { throw Error('Error authenticating with GitHub'); }
+
+    if (cancellationToken && cancellationToken.isCancellationRequested) {
+      return;
+    }
+
+    let repoTask: VariantAnalysisRepoTask;
+    try {
+      repoTask = await ghApiClient.getVariantAnalysisRepo(
+        credentials,
+        variantAnalysisSummary.controller_repo.id,
+        variantAnalysisSummary.id,
+        scannedRepo.repository.id
+      );
+    }
+    catch (e) { throw new Error(`Could not download the results for variant analysis with id: ${variantAnalysisSummary.id}. Error: ${getErrorMessage(e)}`); }
+
+    if (repoTask.artifact_url) {
+      const resultDirectory = path.join(
+        this.ctx.globalStorageUri.fsPath,
+        'variant-analyses',
+        `${variantAnalysisSummary.id}`,
+        scannedRepo.repository.full_name
+      );
+
+      const storagePath = path.join(
+        resultDirectory,
+        scannedRepo.repository.full_name
+      );
+
+      const result = await ghApiClient.getVariantAnalysisRepoResult(
+        credentials,
+        repoTask.artifact_url
+      );
+
+      fs.mkdirSync(resultDirectory, { recursive: true });
+      await fs.writeFile(storagePath, JSON.stringify(result, null, 2), 'utf8');
+    }
+  }
+}

extensions/ql-vscode/src/remote-queries/variant-analysis-monitor.ts

@@ -1,4 +1,4 @@
-import * as vscode from 'vscode';
+import { ExtensionContext, CancellationToken, commands } from 'vscode';
 import { Credentials } from '../authentication';
 import { Logger } from '../logging';
 import * as ghApiClient from './gh-api/gh-api-client';
@@ -17,14 +17,14 @@ export class VariantAnalysisMonitor {
   public static sleepTime = 5000;
 
   constructor(
-    private readonly extensionContext: vscode.ExtensionContext,
+    private readonly extensionContext: ExtensionContext,
     private readonly logger: Logger
   ) {
   }
 
   public async monitorVariantAnalysis(
     variantAnalysis: VariantAnalysis,
-    cancellationToken: vscode.CancellationToken
+    cancellationToken: CancellationToken
   ): Promise<VariantAnalysisMonitorResult> {
 
     const credentials = await Credentials.initialize(this.extensionContext);
@@ -64,6 +64,7 @@ export class VariantAnalysisMonitor {
       if (variantAnalysisSummary.scanned_repositories) {
         variantAnalysisSummary.scanned_repositories.forEach(scannedRepo => {
           if (!scannedReposDownloaded.includes(scannedRepo.repository.id) && scannedRepo.analysis_status === 'succeeded') {
+            void commands.executeCommand('codeQL.autoDownloadVariantAnalysisResult', scannedRepo, variantAnalysisSummary);
             scannedReposDownloaded.push(scannedRepo.repository.id);
           }
         });

extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/run-remote-query.test.ts

@@ -1,7 +1,7 @@
 import { assert, expect } from 'chai';
 import * as path from 'path';
 import * as sinon from 'sinon';
-import { CancellationToken, extensions, QuickPickItem, Uri, window } from 'vscode';
+import { CancellationTokenSource, extensions, QuickPickItem, Uri, window } from 'vscode';
 import * as fs from 'fs-extra';
 import * as os from 'os';
 import * as yaml from 'js-yaml';
@@ -32,7 +32,7 @@ describe('Remote queries', function() {
 
   let cli: CodeQLCliServer;
   let credentials: Credentials = {} as unknown as Credentials;
-  let token: CancellationToken;
+  let cancellationTokenSource: CancellationTokenSource;
   let progress: sinon.SinonSpy;
   let showQuickPickSpy: sinon.SinonStub;
   let getRepositoryFromNwoStub: sinon.SinonStub;
@@ -55,9 +55,15 @@ describe('Remote queries', function() {
       this.skip();
     }
     credentials = {} as unknown as Credentials;
-    token = {
-      isCancellationRequested: false
-    } as unknown as CancellationToken;
+
+    cancellationTokenSource = {
+      token: {
+        isCancellationRequested: false,
+        onCancellationRequested: sandbox.stub()
+      },
+      cancel: sandbox.stub(),
+      dispose: sandbox.stub()
+    };
 
     progress = sandbox.spy();
     // Should not have asked for a language
@@ -88,7 +94,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is part of a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
       expect(querySubmissionResult).to.be.ok;
       const queryPackRootDir = querySubmissionResult!.queryDirPath!;
       printDirectoryContents(queryPackRootDir);
@@ -149,7 +155,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is not part of a qlpack', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
       expect(querySubmissionResult).to.be.ok;
       const queryPackRootDir = querySubmissionResult!.queryDirPath!;
 
@@ -212,7 +218,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is nested inside a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
       expect(querySubmissionResult).to.be.ok;
       const queryPackRootDir = querySubmissionResult!.queryDirPath!;
 
@@ -274,9 +280,9 @@ describe('Remote queries', function() {
     it('should cancel a run before uploading', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
 
-      token.isCancellationRequested = true;
+      cancellationTokenSource.token.isCancellationRequested = true;
 
       try {
         await promise;
@@ -300,7 +306,7 @@ describe('Remote queries', function() {
     it('should run a variant analysis that is part of a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
       expect(querySubmissionResult).to.be.ok;
       const variantAnalysis = querySubmissionResult!.variantAnalysis!;
       expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -313,7 +319,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is not part of a qlpack', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
       expect(querySubmissionResult).to.be.ok;
       const variantAnalysis = querySubmissionResult!.variantAnalysis!;
       expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -326,7 +332,7 @@ describe('Remote queries', function() {
     it('should run a remote query that is nested inside a qlpack', async () => {
       const fileUri = getFile('data-remote-qlpack-nested/subfolder/in-pack.ql');
 
-      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const querySubmissionResult = await runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
       expect(querySubmissionResult).to.be.ok;
       const variantAnalysis = querySubmissionResult!.variantAnalysis!;
       expect(variantAnalysis.id).to.be.equal(mockApiResponse.id);
@@ -339,9 +345,9 @@ describe('Remote queries', function() {
     it('should cancel a run before uploading', async () => {
       const fileUri = getFile('data-remote-no-qlpack/in-pack.ql');
 
-      const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, token);
+      const promise = runRemoteQuery(cli, credentials, fileUri, true, progress, cancellationTokenSource.token);
 
-      token.isCancellationRequested = true;
+      cancellationTokenSource.token.isCancellationRequested = true;
 
       try {
         await promise;