Merge branch 'main' into koesie10/show-variant-analysis-view

Author: koesie10

.github/workflows/main.yml

@@ -139,7 +139,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        version: ['v2.6.3', 'v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'nightly']
+        version: ['v2.6.3', 'v2.7.6', 'v2.8.5', 'v2.9.4', 'v2.10.5', 'v2.11.0', 'nightly']
     env:
       CLI_VERSION: ${{ matrix.version }}
       NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}

extensions/ql-vscode/package-lock.json

@@ -1206,7 +1206,7 @@
   },
   "dependencies": {
     "@octokit/plugin-retry": "^3.0.9",
-    "@octokit/rest": "^18.5.6",
+    "@octokit/rest": "^19.0.4",
     "@primer/octicons-react": "^16.3.0",
     "@primer/react": "^35.0.0",
     "@vscode/codicons": "^0.0.31",
@@ -1246,6 +1246,7 @@
   "devDependencies": {
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
+    "@faker-js/faker": "^7.5.0",
     "@storybook/addon-actions": "^6.5.10",
     "@storybook/addon-essentials": "^6.5.10",
     "@storybook/addon-interactions": "^6.5.10",

extensions/ql-vscode/package.json

@@ -105,6 +105,8 @@ import { createInitialQueryInfo } from './run-queries-shared';
 import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
 import { QueryRunner } from './queryRunner';
 import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
+import { VariantAnalysisMonitor } from './remote-queries/variant-analysis-monitor';
+import { VariantAnalysis } from './remote-queries/shared/variant-analysis';
 
 /**
  * extension.ts
@@ -894,6 +896,16 @@ async function activateWithInstalledDistribution(
     })
   );
 
+  const variantAnalysisMonitor = new VariantAnalysisMonitor(ctx, logger);
+  ctx.subscriptions.push(
+    commandRunner('codeQL.monitorVariantAnalysis', async (
+      variantAnalysis: VariantAnalysis,
+      token: CancellationToken
+    ) => {
+      await variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis, token);
+    })
+  );
+
   ctx.subscriptions.push(
     commandRunner('codeQL.autoDownloadRemoteQueryResults', async (
       queryResult: RemoteQueryResult,

extensions/ql-vscode/src/extension.ts

@@ -62,7 +62,7 @@ export interface VariantAnalysisSkippedRepositoryGroup {
 
 export interface VariantAnalysisNotFoundRepositoryGroup {
   repository_count: number,
-  repository_nwos: string[]
+  repository_full_names: string[]
 }
 export interface VariantAnalysisRepoTask {
   repository: Repository,

extensions/ql-vscode/src/remote-queries/gh-api/variant-analysis.ts

@@ -26,8 +26,9 @@ import { QueryMetadata } from '../pure/interface-types';
 import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
 import * as ghApiClient from './gh-api/gh-api-client';
 import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
-import { parseVariantAnalysisQueryLanguage, VariantAnalysis, VariantAnalysisStatus, VariantAnalysisSubmission } from './shared/variant-analysis';
+import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
 import { Repository } from './shared/repository';
+import { processVariantAnalysis } from './variant-analysis-processor';
 
 export interface QlPack {
   name: string;
@@ -270,30 +271,16 @@ export async function runRemoteQuery(
         variantAnalysisSubmission
       );
 
-      const variantAnalysis: VariantAnalysis = {
-        id: variantAnalysisResponse.id,
-        controllerRepoId: variantAnalysisResponse.controller_repo.id,
-        query: {
-          name: variantAnalysisSubmission.query.name,
-          filePath: variantAnalysisSubmission.query.filePath,
-          language: variantAnalysisSubmission.query.language,
-        },
-        databases: {
-          repositories: variantAnalysisSubmission.databases.repositories,
-          repositoryLists: variantAnalysisSubmission.databases.repositoryLists,
-          repositoryOwners: variantAnalysisSubmission.databases.repositoryOwners,
-        },
-        status: VariantAnalysisStatus.InProgress,
-      };
-
-      void logger.log(`Variant analysis:\n${JSON.stringify(variantAnalysis, null, 2)}`);
+      const processedVariantAnalysis = processVariantAnalysis(variantAnalysisSubmission, variantAnalysisResponse);
 
-      void showAndLogInformationMessage(`Variant analysis ${variantAnalysis.query.name} submitted for processing`);
+      void logger.log(`Variant analysis:\n${JSON.stringify(processedVariantAnalysis, null, 2)}`);
 
-      void commands.executeCommand('codeQL.openVariantAnalysisView', variantAnalysis.id);
+      void showAndLogInformationMessage(`Variant analysis ${processedVariantAnalysis.query.name} submitted for processing`);
 
-      return { variantAnalysis };
+      void commands.executeCommand('codeQL.openVariantAnalysisView', processedVariantAnalysis.id);
+      void commands.executeCommand('codeQL.monitorVariantAnalysis', processedVariantAnalysis);
 
+      return { variantAnalysis: processedVariantAnalysis };
     } else {
       const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, controllerRepo, base64Pack, dryRun);
 

extensions/ql-vscode/src/remote-queries/run-remote-query.ts

@@ -0,0 +1,16 @@
+import { VariantAnalysis } from './variant-analysis';
+
+export type VariantAnalysisMonitorStatus =
+  | 'InProgress'
+  | 'CompletedSuccessfully'
+  | 'CompletedUnsuccessfully'
+  | 'Failed'
+  | 'Cancelled'
+  | 'TimedOut';
+
+export interface VariantAnalysisMonitorResult {
+  status: VariantAnalysisMonitorStatus;
+  error?: string;
+  scannedReposDownloaded?: number[],
+  variantAnalysis?: VariantAnalysis
+}

extensions/ql-vscode/src/remote-queries/shared/variant-analysis-monitor-result.ts

@@ -1,4 +1,5 @@
 import { Repository } from './repository';
+import { AnalysisAlert, AnalysisRawResults } from './analysis-result';
 
 export interface VariantAnalysis {
   id: number,
@@ -72,10 +73,19 @@ export interface VariantAnalysisSkippedRepositories {
 
 export interface VariantAnalysisSkippedRepositoryGroup {
   repositoryCount: number,
-  repositories: Array<{
-    id?: number,
-    fullName: string
-  }>
+  repositories: VariantAnalysisSkippedRepository[],
+}
+
+export interface VariantAnalysisSkippedRepository {
+  id?: number,
+  fullName: string,
+  private?: boolean,
+}
+
+export interface VariantAnalysisScannedRepositoryResult {
+  repositoryId: number;
+  interpretedResults?: AnalysisAlert[];
+  rawResults?: AnalysisRawResults;
 }
 
 /**
@@ -102,16 +112,24 @@ export interface VariantAnalysisSubmission {
 }
 
 /**
- * @param repo
- * @returns whether the repo scan is in a completed state, i.e. it cannot normally change state anymore
+ * @param status
+ * @returns whether the status is in a completed state, i.e. it cannot normally change state anymore
  */
-export function hasRepoScanCompleted(repo: VariantAnalysisScannedRepository): boolean {
+export function isCompletedAnalysisRepoStatus(status: VariantAnalysisRepoStatus): boolean {
   return [
     // All states that indicates the repository has been scanned and cannot
     // change status anymore.
     VariantAnalysisRepoStatus.Succeeded, VariantAnalysisRepoStatus.Failed,
     VariantAnalysisRepoStatus.Canceled, VariantAnalysisRepoStatus.TimedOut,
-  ].includes(repo.analysisStatus);
+  ].includes(status);
+}
+
+/**
+ * @param repo
+ * @returns whether the repo scan is in a completed state, i.e. it cannot normally change state anymore
+ */
+export function hasRepoScanCompleted(repo: VariantAnalysisScannedRepository): boolean {
+  return isCompletedAnalysisRepoStatus(repo.analysisStatus);
 }
 
 /**

extensions/ql-vscode/src/remote-queries/shared/variant-analysis.ts

@@ -0,0 +1,85 @@
+import * as vscode from 'vscode';
+import { Credentials } from '../authentication';
+import { Logger } from '../logging';
+import * as ghApiClient from './gh-api/gh-api-client';
+
+import { VariantAnalysis, VariantAnalysisStatus } from './shared/variant-analysis';
+import {
+  VariantAnalysis as VariantAnalysisApiResponse
+} from './gh-api/variant-analysis';
+import { VariantAnalysisMonitorResult } from './shared/variant-analysis-monitor-result';
+import { processFailureReason } from './variant-analysis-processor';
+
+export class VariantAnalysisMonitor {
+  // With a sleep of 5 seconds, the maximum number of attempts takes
+  // us to just over 2 days worth of monitoring.
+  public static maxAttemptCount = 17280;
+  public static sleepTime = 5000;
+
+  constructor(
+    private readonly extensionContext: vscode.ExtensionContext,
+    private readonly logger: Logger
+  ) {
+  }
+
+  public async monitorVariantAnalysis(
+    variantAnalysis: VariantAnalysis,
+    cancellationToken: vscode.CancellationToken
+  ): Promise<VariantAnalysisMonitorResult> {
+
+    const credentials = await Credentials.initialize(this.extensionContext);
+    if (!credentials) {
+      throw Error('Error authenticating with GitHub');
+    }
+
+    let variantAnalysisSummary: VariantAnalysisApiResponse;
+    let attemptCount = 0;
+    const scannedReposDownloaded: number[] = [];
+
+    while (attemptCount <= VariantAnalysisMonitor.maxAttemptCount) {
+      await this.sleep(VariantAnalysisMonitor.sleepTime);
+
+      if (cancellationToken && cancellationToken.isCancellationRequested) {
+        return { status: 'Cancelled', error: 'Variant Analysis was canceled.' };
+      }
+
+      variantAnalysisSummary = await ghApiClient.getVariantAnalysis(
+        credentials,
+        variantAnalysis.controllerRepoId,
+        variantAnalysis.id
+      );
+
+      if (variantAnalysisSummary.failure_reason) {
+        variantAnalysis.status = VariantAnalysisStatus.Failed;
+        variantAnalysis.failureReason = processFailureReason(variantAnalysisSummary.failure_reason);
+        return {
+          status: 'Failed',
+          error: `Variant Analysis has failed: ${variantAnalysisSummary.failure_reason}`,
+          variantAnalysis: variantAnalysis
+        };
+      }
+
+      void this.logger.log('****** Retrieved variant analysis' + JSON.stringify(variantAnalysisSummary));
+
+      if (variantAnalysisSummary.scanned_repositories) {
+        variantAnalysisSummary.scanned_repositories.forEach(scannedRepo => {
+          if (!scannedReposDownloaded.includes(scannedRepo.repository.id) && scannedRepo.analysis_status === 'succeeded') {
+            scannedReposDownloaded.push(scannedRepo.repository.id);
+          }
+        });
+      }
+
+      if (variantAnalysisSummary.status === 'completed') {
+        break;
+      }
+
+      attemptCount++;
+    }
+
+    return { status: 'CompletedSuccessfully', scannedReposDownloaded: scannedReposDownloaded };
+  }
+
+  private async sleep(ms: number) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}