Skip to content

Commit 2065c7d

Browse files
elenatanasoiuelenatanasoiu
authored
Merge pull request #1545 from github/elenatanasoiu/monitor-variant-analysis
Implement monitoring for variant analysis live results
2 parents afdc816 + ff4ea3e commit 2065c7d

15 files changed

+726
-50
lines changed

‎extensions/ql-vscode/package-lock.json‎

Lines changed: 17 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎extensions/ql-vscode/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1246,6 +1246,7 @@
12461246
"devDependencies": {
12471247
"@babel/core": "^7.18.13",
12481248
"@babel/plugin-transform-modules-commonjs": "^7.18.6",
1249+
"@faker-js/faker": "^7.5.0",
12491250
"@storybook/addon-actions": "^6.5.10",
12501251
"@storybook/addon-essentials": "^6.5.10",
12511252
"@storybook/addon-interactions": "^6.5.10",

‎extensions/ql-vscode/src/extension.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ import { createInitialQueryInfo } from './run-queries-shared';
105105
import { LegacyQueryRunner } from './legacy-query-server/legacyRunner';
106106
import { QueryRunner } from './queryRunner';
107107
import { VariantAnalysisView } from './remote-queries/variant-analysis-view';
108+
import { VariantAnalysisMonitor } from './remote-queries/variant-analysis-monitor';
109+
import { VariantAnalysis } from './remote-queries/shared/variant-analysis';
108110

109111
/**
110112
* extension.ts
@@ -894,6 +896,16 @@ async function activateWithInstalledDistribution(
894896
})
895897
);
896898

899+
const variantAnalysisMonitor = new VariantAnalysisMonitor(ctx, logger);
900+
ctx.subscriptions.push(
901+
commandRunner('codeQL.monitorVariantAnalysis', async (
902+
variantAnalysis: VariantAnalysis,
903+
token: CancellationToken
904+
) => {
905+
await variantAnalysisMonitor.monitorVariantAnalysis(variantAnalysis, token);
906+
})
907+
);
908+
897909
ctx.subscriptions.push(
898910
commandRunner('codeQL.autoDownloadRemoteQueryResults', async (
899911
queryResult: RemoteQueryResult,

‎extensions/ql-vscode/src/remote-queries/gh-api/variant-analysis.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ export interface VariantAnalysisSkippedRepositoryGroup {
6262

6363
export interface VariantAnalysisNotFoundRepositoryGroup {
6464
repository_count: number,
65-
repository_nwos: string[]
65+
repository_full_names: string[]
6666
}
6767
export interface VariantAnalysisRepoTask {
6868
repository: Repository,

‎extensions/ql-vscode/src/remote-queries/run-remote-query.ts‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { CancellationToken, Uri, window } from 'vscode';
1+
import { CancellationToken, commands, Uri, window } from 'vscode';
22
import * as path from 'path';
33
import * as yaml from 'js-yaml';
44
import * as fs from 'fs-extra';
@@ -26,8 +26,9 @@ import { QueryMetadata } from '../pure/interface-types';
2626
import { getErrorMessage, REPO_REGEX } from '../pure/helpers-pure';
2727
import * as ghApiClient from './gh-api/gh-api-client';
2828
import { getRepositorySelection, isValidSelection, RepositorySelection } from './repository-selection';
29-
import { parseVariantAnalysisQueryLanguage, VariantAnalysis, VariantAnalysisStatus, VariantAnalysisSubmission } from './shared/variant-analysis';
29+
import { parseVariantAnalysisQueryLanguage, VariantAnalysisSubmission } from './shared/variant-analysis';
3030
import { Repository } from './shared/repository';
31+
import { processVariantAnalysis } from './variant-analysis-processor';
3132

3233
export interface QlPack {
3334
name: string;
@@ -270,28 +271,15 @@ export async function runRemoteQuery(
270271
variantAnalysisSubmission
271272
);
272273

273-
const variantAnalysis: VariantAnalysis = {
274-
id: variantAnalysisResponse.id,
275-
controllerRepoId: variantAnalysisResponse.controller_repo.id,
276-
query: {
277-
name: variantAnalysisSubmission.query.name,
278-
filePath: variantAnalysisSubmission.query.filePath,
279-
language: variantAnalysisSubmission.query.language,
280-
},
281-
databases: {
282-
repositories: variantAnalysisSubmission.databases.repositories,
283-
repositoryLists: variantAnalysisSubmission.databases.repositoryLists,
284-
repositoryOwners: variantAnalysisSubmission.databases.repositoryOwners,
285-
},
286-
status: VariantAnalysisStatus.InProgress,
287-
};
274+
const processedVariantAnalysis = processVariantAnalysis(variantAnalysisSubmission, variantAnalysisResponse);
288275

289276
// TODO: Remove once we have a proper notification
290277
void showAndLogInformationMessage('Variant analysis submitted for processing');
291-
void logger.log(`Variant analysis:\n${JSON.stringify(variantAnalysis, null, 2)}`);
278+
void logger.log(`Variant analysis:\n${JSON.stringify(processedVariantAnalysis, null, 2)}`);
292279

293-
return { variantAnalysis };
280+
void commands.executeCommand('codeQL.monitorVariantAnalysis', processedVariantAnalysis);
294281

282+
return { variantAnalysis: processedVariantAnalysis };
295283
} else {
296284
const apiResponse = await runRemoteQueriesApiRequest(credentials, actionBranch, language, repoSelection, controllerRepo, base64Pack, dryRun);
297285

‎extensions/ql-vscode/src/remote-queries/shared/variant-analysis-monitor-result.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { VariantAnalysis } from './variant-analysis';
2+
3+
export type VariantAnalysisMonitorStatus =
4+
| 'InProgress'
5+
| 'CompletedSuccessfully'
6+
| 'CompletedUnsuccessfully'
7+
| 'Failed'
8+
| 'Cancelled'
9+
| 'TimedOut';
10+
11+
export interface VariantAnalysisMonitorResult {
12+
status: VariantAnalysisMonitorStatus;
13+
error?: string;
14+
scannedReposDownloaded?: number[],
15+
variantAnalysis?: VariantAnalysis
16+
}

‎extensions/ql-vscode/src/remote-queries/variant-analysis-monitor.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import * as vscode from 'vscode';
2+
import { Credentials } from '../authentication';
3+
import { Logger } from '../logging';
4+
import * as ghApiClient from './gh-api/gh-api-client';
5+
6+
import { VariantAnalysis, VariantAnalysisStatus } from './shared/variant-analysis';
7+
import {
8+
VariantAnalysis as VariantAnalysisApiResponse
9+
} from './gh-api/variant-analysis';
10+
import { VariantAnalysisMonitorResult } from './shared/variant-analysis-monitor-result';
11+
import { processFailureReason } from './variant-analysis-processor';
12+
13+
export class VariantAnalysisMonitor {
14+
// With a sleep of 5 seconds, the maximum number of attempts takes
15+
// us to just over 2 days worth of monitoring.
16+
public static maxAttemptCount = 17280;
17+
public static sleepTime = 5000;
18+
19+
constructor(
20+
private readonly extensionContext: vscode.ExtensionContext,
21+
private readonly logger: Logger
22+
) {
23+
}
24+
25+
public async monitorVariantAnalysis(
26+
variantAnalysis: VariantAnalysis,
27+
cancellationToken: vscode.CancellationToken
28+
): Promise<VariantAnalysisMonitorResult> {
29+
30+
const credentials = await Credentials.initialize(this.extensionContext);
31+
if (!credentials) {
32+
throw Error('Error authenticating with GitHub');
33+
}
34+
35+
let variantAnalysisSummary: VariantAnalysisApiResponse;
36+
let attemptCount = 0;
37+
const scannedReposDownloaded: number[] = [];
38+
39+
while (attemptCount <= VariantAnalysisMonitor.maxAttemptCount) {
40+
await this.sleep(VariantAnalysisMonitor.sleepTime);
41+
42+
if (cancellationToken && cancellationToken.isCancellationRequested) {
43+
return { status: 'Cancelled', error: 'Variant Analysis was canceled.' };
44+
}
45+
46+
variantAnalysisSummary = await ghApiClient.getVariantAnalysis(
47+
credentials,
48+
variantAnalysis.controllerRepoId,
49+
variantAnalysis.id
50+
);
51+
52+
if (variantAnalysisSummary.failure_reason) {
53+
variantAnalysis.status = VariantAnalysisStatus.Failed;
54+
variantAnalysis.failureReason = processFailureReason(variantAnalysisSummary.failure_reason);
55+
return {
56+
status: 'Failed',
57+
error: `Variant Analysis has failed: ${variantAnalysisSummary.failure_reason}`,
58+
variantAnalysis: variantAnalysis
59+
};
60+
}
61+
62+
void this.logger.log('****** Retrieved variant analysis' + JSON.stringify(variantAnalysisSummary));
63+
64+
if (variantAnalysisSummary.scanned_repositories) {
65+
variantAnalysisSummary.scanned_repositories.forEach(scannedRepo => {
66+
if (!scannedReposDownloaded.includes(scannedRepo.repository.id) && scannedRepo.analysis_status === 'succeeded') {
67+
scannedReposDownloaded.push(scannedRepo.repository.id);
68+
}
69+
});
70+
}
71+
72+
if (variantAnalysisSummary.status === 'completed') {
73+
break;
74+
}
75+
76+
attemptCount++;
77+
}
78+
79+
return { status: 'CompletedSuccessfully', scannedReposDownloaded: scannedReposDownloaded };
80+
}
81+
82+
private async sleep(ms: number) {
83+
return new Promise(resolve => setTimeout(resolve, ms));
84+
}
85+
}

‎extensions/ql-vscode/src/remote-queries/variant-analysis-processor.ts‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import {
2+
VariantAnalysis as ApiVariantAnalysis,
3+
VariantAnalysisScannedRepository as ApiVariantAnalysisScannedRepository,
4+
VariantAnalysisSkippedRepositories as ApiVariantAnalysisSkippedRepositories,
5+
VariantAnalysisRepoStatus as ApiVariantAnalysisRepoStatus,
6+
VariantAnalysisFailureReason as ApiVariantAnalysisFailureReason,
7+
VariantAnalysisStatus as ApiVariantAnalysisStatus,
8+
VariantAnalysisSkippedRepositoryGroup as ApiVariantAnalysisSkippedRepositoryGroup,
9+
VariantAnalysisNotFoundRepositoryGroup as ApiVariantAnalysisNotFoundRepositoryGroup
10+
} from './gh-api/variant-analysis';
11+
import {
12+
VariantAnalysis,
13+
VariantAnalysisFailureReason,
14+
VariantAnalysisScannedRepository,
15+
VariantAnalysisSkippedRepositories,
16+
VariantAnalysisStatus,
17+
VariantAnalysisRepoStatus,
18+
VariantAnalysisSubmission,
19+
VariantAnalysisSkippedRepositoryGroup
20+
} from './shared/variant-analysis';
21+
22+
export function processVariantAnalysis(
23+
submission: VariantAnalysisSubmission,
24+
response: ApiVariantAnalysis
25+
): VariantAnalysis {
26+
27+
let scannedRepos: VariantAnalysisScannedRepository[] = [];
28+
let skippedRepos: VariantAnalysisSkippedRepositories = {};
29+
30+
if (response.scanned_repositories) {
31+
scannedRepos = processScannedRepositories(response.scanned_repositories as ApiVariantAnalysisScannedRepository[]);
32+
}
33+
34+
if (response.skipped_repositories) {
35+
skippedRepos = processSkippedRepositories(response.skipped_repositories as ApiVariantAnalysisSkippedRepositories);
36+
}
37+
38+
const variantAnalysis: VariantAnalysis = {
39+
id: response.id,
40+
controllerRepoId: response.controller_repo.id,
41+
query: {
42+
name: submission.query.name,
43+
filePath: submission.query.filePath,
44+
language: submission.query.language
45+
},
46+
databases: submission.databases,
47+
status: processApiStatus(response.status),
48+
actionsWorkflowRunId: response.actions_workflow_run_id,
49+
scannedRepos: scannedRepos,
50+
skippedRepos: skippedRepos
51+
};
52+
53+
if (response.failure_reason) {
54+
variantAnalysis.failureReason = processFailureReason(response.failure_reason);
55+
}
56+
57+
return variantAnalysis;
58+
}
59+
60+
function processScannedRepositories(
61+
scannedRepos: ApiVariantAnalysisScannedRepository[]
62+
): VariantAnalysisScannedRepository[] {
63+
return scannedRepos.map(scannedRepo => {
64+
return {
65+
repository: {
66+
id: scannedRepo.repository.id,
67+
fullName: scannedRepo.repository.full_name,
68+
private: scannedRepo.repository.private,
69+
},
70+
analysisStatus: processApiRepoStatus(scannedRepo.analysis_status),
71+
resultCount: scannedRepo.result_count,
72+
artifactSizeInBytes: scannedRepo.artifact_size_in_bytes,
73+
failureMessage: scannedRepo.failure_message
74+
};
75+
});
76+
}
77+
78+
function processSkippedRepositories(
79+
skippedRepos: ApiVariantAnalysisSkippedRepositories
80+
): VariantAnalysisSkippedRepositories {
81+
82+
return {
83+
accessMismatchRepos: processRepoGroup(skippedRepos.access_mismatch_repos),
84+
notFoundRepos: processNotFoundRepoGroup(skippedRepos.not_found_repo_nwos),
85+
noCodeqlDbRepos: processRepoGroup(skippedRepos.no_codeql_db_repos),
86+
overLimitRepos: processRepoGroup(skippedRepos.over_limit_repos)
87+
};
88+
}
89+
90+
function processRepoGroup(repoGroup: ApiVariantAnalysisSkippedRepositoryGroup): VariantAnalysisSkippedRepositoryGroup {
91+
const repos = repoGroup.repositories.map(repo => {
92+
return {
93+
id: repo.id,
94+
fullName: repo.full_name
95+
};
96+
});
97+
98+
return {
99+
repositoryCount: repoGroup.repository_count,
100+
repositories: repos
101+
};
102+
}
103+
104+
function processNotFoundRepoGroup(repoGroup: ApiVariantAnalysisNotFoundRepositoryGroup): VariantAnalysisSkippedRepositoryGroup {
105+
const repo_full_names = repoGroup.repository_full_names.map(nwo => {
106+
return {
107+
fullName: nwo
108+
};
109+
});
110+
111+
return {
112+
repositoryCount: repoGroup.repository_count,
113+
repositories: repo_full_names
114+
};
115+
}
116+
117+
function processApiRepoStatus(analysisStatus: ApiVariantAnalysisRepoStatus): VariantAnalysisRepoStatus {
118+
switch (analysisStatus) {
119+
case 'pending':
120+
return VariantAnalysisRepoStatus.Pending;
121+
case 'in_progress':
122+
return VariantAnalysisRepoStatus.InProgress;
123+
case 'succeeded':
124+
return VariantAnalysisRepoStatus.Succeeded;
125+
case 'failed':
126+
return VariantAnalysisRepoStatus.Failed;
127+
case 'canceled':
128+
return VariantAnalysisRepoStatus.Canceled;
129+
case 'timed_out':
130+
return VariantAnalysisRepoStatus.TimedOut;
131+
}
132+
}
133+
134+
function processApiStatus(status: ApiVariantAnalysisStatus): VariantAnalysisStatus {
135+
switch (status) {
136+
case 'in_progress':
137+
return VariantAnalysisStatus.InProgress;
138+
case 'completed':
139+
return VariantAnalysisStatus.Succeeded;
140+
}
141+
}
142+
143+
export function processFailureReason(failureReason: ApiVariantAnalysisFailureReason): VariantAnalysisFailureReason {
144+
switch (failureReason) {
145+
case 'no_repos_queried':
146+
return VariantAnalysisFailureReason.NoReposQueried;
147+
case 'internal_error':
148+
return VariantAnalysisFailureReason.InternalError;
149+
}
150+
}

0 commit comments

Comments
 (0)