Skip to content

Commit a20d910

Browse files
koesie10koesie10
committed
Add variant analysis results manager
This adds a new variant analysis results manager which is responsible for downloading and loading variant analysis results to/from the filesystem. It is essentially the `AnalysesResultsManager` modified to suit the variant analysis results.
1 parent 131d252 commit a20d910

File tree

9 files changed

+292
-24
lines changed

9 files changed

+292
-24
lines changed

‎extensions/ql-vscode/src/extension.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,10 +457,16 @@ async function activateWithInstalledDistribution(
457457
const localQueryResultsView = new ResultsView(ctx, dbm, cliServer, queryServerLogger, labelProvider);
458458
ctx.subscriptions.push(localQueryResultsView);
459459

460-
void logger.log('Initializing variant analysis manager.');
460+
void logger.log('Initializing remote queries manager.');
461461
const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger);
462462
ctx.subscriptions.push(rqm);
463463

464+
void logger.log('Initializing variant analysis manager.');
465+
const variantAnalysisStorageDir = path.join(ctx.globalStorageUri.fsPath, 'variant-analyses');
466+
await fs.ensureDir(variantAnalysisStorageDir);
467+
const variantAnalysisManager = new VariantAnalysisManager(ctx, cliServer, variantAnalysisStorageDir, logger);
468+
ctx.subscriptions.push(variantAnalysisManager);
469+
464470
void logger.log('Initializing query history.');
465471
const qhm = new QueryHistoryManager(
466472
qs,
@@ -899,7 +905,6 @@ async function activateWithInstalledDistribution(
899905
})
900906
);
901907

902-
const variantAnalysisManager = new VariantAnalysisManager(ctx, logger);
903908
ctx.subscriptions.push(
904909
commandRunner('codeQL.monitorVariantAnalysis', async (
905910
variantAnalysis: VariantAnalysis,

‎extensions/ql-vscode/src/remote-queries/shared/variant-analysis.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ export interface VariantAnalysisScannedRepositoryState {
9595
}
9696

9797
export interface VariantAnalysisScannedRepositoryResult {
98+
variantAnalysisId: number;
9899
repositoryId: number;
99100
interpretedResults?: AnalysisAlert[];
100101
rawResults?: AnalysisRawResults;

‎extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
import * as ghApiClient from './gh-api/gh-api-client';
2-
import * as path from 'path';
3-
import * as fs from 'fs-extra';
42
import { CancellationToken, ExtensionContext } from 'vscode';
53
import { DisposableObject } from '../pure/disposable-object';
64
import { Logger } from '../logging';
@@ -19,18 +17,25 @@ import {
1917
import { getErrorMessage } from '../pure/helpers-pure';
2018
import { VariantAnalysisView } from './variant-analysis-view';
2119
import { VariantAnalysisViewManager } from './variant-analysis-view-manager';
20+
import { VariantAnalysisResultsManager } from './variant-analysis-results-manager';
21+
import { CodeQLCliServer } from '../cli';
2222

2323
export class VariantAnalysisManager extends DisposableObject implements VariantAnalysisViewManager<VariantAnalysisView> {
2424
private readonly variantAnalysisMonitor: VariantAnalysisMonitor;
25+
private readonly variantAnalysisResultsManager: VariantAnalysisResultsManager;
2526
private readonly views = new Map<number, VariantAnalysisView>();
2627

2728
constructor(
2829
private readonly ctx: ExtensionContext,
30+
cliServer: CodeQLCliServer,
31+
storagePath: string,
2932
logger: Logger,
3033
) {
3134
super();
3235
this.variantAnalysisMonitor = this.push(new VariantAnalysisMonitor(ctx, logger));
3336
this.variantAnalysisMonitor.onVariantAnalysisChange(this.onVariantAnalysisUpdated.bind(this));
37+
38+
this.variantAnalysisResultsManager = this.push(new VariantAnalysisResultsManager(cliServer, storagePath, logger));
3439
}
3540

3641
public async showView(variantAnalysisId: number): Promise<void> {
@@ -118,25 +123,7 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
118123
repoState.downloadStatus = VariantAnalysisScannedRepositoryDownloadStatus.InProgress;
119124
await this.onRepoStateUpdated(variantAnalysisSummary.id, repoState);
120125

121-
const resultDirectory = path.join(
122-
this.ctx.globalStorageUri.fsPath,
123-
'variant-analyses',
124-
`${variantAnalysisSummary.id}`,
125-
scannedRepo.repository.full_name
126-
);
127-
128-
const storagePath = path.join(
129-
resultDirectory,
130-
scannedRepo.repository.full_name
131-
);
132-
133-
const result = await ghApiClient.getVariantAnalysisRepoResult(
134-
credentials,
135-
repoTask.artifact_url
136-
);
137-
138-
fs.mkdirSync(resultDirectory, { recursive: true });
139-
await fs.writeFile(storagePath, JSON.stringify(result, null, 2), 'utf8');
126+
await this.variantAnalysisResultsManager.download(credentials, variantAnalysisSummary.id, repoTask);
140127
}
141128

142129
repoState.downloadStatus = VariantAnalysisScannedRepositoryDownloadStatus.Succeeded;

‎extensions/ql-vscode/src/remote-queries/variant-analysis-results-manager.ts‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import * as fs from 'fs-extra';
2+
import * as os from 'os';
3+
import * as path from 'path';
4+
5+
import { Credentials } from '../authentication';
6+
import { Logger } from '../logging';
7+
import { AnalysisAlert, AnalysisRawResults } from './shared/analysis-result';
8+
import { sarifParser } from '../sarif-parser';
9+
import { extractAnalysisAlerts } from './sarif-processing';
10+
import { CodeQLCliServer } from '../cli';
11+
import { extractRawResults } from './bqrs-processing';
12+
import { VariantAnalysisScannedRepositoryResult } from './shared/variant-analysis';
13+
import { DisposableObject, DisposeHandler } from '../pure/disposable-object';
14+
import { VariantAnalysisRepoTask } from './gh-api/variant-analysis';
15+
import * as ghApiClient from './gh-api/gh-api-client';
16+
import { EventEmitter } from 'vscode';
17+
18+
type CacheKey = `${number}/${string}`;
19+
20+
const createCacheKey = (variantAnalysisId: number, repoTask: VariantAnalysisRepoTask): CacheKey => `${variantAnalysisId}/${repoTask.repository.full_name}`;
21+
22+
export type ResultDownloadedEvent = {
23+
variantAnalysisId: number;
24+
repoTask: VariantAnalysisRepoTask;
25+
}
26+
27+
export class VariantAnalysisResultsManager extends DisposableObject {
28+
private readonly cachedResults: Map<CacheKey, VariantAnalysisScannedRepositoryResult>;
29+
30+
private readonly _onResultDownloaded = this.push(new EventEmitter<ResultDownloadedEvent>());
31+
readonly onResultDownloaded = this._onResultDownloaded.event;
32+
33+
private readonly _onResultLoaded = this.push(new EventEmitter<VariantAnalysisScannedRepositoryResult>());
34+
readonly onResultLoaded = this._onResultLoaded.event;
35+
36+
constructor(
37+
private readonly cliServer: CodeQLCliServer,
38+
private readonly storagePath: string,
39+
private readonly logger: Logger,
40+
) {
41+
super();
42+
this.cachedResults = new Map();
43+
}
44+
45+
public async download(
46+
credentials: Credentials,
47+
variantAnalysisId: number,
48+
repoTask: VariantAnalysisRepoTask,
49+
): Promise<void> {
50+
if (!repoTask.artifact_url) {
51+
throw new Error('Missing artifact URL');
52+
}
53+
54+
const resultDirectory = this.getRepoStorageDirectory(variantAnalysisId, repoTask.repository.full_name);
55+
56+
const result = await ghApiClient.getVariantAnalysisRepoResult(
57+
credentials,
58+
repoTask.artifact_url
59+
);
60+
61+
fs.mkdirSync(resultDirectory, { recursive: true });
62+
await fs.writeFile(path.join(resultDirectory, 'results.zip'), JSON.stringify(result, null, 2), 'utf8');
63+
64+
this._onResultDownloaded.fire({
65+
variantAnalysisId,
66+
repoTask,
67+
});
68+
}
69+
70+
public async loadResults(
71+
variantAnalysisId: number,
72+
repoTask: VariantAnalysisRepoTask
73+
): Promise<VariantAnalysisScannedRepositoryResult> {
74+
const result = this.cachedResults.get(createCacheKey(variantAnalysisId, repoTask));
75+
if (result) {
76+
return result;
77+
}
78+
79+
return this.loadResultsIntoMemory(variantAnalysisId, repoTask);
80+
}
81+
82+
private async loadResultsIntoMemory(
83+
variantAnalysisId: number,
84+
repoTask: VariantAnalysisRepoTask,
85+
): Promise<VariantAnalysisScannedRepositoryResult> {
86+
const result = await this.loadResultsFromStorage(variantAnalysisId, repoTask);
87+
this.cachedResults.set(createCacheKey(variantAnalysisId, repoTask), result);
88+
this._onResultLoaded.fire(result);
89+
return result;
90+
}
91+
92+
private async loadResultsFromStorage(
93+
variantAnalysisId: number,
94+
repoTask: VariantAnalysisRepoTask,
95+
): Promise<VariantAnalysisScannedRepositoryResult> {
96+
if (!(await this.isVariantAnalysisRepoDownloaded(variantAnalysisId, repoTask))) {
97+
throw new Error('Variant analysis results not downloaded');
98+
}
99+
100+
if (!repoTask.database_commit_sha || !repoTask.source_location_prefix) {
101+
throw new Error('Missing database commit SHA');
102+
}
103+
104+
const fileLinkPrefix = this.createGitHubDotcomFileLinkPrefix(repoTask.repository.full_name, repoTask.database_commit_sha);
105+
106+
const storageDirectory = this.getRepoStorageDirectory(variantAnalysisId, repoTask.repository.full_name);
107+
108+
const sarifPath = path.join(storageDirectory, 'results.sarif');
109+
const bqrsPath = path.join(storageDirectory, 'results.bqrs');
110+
if (await fs.pathExists(sarifPath)) {
111+
const interpretedResults = await this.readSarifResults(sarifPath, fileLinkPrefix);
112+
113+
return {
114+
variantAnalysisId,
115+
repositoryId: repoTask.repository.id,
116+
interpretedResults,
117+
};
118+
}
119+
120+
if (await fs.pathExists(bqrsPath)) {
121+
const rawResults = await this.readBqrsResults(bqrsPath, fileLinkPrefix, repoTask.source_location_prefix);
122+
123+
return {
124+
variantAnalysisId,
125+
repositoryId: repoTask.repository.id,
126+
rawResults,
127+
};
128+
}
129+
130+
throw new Error('Missing results file');
131+
}
132+
133+
private async isVariantAnalysisRepoDownloaded(
134+
variantAnalysisId: number,
135+
repoTask: VariantAnalysisRepoTask,
136+
): Promise<boolean> {
137+
return await fs.pathExists(this.getRepoStorageDirectory(variantAnalysisId, repoTask.repository.full_name));
138+
}
139+
140+
private async readBqrsResults(filePath: string, fileLinkPrefix: string, sourceLocationPrefix: string): Promise<AnalysisRawResults> {
141+
return await extractRawResults(this.cliServer, this.logger, filePath, fileLinkPrefix, sourceLocationPrefix);
142+
}
143+
144+
private async readSarifResults(filePath: string, fileLinkPrefix: string): Promise<AnalysisAlert[]> {
145+
const sarifLog = await sarifParser(filePath);
146+
147+
const processedSarif = extractAnalysisAlerts(sarifLog, fileLinkPrefix);
148+
if (processedSarif.errors.length) {
149+
void this.logger.log(`Error processing SARIF file: ${os.EOL}${processedSarif.errors.join(os.EOL)}`);
150+
}
151+
152+
return processedSarif.alerts;
153+
}
154+
155+
private getStorageDirectory(variantAnalysisId: number): string {
156+
return path.join(
157+
this.storagePath,
158+
`${variantAnalysisId}`
159+
);
160+
}
161+
162+
private getRepoStorageDirectory(variantAnalysisId: number, fullName: string): string {
163+
return path.join(
164+
this.getStorageDirectory(variantAnalysisId),
165+
fullName
166+
);
167+
}
168+
169+
private createGitHubDotcomFileLinkPrefix(fullName: string, sha: string): string {
170+
return `https://github.com/${fullName}/blob/${sha}`;
171+
}
172+
173+
public dispose(disposeHandler?: DisposeHandler) {
174+
super.dispose(disposeHandler);
175+
176+
this.cachedResults.clear();
177+
}
178+
}

‎extensions/ql-vscode/src/stories/variant-analysis/VariantAnalysisAnalyzedRepos.stories.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,17 @@ Example.args = {
100100
},
101101
repositoryResults: [
102102
{
103+
variantAnalysisId: 1,
103104
repositoryId: 63537249,
104105
interpretedResults: interpretedResultsForRepo('facebook/create-react-app'),
105106
},
106107
{
108+
variantAnalysisId: 1,
107109
repositoryId: 167174,
108110
interpretedResults: interpretedResultsForRepo('jquery/jquery'),
109111
},
110112
{
113+
variantAnalysisId: 1,
111114
repositoryId: 237159,
112115
interpretedResults: interpretedResultsForRepo('expressjs/express'),
113116
}

‎extensions/ql-vscode/src/view/variant-analysis/VariantAnalysis.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ const variantAnalysis: VariantAnalysisDomainModel = {
165165

166166
const repositoryResults: VariantAnalysisScannedRepositoryResult[] = [
167167
{
168+
variantAnalysisId: 1,
168169
repositoryId: 1,
169170
rawResults: {
170171
schema: {

‎extensions/ql-vscode/src/view/variant-analysis/__tests__/VariantAnalysisAnalyzedRepos.spec.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ describe(VariantAnalysisAnalyzedRepos.name, () => {
7878
render({
7979
repositoryResults: [
8080
{
81+
variantAnalysisId: 1,
8182
repositoryId: 2,
8283
interpretedResults: [
8384
{

‎extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,12 @@ import {
1616
import { createMockApiResponse } from '../../factories/remote-queries/gh-api/variant-analysis-api-response';
1717
import { createMockScannedRepos } from '../../factories/remote-queries/gh-api/scanned-repositories';
1818
import { createMockVariantAnalysisRepoTask } from '../../factories/remote-queries/gh-api/variant-analysis-repo-task';
19+
import { CodeQLCliServer } from '../../../cli';
20+
import { storagePath } from '../global.helper';
1921

2022
describe('Variant Analysis Manager', async function() {
2123
let sandbox: sinon.SinonSandbox;
24+
let cli: CodeQLCliServer;
2225
let cancellationTokenSource: CancellationTokenSource;
2326
let variantAnalysisManager: VariantAnalysisManager;
2427
let variantAnalysis: VariantAnalysisApiResponse;
@@ -47,7 +50,8 @@ describe('Variant Analysis Manager', async function() {
4750

4851
try {
4952
const extension = await extensions.getExtension<CodeQLExtensionInterface | Record<string, never>>('GitHub.vscode-codeql')!.activate();
50-
variantAnalysisManager = new VariantAnalysisManager(extension.ctx, logger);
53+
cli = extension.cliServer;
54+
variantAnalysisManager = new VariantAnalysisManager(extension.ctx, cli, storagePath, logger);
5155
} catch (e) {
5256
fail(e as Error);
5357
}

0 commit comments

Comments
 (0)