Merge pull request #2014 from github/koesie10/download-progress

Report download progress when downloading variant analysis results

Author: koesie10

extensions/ql-vscode/src/extension.ts

@@ -633,7 +633,6 @@ async function activateWithInstalledDistribution(
   );
   await ensureDir(variantAnalysisStorageDir);
   const variantAnalysisResultsManager = new VariantAnalysisResultsManager(
-    app.credentials,
     cliServer,
     extLogger,
   );

extensions/ql-vscode/src/remote-queries/gh-api/gh-api-client.ts

@@ -81,16 +81,6 @@ export async function getVariantAnalysisRepo(
   return response.data;
 }
 
-export async function getVariantAnalysisRepoResult(
-  credentials: Credentials,
-  downloadUrl: string,
-): Promise<ArrayBuffer> {
-  const octokit = await credentials.getOctokit();
-  const response = await octokit.request(`GET ${downloadUrl}`);
-
-  return response.data;
-}
-
 export async function getRepositoryFromNwo(
   credentials: Credentials,
   owner: string,

extensions/ql-vscode/src/remote-queries/shared/variant-analysis.ts

@@ -127,6 +127,7 @@ export enum VariantAnalysisScannedRepositoryDownloadStatus {
 export interface VariantAnalysisScannedRepositoryState {
   repositoryId: number;
   downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus;
+  downloadPercentage?: number;
 }
 
 export interface VariantAnalysisScannedRepositoryResult {

extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts

@@ -68,6 +68,7 @@ export class VariantAnalysisManager
   implements VariantAnalysisViewManager<VariantAnalysisView>
 {
   private static readonly REPO_STATES_FILENAME = "repo_states.json";
+  private static readonly DOWNLOAD_PERCENTAGE_UPDATE_DELAY_MS = 3000;
 
   private readonly _onVariantAnalysisAdded = this.push(
     new EventEmitter<VariantAnalysis>(),
@@ -514,10 +515,27 @@ export class VariantAnalysisManager
       await this.onRepoStateUpdated(variantAnalysis.id, repoState);
 
       try {
+        let lastRepoStateUpdate = 0;
+        const updateRepoStateCallback = async (downloadPercentage: number) => {
+          const now = new Date().getTime();
+          if (
+            lastRepoStateUpdate <
+            now - VariantAnalysisManager.DOWNLOAD_PERCENTAGE_UPDATE_DELAY_MS
+          ) {
+            lastRepoStateUpdate = now;
+            await this.onRepoStateUpdated(variantAnalysis.id, {
+              repositoryId: scannedRepo.repository.id,
+              downloadStatus:
+                VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
+              downloadPercentage,
+            });
+          }
+        };
         await this.variantAnalysisResultsManager.download(
           variantAnalysis.id,
           repoTask,
           this.getVariantAnalysisStorageLocation(variantAnalysis.id),
+          updateRepoStateCallback,
         );
       } catch (e) {
         repoState.downloadStatus =

extensions/ql-vscode/src/remote-queries/variant-analysis-results-manager.ts

@@ -1,14 +1,8 @@
-import {
-  pathExists,
-  mkdir,
-  outputJson,
-  writeFileSync,
-  readJson,
-} from "fs-extra";
+import { appendFile, pathExists, mkdir, outputJson, readJson } from "fs-extra";
+import fetch from "node-fetch";
 import { EOL } from "os";
 import { join } from "path";
 
-import { Credentials } from "../common/authentication";
 import { Logger } from "../common";
 import { AnalysisAlert, AnalysisRawResults } from "./shared/analysis-result";
 import { sarifParser } from "../sarif-parser";
@@ -21,7 +15,6 @@ import {
   VariantAnalysisScannedRepositoryResult,
 } from "./shared/variant-analysis";
 import { DisposableObject, DisposeHandler } from "../pure/disposable-object";
-import { getVariantAnalysisRepoResult } from "./gh-api/gh-api-client";
 import { EventEmitter } from "vscode";
 import { unzipFile } from "../pure/zip";
 
@@ -63,7 +56,6 @@ export class VariantAnalysisResultsManager extends DisposableObject {
   readonly onResultLoaded = this._onResultLoaded.event;
 
   constructor(
-    private readonly credentials: Credentials,
     private readonly cliServer: CodeQLCliServer,
     private readonly logger: Logger,
   ) {
@@ -75,6 +67,7 @@ export class VariantAnalysisResultsManager extends DisposableObject {
     variantAnalysisId: number,
     repoTask: VariantAnalysisRepositoryTask,
     variantAnalysisStoragePath: string,
+    onDownloadPercentageChanged: (downloadPercentage: number) => Promise<void>,
   ): Promise<void> {
     if (!repoTask.artifactUrl) {
       throw new Error("Missing artifact URL");
@@ -85,11 +78,6 @@ export class VariantAnalysisResultsManager extends DisposableObject {
       repoTask.repository.fullName,
     );
 
-    const result = await getVariantAnalysisRepoResult(
-      this.credentials,
-      repoTask.artifactUrl,
-    );
-
     if (!(await pathExists(resultDirectory))) {
       await mkdir(resultDirectory, { recursive: true });
     }
@@ -100,12 +88,22 @@ export class VariantAnalysisResultsManager extends DisposableObject {
     );
 
     const zipFilePath = join(resultDirectory, "results.zip");
+
+    const response = await fetch(repoTask.artifactUrl);
+    let amountDownloaded = 0;
+    for await (const chunk of response.body) {
+      await appendFile(zipFilePath, Buffer.from(chunk));
+      amountDownloaded += chunk.length;
+      await onDownloadPercentageChanged(
+        Math.floor((amountDownloaded / response.size) * 100),
+      );
+    }
+
     const unzippedFilesDirectory = join(
       resultDirectory,
       VariantAnalysisResultsManager.RESULTS_DIRECTORY,
     );
 
-    writeFileSync(zipFilePath, Buffer.from(result));
     await unzipFile(zipFilePath, unzippedFilesDirectory);
 
     this._onResultDownloaded.fire({

extensions/ql-vscode/src/stories/variant-analysis/RepoRow.stories.tsx

@@ -15,7 +15,7 @@ import { createMockRepositoryWithMetadata } from "../../../test/factories/remote
 
 import * as analysesResults from "../remote-queries/data/analysesResultsMessage.json";
 import * as rawResults from "../remote-queries/data/rawResults.json";
-import { RepoRow } from "../../view/variant-analysis/RepoRow";
+import { RepoRow, RepoRowProps } from "../../view/variant-analysis/RepoRow";
 
 export default {
   title: "Variant Analysis/Repo Row",
@@ -29,7 +29,7 @@ export default {
   ],
 } as ComponentMeta<typeof RepoRow>;
 
-const Template: ComponentStory<typeof RepoRow> = (args) => (
+const Template: ComponentStory<typeof RepoRow> = (args: RepoRowProps) => (
   <RepoRow {...args} />
 );
 
@@ -77,23 +77,44 @@ SucceededDownloading.args = {
   ...Pending.args,
   status: VariantAnalysisRepoStatus.Succeeded,
   resultCount: 198,
-  downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
+  downloadState: {
+    repositoryId: 63537249,
+    downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
+  },
+};
+
+export const SucceededDownloadingWithPercentage = Template.bind({});
+SucceededDownloadingWithPercentage.args = {
+  ...Pending.args,
+  status: VariantAnalysisRepoStatus.Succeeded,
+  resultCount: 198,
+  downloadState: {
+    repositoryId: 63537249,
+    downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.InProgress,
+    downloadPercentage: 42,
+  },
 };
 
 export const SucceededSuccessfulDownload = Template.bind({});
 SucceededSuccessfulDownload.args = {
   ...Pending.args,
   status: VariantAnalysisRepoStatus.Succeeded,
   resultCount: 198,
-  downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
+  downloadState: {
+    repositoryId: 63537249,
+    downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Succeeded,
+  },
 };
 
 export const SucceededFailedDownload = Template.bind({});
 SucceededFailedDownload.args = {
   ...Pending.args,
   status: VariantAnalysisRepoStatus.Succeeded,
   resultCount: 198,
-  downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Failed,
+  downloadState: {
+    repositoryId: 63537249,
+    downloadStatus: VariantAnalysisScannedRepositoryDownloadStatus.Failed,
+  },
 };
 
 export const InterpretedResults = Template.bind({});

extensions/ql-vscode/src/view/common/icon/Codicon.tsx

@@ -17,6 +17,7 @@ export const Codicon = ({ name, label, className, slot }: Props) => (
   <CodiconIcon
     role="img"
     aria-label={label}
+    title={label}
     className={classNames("codicon", `codicon-${name}`, className)}
     slot={slot}
   />

extensions/ql-vscode/src/view/variant-analysis/RepoRow.tsx

@@ -6,6 +6,7 @@ import {
   isCompletedAnalysisRepoStatus,
   VariantAnalysisRepoStatus,
   VariantAnalysisScannedRepositoryDownloadStatus,
+  VariantAnalysisScannedRepositoryState,
 } from "../../remote-queries/shared/variant-analysis";
 import { formatDecimal } from "../../pure/number";
 import {
@@ -91,7 +92,7 @@ export type RepoRowProps = {
   repository: Partial<RepositoryWithMetadata> &
     Pick<RepositoryWithMetadata, "fullName">;
   status?: VariantAnalysisRepoStatus;
-  downloadStatus?: VariantAnalysisScannedRepositoryDownloadStatus;
+  downloadState?: VariantAnalysisScannedRepositoryState;
   resultCount?: number;
 
   interpretedResults?: AnalysisAlert[];
@@ -163,7 +164,7 @@ const filterRepoRowExpandedTelemetry = (v: boolean) => v;
 export const RepoRow = ({
   repository,
   status,
-  downloadStatus,
+  downloadState,
   resultCount,
   interpretedResults,
   rawResults,
@@ -185,7 +186,7 @@ export const RepoRow = ({
     if (
       resultsLoaded ||
       status !== VariantAnalysisRepoStatus.Succeeded ||
-      downloadStatus !==
+      downloadState?.downloadStatus !==
         VariantAnalysisScannedRepositoryDownloadStatus.Succeeded
     ) {
       setExpanded((oldIsExpanded) => !oldIsExpanded);
@@ -203,7 +204,7 @@ export const RepoRow = ({
     resultsLoaded,
     repository.fullName,
     status,
-    downloadStatus,
+    downloadState,
     setExpanded,
   ]);
 
@@ -234,10 +235,11 @@ export const RepoRow = ({
     [onSelectedChange, repository],
   );
 
-  const disabled = !canExpand(status, downloadStatus) || resultsLoading;
+  const disabled =
+    !canExpand(status, downloadState?.downloadStatus) || resultsLoading;
   const expandableContentLoaded = isExpandableContentLoaded(
     status,
-    downloadStatus,
+    downloadState?.downloadStatus,
     resultsLoaded,
   );
 
@@ -252,7 +254,9 @@ export const RepoRow = ({
           onChange={onChangeCheckbox}
           onClick={onClickCheckbox}
           checked={selected}
-          disabled={!repository.id || !canSelect(status, downloadStatus)}
+          disabled={
+            !repository.id || !canSelect(status, downloadState?.downloadStatus)
+          }
         />
         {isExpanded && (
           <ExpandCollapseCodicon name="chevron-down" label="Collapse" />
@@ -278,11 +282,17 @@ export const RepoRow = ({
           )}
           {!status && <WarningIcon />}
         </span>
-        {downloadStatus ===
+        {downloadState?.downloadStatus ===
           VariantAnalysisScannedRepositoryDownloadStatus.InProgress && (
-          <LoadingIcon label="Downloading" />
+          <LoadingIcon
+            label={
+              downloadState.downloadPercentage !== undefined
+                ? `Downloading: ${downloadState.downloadPercentage}%`
+                : "Downloading"
+            }
+          />
         )}
-        {downloadStatus ===
+        {downloadState?.downloadStatus ===
           VariantAnalysisScannedRepositoryDownloadStatus.Failed && (
           <WarningIcon label="Failed to download the results" />
         )}
@@ -296,7 +306,7 @@ export const RepoRow = ({
       {isExpanded && expandableContentLoaded && (
         <AnalyzedRepoItemContent
           status={status}
-          downloadStatus={downloadStatus}
+          downloadStatus={downloadState?.downloadStatus}
           interpretedResults={interpretedResults}
           rawResults={rawResults}
         />

extensions/ql-vscode/src/view/variant-analysis/VariantAnalysisAnalyzedRepos.tsx

@@ -89,7 +89,7 @@ export const VariantAnalysisAnalyzedRepos = ({
             key={repository.repository.id}
             repository={repository.repository}
             status={repository.analysisStatus}
-            downloadStatus={state?.downloadStatus}
+            downloadState={state}
             resultCount={repository.resultCount}
             interpretedResults={results?.interpretedResults}
             rawResults={results?.rawResults}