Download results in batches

This makes use of the p-queue package to download our variant analysis
results in batches of 3 at a time.

Author: elenatanasoiu

extensions/ql-vscode/src/extension.ts

@@ -951,7 +951,14 @@ async function activateWithInstalledDistribution(
       variantAnalysisSummary: VariantAnalysisApiResponse,
       token: CancellationToken
     ) => {
-      await variantAnalysisManager.autoDownloadVariantAnalysisResult(scannedRepo, variantAnalysisSummary, token);
+
+      const input = [
+        variantAnalysisManager.queue.add(async () => {
+          await variantAnalysisManager.autoDownloadVariantAnalysisResult(scannedRepo, variantAnalysisSummary, token);
+        })
+      ];
+
+      await Promise.all(input);
     })
   );
 

extensions/ql-vscode/src/remote-queries/variant-analysis-manager.ts

@@ -22,6 +22,7 @@ import { VariantAnalysisResultsManager } from './variant-analysis-results-manage
 import { CodeQLCliServer } from '../cli';
 import { getControllerRepo } from './run-remote-query';
 import { processUpdatedVariantAnalysis } from './variant-analysis-processor';
+import PQueue from 'p-queue';
 
 export class VariantAnalysisManager extends DisposableObject implements VariantAnalysisViewManager<VariantAnalysisView> {
   private readonly _onVariantAnalysisAdded = this.push(new EventEmitter<VariantAnalysis>());
@@ -31,6 +32,8 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
   private readonly variantAnalysisResultsManager: VariantAnalysisResultsManager;
   private readonly variantAnalyses = new Map<number, VariantAnalysis>();
   private readonly views = new Map<number, VariantAnalysisView>();
+  private static readonly maxConcurrentTasks = 3;
+  public queue: PQueue;
 
   constructor(
     private readonly ctx: ExtensionContext,
@@ -44,6 +47,8 @@ export class VariantAnalysisManager extends DisposableObject implements VariantA
 
     this.variantAnalysisResultsManager = this.push(new VariantAnalysisResultsManager(cliServer, storagePath, logger));
     this.variantAnalysisResultsManager.onResultLoaded(this.onRepoResultLoaded.bind(this));
+
+    this.queue = new PQueue({ concurrency: VariantAnalysisManager.maxConcurrentTasks });
   }
 
   public async showView(variantAnalysisId: number): Promise<void> {

extensions/ql-vscode/src/vscode-tests/cli-integration/remote-queries/variant-analysis-manager.test.ts

@@ -149,6 +149,41 @@ describe('Variant Analysis Manager', async function() {
 
         expect(getVariantAnalysisRepoResultStub.calledOnce).to.be.true;
       });
+
+      it('should pop download tasks off the queue', async () => {
+        const getResultsSpy = sandbox.spy(variantAnalysisManager, 'autoDownloadVariantAnalysisResult');
+
+        const input = [
+          variantAnalysisManager.queue.add(async () => {
+            await variantAnalysisManager.autoDownloadVariantAnalysisResult(
+              scannedRepos[0],
+              variantAnalysis,
+              cancellationTokenSource.token
+            );
+          }),
+          variantAnalysisManager.queue.add(async () => {
+            await variantAnalysisManager.autoDownloadVariantAnalysisResult(
+              scannedRepos[0],
+              variantAnalysis,
+              cancellationTokenSource.token
+            );
+          }),
+          variantAnalysisManager.queue.add(async () => {
+            await variantAnalysisManager.autoDownloadVariantAnalysisResult(
+              scannedRepos[0],
+              variantAnalysis,
+              cancellationTokenSource.token
+            );
+          })
+        ];
+
+        expect(variantAnalysisManager.queue.pending).to.equal(3);
+
+        await Promise.all(input);
+
+        expect(variantAnalysisManager.queue.pending).to.equal(0);
+        expect(getResultsSpy).to.have.been.calledThrice;
+      });
     });
   });
 });