Add concurrent unzip implementation

Author: koesie10

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -25,7 +25,7 @@ import {
   showAndLogErrorMessage,
   showAndLogWarningMessage,
 } from "../common/logging";
-import { unzipToDirectory } from "../common/unzip";
+import { unzipToDirectoryConcurrently } from "../common/unzip-concurrently";
 
 /**
  * distribution.ts
@@ -420,7 +420,10 @@ class ExtensionSpecificDistributionManager {
       void extLogger.log(
         `Extracting CodeQL CLI to ${this.getDistributionStoragePath()}`,
       );
-      await unzipToDirectory(archivePath, this.getDistributionStoragePath());
+      await unzipToDirectoryConcurrently(
+        archivePath,
+        this.getDistributionStoragePath(),
+      );
     } finally {
       await remove(tmpDirectory);
     }

extensions/ql-vscode/src/common/unzip-concurrently.ts

@@ -0,0 +1,60 @@
+import { availableParallelism } from "os";
+import { dirname, join } from "path";
+import { createWriteStream, ensureDir } from "fs-extra";
+import {
+  copyStream,
+  openZip,
+  openZipReadStream,
+  readZipEntries,
+} from "./unzip";
+import PQueue from "p-queue";
+
+export async function unzipToDirectoryConcurrently(
+  archivePath: string,
+  destinationPath: string,
+): Promise<void> {
+  const zipFile = await openZip(archivePath, {
+    autoClose: false,
+    strictFileNames: true,
+    lazyEntries: true,
+  });
+
+  try {
+    const entries = await readZipEntries(zipFile);
+
+    const queue = new PQueue({
+      concurrency: availableParallelism(),
+    });
+
+    await queue.addAll(
+      entries.map((entry) => async () => {
+        const path = join(destinationPath, entry.fileName);
+
+        if (/\/$/.test(entry.fileName)) {
+          // Directory file names end with '/'
+
+          await ensureDir(path);
+        } else {
+          // Ensure the directory exists
+          await ensureDir(dirname(path));
+
+          const readable = await openZipReadStream(zipFile, entry);
+
+          let mode: number | undefined = entry.externalFileAttributes >>> 16;
+          if (mode <= 0) {
+            mode = undefined;
+          }
+
+          const writeStream = createWriteStream(path, {
+            autoClose: true,
+            mode,
+          });
+
+          await copyStream(readable, writeStream);
+        }
+      }),
+    );
+  } finally {
+    zipFile.close();
+  }
+}

extensions/ql-vscode/src/common/unzip.ts

@@ -51,7 +51,7 @@ export function readZipEntries(zipFile: ZipFile): Promise<ZipEntry[]> {
   });
 }
 
-function openZipReadStream(
+export function openZipReadStream(
   zipFile: ZipFile,
   entry: ZipEntry,
 ): Promise<Readable> {
@@ -86,7 +86,7 @@ export async function openZipBuffer(
   });
 }
 
-async function copyStream(
+export async function copyStream(
   readable: Readable,
   writeStream: WriteStream,
 ): Promise<void> {
@@ -102,6 +102,14 @@ async function copyStream(
   });
 }
 
+/**
+ * Sequentially unzips all files from a zip archive. Please use
+ * `unzipToDirectoryConcurrently` if you can. This function is only
+ * provided because Jest cannot import `p-queue`.
+ *
+ * @param archivePath
+ * @param destinationPath
+ */
 export async function unzipToDirectory(
   archivePath: string,
   destinationPath: string,

extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts

@@ -16,7 +16,7 @@ import {
 } from "./shared/variant-analysis";
 import { DisposableObject, DisposeHandler } from "../common/disposable-object";
 import { EventEmitter } from "vscode";
-import { unzipToDirectory } from "../common/unzip";
+import { unzipToDirectoryConcurrently } from "../common/unzip-concurrently";
 import { readRepoTask, writeRepoTask } from "./repo-tasks-store";
 
 type CacheKey = `${number}/${string}`;
@@ -106,7 +106,7 @@ export class VariantAnalysisResultsManager extends DisposableObject {
       VariantAnalysisResultsManager.RESULTS_DIRECTORY,
     );
 
-    await unzipToDirectory(zipFilePath, unzippedFilesDirectory);
+    await unzipToDirectoryConcurrently(zipFilePath, unzippedFilesDirectory);
 
     this._onResultDownloaded.fire({
       variantAnalysisId,