C#: Parallelize restore logic of missing packages

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -499,29 +499,38 @@ private void DownloadMissingPackages(List<FileInfo> allFiles)
             var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
             var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
-            foreach (var package in notYetDownloadedPackages)
-            {
-                progressMonitor.NugetInstall(package);
-                using var tempDir = new TemporaryDirectory(GetTemporaryWorkingDirectory(package));
-                var success = dotnet.New(tempDir.DirInfo.FullName);
-                if (!success)
-                {
-                    continue;
-                }
-                success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-                if (!success)
-                {
-                    continue;
-                }
-
-                success = RestoreProject(tempDir.DirInfo.FullName, out var stdout, nugetConfig);
-                Console.WriteLine(stdout);
 
-                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
-                if (!success)
-                {
-                    progressMonitor.FailedToRestoreNugetPackage(package);
-                }
+            var stdoutLines = notYetDownloadedPackages
+                .AsParallel()
+                .WithDegreeOfParallelism(options.Threads)
+                .Select(package =>
+                    {
+                        progressMonitor.NugetInstall(package);
+                        using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
+                        var success = dotnet.New(tempDir.DirInfo.FullName, out var stdout1);
+                        if (!success)
+                        {
+                            return new[] { stdout1 };
+                        }
+
+                        success = dotnet.AddPackage(tempDir.DirInfo.FullName, package, out var stdout2);
+                        if (!success)
+                        {
+                            return new[] { stdout1, stdout2 };
+                        }
+
+                        success = RestoreProject(tempDir.DirInfo.FullName, out var stdout3, nugetConfig);
+                        // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+                        if (!success)
+                        {
+                            progressMonitor.FailedToRestoreNugetPackage(package);
+                        }
+                        return new[] { stdout1, stdout2, stdout3 };
+                    })
+                .ToList();
+            foreach (var line in stdoutLines.SelectMany(l => l))
+            {
+                Console.WriteLine(line);
             }
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -37,6 +37,13 @@ private void Info()
             }
         }
 
+        private bool RunCommand(string args, out string stdout)
+        {
+            var success = dotnetCliInvoker.RunCommand(args, out var output);
+            stdout = string.Join("\n", output);
+            return success;
+        }
+
         private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
             $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
 
@@ -47,9 +54,8 @@ public bool RestoreProjectToDirectory(string projectFile, string packageDirector
             {
                 args += $" --configfile \"{pathToNugetConfig}\"";
             }
-            var success = dotnetCliInvoker.RunCommand(args, out var output);
-            stdout = string.Join("\n", output);
-            return success;
+
+            return RunCommand(args, out stdout);
         }
 
         public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects)
@@ -70,16 +76,16 @@ public bool RestoreSolutionToDirectory(string solutionFile, string packageDirect
             return false;
         }
 
-        public bool New(string folder)
+        public bool New(string folder, out string stdout)
         {
             var args = $"new console --no-restore --output \"{folder}\"";
-            return dotnetCliInvoker.RunCommand(args);
+            return RunCommand(args, out stdout);
         }
 
-        public bool AddPackage(string folder, string package)
+        public bool AddPackage(string folder, string package, out string stdout)
         {
             var args = $"add \"{folder}\" package \"{package}\" --no-restore";
-            return dotnetCliInvoker.RunCommand(args);
+            return RunCommand(args, out stdout);
         }
 
         public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
@@ -88,7 +94,7 @@ public bool AddPackage(string folder, string package)
 
         private IList<string> GetListed(string args, string artifact)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var artifacts))
+            if (dotnetCliInvoker.RunCommand(args, out IList<string> artifacts))
             {
                 progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
                 return artifacts;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -6,8 +6,8 @@ internal interface IDotNet
     {
         bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null);
         bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects);
-        bool New(string folder);
-        bool AddPackage(string folder, string package);
+        bool New(string folder, out string stdout);
+        bool AddPackage(string folder, string package, out string stdout);
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(string execArgs);

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -164,7 +164,7 @@ public void TestDotnetNew()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.New("myfolder");
+            dotnet.New("myfolder", out var _);
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -179,7 +179,7 @@ public void TestDotnetAddPackage()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            dotnet.AddPackage("myfolder", "mypackage");
+            dotnet.AddPackage("myfolder", "mypackage", out var _);
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -15,9 +15,17 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
             this.runtimes = runtimes;
             this.sdks = sdks;
         }
-        public bool AddPackage(string folder, string package) => true;
+        public bool AddPackage(string folder, string package, out string stdout)
+        {
+            stdout = "";
+            return true;
+        }
 
-        public bool New(string folder) => true;
+        public bool New(string folder, out string stdout)
+        {
+            stdout = "";
+            return true;
+        }
 
         public bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null)
         {