Skip to content

Commit df8d453

Browse files
tamasvajktamasvajk
authored
Merge pull request #15395 from tamasvajk/feature/standalone-nuget-restore-retry
C#: Try fallback `dotnet restore` without nuget.config
2 parents f1d6f56 + de4e396 commit df8d453

12 files changed

Lines changed: 109 additions & 18 deletions

File tree

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -650,12 +650,6 @@ private void AnalyseProject(FileInfo project)
650650

651651
}
652652

653-
private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null) =>
654-
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, out assets, pathToNugetConfig);
655-
656-
private bool RestoreSolution(string solution, out IEnumerable<string> projects, out IEnumerable<string> assets) =>
657-
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects, out assets);
658-
659653
/// <summary>
660654
/// Executes `dotnet restore` on all solution files in solutions.
661655
/// As opposed to RestoreProjects this is not run in parallel using PLINQ
@@ -670,7 +664,7 @@ private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out
670664
var assetFiles = new List<string>();
671665
var projects = solutions.SelectMany(solution =>
672666
{
673-
RestoreSolution(solution, out var restoredProjects, out var a);
667+
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a);
674668
assetFiles.AddRange(a);
675669
return restoredProjects;
676670
});
@@ -689,7 +683,7 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
689683
var assetFiles = new List<string>();
690684
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
691685
{
692-
RestoreProject(project, forceDotnetRefAssemblyFetching: true, out var a);
686+
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _);
693687
assetFiles.AddRange(a);
694688
});
695689
assets = assetFiles;
@@ -736,11 +730,21 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
736730
return;
737731
}
738732

739-
dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, pathToNugetConfig: nugetConfig);
740-
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
733+
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig);
741734
if (!success)
742735
{
743-
progressMonitor.FailedToRestoreNugetPackage(package);
736+
if (outputLines?.Any(s => s.Contains("NU1301")) == true)
737+
{
738+
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
739+
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true);
740+
}
741+
742+
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
743+
744+
if (!success)
745+
{
746+
progressMonitor.FailedToRestoreNugetPackage(package);
747+
}
744748
}
745749
});
746750

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,16 +72,21 @@ private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines)
7272
private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>
7373
GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
7474

75-
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)
75+
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
7676
{
7777
var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
7878
if (pathToNugetConfig != null)
7979
{
8080
args += $" --configfile \"{pathToNugetConfig}\"";
8181
}
8282

83-
var success = dotnetCliInvoker.RunCommand(args, out var output);
84-
assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
83+
if (force)
84+
{
85+
args += " --force";
86+
}
87+
88+
var success = dotnetCliInvoker.RunCommand(args, out outputLines);
89+
assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty<string>();
8590
return success;
8691
}
8792

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
44
{
55
internal interface IDotNet
66
{
7-
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null);
7+
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false);
88
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);
99
bool New(string folder);
1010
bool AddPackage(string folder, string package);

‎csharp/extractor/Semmle.Extraction.Tests/DotNet.cs‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public void TestDotnetRestoreProjectToDirectory1()
101101
var dotnet = MakeDotnet(dotnetCliInvoker);
102102

103103
// Execute
104-
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets);
104+
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _);
105105

106106
// Verify
107107
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -116,7 +116,7 @@ public void TestDotnetRestoreProjectToDirectory2()
116116
var dotnet = MakeDotnet(dotnetCliInvoker);
117117

118118
// Execute
119-
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, "myconfig.config");
119+
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config");
120120

121121
// Verify
122122
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -126,6 +126,24 @@ public void TestDotnetRestoreProjectToDirectory2()
126126
Assert.Contains("/path/to/project2.assets.json", assets);
127127
}
128128

129+
[Fact]
130+
public void TestDotnetRestoreProjectToDirectory3()
131+
{
132+
// Setup
133+
var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetRestoreOutput());
134+
var dotnet = MakeDotnet(dotnetCliInvoker);
135+
136+
// Execute
137+
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true);
138+
139+
// Verify
140+
var lastArgs = dotnetCliInvoker.GetLastArgs();
141+
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs);
142+
Assert.Equal(2, assets.Count());
143+
Assert.Contains("/path/to/project.assets.json", assets);
144+
Assert.Contains("/path/to/project2.assets.json", assets);
145+
}
146+
129147
[Fact]
130148
public void TestDotnetRestoreSolutionToDirectory1()
131149
{

‎csharp/extractor/Semmle.Extraction.Tests/Runtime.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
1919

2020
public bool New(string folder) => true;
2121

22-
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)
22+
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
2323
{
2424
assets = Array.Empty<string>();
25+
outputLines = Array.Empty<string>();
2526
return true;
2627
}
2728

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/Assemblies.expected‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
| newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/Assemblies.ql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import csharp
2+
3+
private string getPath(Assembly a) {
4+
not a.getCompilation().getOutputAssembly() = a and
5+
exists(string s | s = a.getFile().getAbsolutePath() |
6+
result = s.substring(s.indexOf("newtonsoft.json"), s.length())
7+
)
8+
}
9+
10+
from Assembly a
11+
select getPath(a)

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/Program.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
class Program
2+
{
3+
static void Main(string[] args)
4+
{
5+
}
6+
}

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/nuget.config‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<configuration>
3+
<packageSources>
4+
<clear />
5+
<add key="x" value="https://abc.abc/packages/" />
6+
</packageSources>
7+
</configuration>

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/proj/proj.csproj‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFrameworks>net8.0</TargetFrameworks>
6+
</PropertyGroup>
7+
8+
<Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
9+
<RemoveDir Directories=".\bin" />
10+
<RemoveDir Directories=".\obj" />
11+
</Target>
12+
13+
<ItemGroup>
14+
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
15+
</ItemGroup>
16+
</Project>

0 commit comments

Comments
 (0)