Skip to content

Commit 38892bb

Browse files
mbgmbg
authored
Merge pull request #13999 from github/mbg/csharp/standalone/dotnet-version
C# Standalone: Install .NET SDK specified in `global.json`
2 parents ec0529d + ccbc6f4 commit 38892bb

10 files changed

Lines changed: 136 additions & 30 deletions

File tree

‎csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,7 @@ public void TestVcVarsAllBatFiles()
557557
[Fact]
558558
public void TestLinuxBuildlessExtractionSuccess()
559559
{
560-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 0;
560+
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
561561
actions.FileExists["csharp.log"] = true;
562562
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
563563
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -571,7 +571,7 @@ public void TestLinuxBuildlessExtractionSuccess()
571571
[Fact]
572572
public void TestLinuxBuildlessExtractionFailed()
573573
{
574-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 10;
574+
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
575575
actions.FileExists["csharp.log"] = true;
576576
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
577577
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -585,7 +585,7 @@ public void TestLinuxBuildlessExtractionFailed()
585585
[Fact]
586586
public void TestLinuxBuildlessExtractionSolution()
587587
{
588-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0;
588+
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln"] = 0;
589589
actions.FileExists["csharp.log"] = true;
590590
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
591591
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -873,7 +873,7 @@ public void TestSkipNugetMsBuild()
873873
[Fact]
874874
public void TestSkipNugetBuildless()
875875
{
876-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0;
876+
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --skip-nuget"] = 0;
877877
actions.FileExists["csharp.log"] = true;
878878
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
879879
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";

‎csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,11 @@ public override BuildScript GetBuildScript()
4848
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
4949
break;
5050
case CSharpBuildStrategy.Buildless:
51-
// No need to check that the extractor has been executed in buildless mode
52-
attempt = new StandaloneBuildRule().Analyse(this, false);
51+
attempt = DotNetRule.WithDotNet(this, (dotNetPath, env) =>
52+
{
53+
// No need to check that the extractor has been executed in buildless mode
54+
return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
55+
});
5356
break;
5457
case CSharpBuildStrategy.MSBuild:
5558
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);

‎csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,16 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
7070
});
7171
}
7272

73-
private static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
73+
/// <summary>
74+
/// Returns a script that attempts to download relevant version(s) of the
75+
/// .NET Core SDK, followed by running the script generated by <paramref name="f"/>.
76+
///
77+
/// The arguments to <paramref name="f"/> are the path to the directory in which the
78+
/// .NET Core SDK(s) were installed and any additional required environment
79+
/// variables needed by the installed .NET Core (<code>null</code> when no variables
80+
/// are needed).
81+
/// </summary>
82+
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
7483
{
7584
var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet");
7685
var installScript = DownloadDotNet(builder, installDir);

‎csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ namespace Semmle.Autobuild.CSharp
88
/// </summary>
99
internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
1010
{
11+
private readonly string? dotNetPath;
12+
13+
internal StandaloneBuildRule(string? dotNetPath)
14+
{
15+
this.dotNetPath = dotNetPath;
16+
}
17+
1118
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
1219
{
1320
BuildScript GetCommand(string? solution)
@@ -28,13 +35,17 @@ BuildScript GetCommand(string? solution)
2835
if (solution is not null)
2936
cmd.QuoteArgument(solution);
3037

31-
cmd.Argument("--references:.");
32-
3338
if (!builder.Options.NugetRestore)
3439
{
3540
cmd.Argument("--skip-nuget");
3641
}
3742

43+
if (!string.IsNullOrEmpty(this.dotNetPath))
44+
{
45+
cmd.Argument("--dotnet");
46+
cmd.QuoteArgument(this.dotNetPath);
47+
}
48+
3849
return cmd.Script;
3950
}
4051

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,24 @@ internal class AssemblyCache
1515
/// <summary>
1616
/// Locate all reference files and index them.
1717
/// </summary>
18-
/// <param name="dirs">Directories to search.</param>
18+
/// <param name="paths">
19+
/// Paths to search. Directories are searched recursively. Files are added directly to the
20+
/// assembly cache.
21+
/// </param>
1922
/// <param name="progressMonitor">Callback for progress.</param>
20-
public AssemblyCache(IEnumerable<string> dirs, ProgressMonitor progressMonitor)
23+
public AssemblyCache(IEnumerable<string> paths, ProgressMonitor progressMonitor)
2124
{
22-
foreach (var dir in dirs)
25+
foreach (var path in paths)
2326
{
24-
progressMonitor.FindingFiles(dir);
25-
AddReferenceDirectory(dir);
27+
if (File.Exists(path))
28+
{
29+
pendingDllsToIndex.Enqueue(path);
30+
}
31+
else
32+
{
33+
progressMonitor.FindingFiles(path);
34+
AddReferenceDirectory(path);
35+
}
2636
}
2737
IndexReferences();
2838
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using System;
1+
using System;
22
using System.Collections.Concurrent;
33
using System.Collections.Generic;
44
using System.IO;
@@ -31,6 +31,7 @@ public sealed class DependencyManager : IDisposable
3131
private readonly FileContent fileContent;
3232
private readonly TemporaryDirectory packageDirectory;
3333
private TemporaryDirectory? razorWorkingDirectory;
34+
private readonly Git git;
3435

3536

3637
/// <summary>
@@ -48,7 +49,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
4849

4950
try
5051
{
51-
this.dotnet = new DotNet(progressMonitor);
52+
this.dotnet = new DotNet(options, progressMonitor);
5253
}
5354
catch
5455
{
@@ -68,7 +69,11 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
6869
? new[] { options.SolutionFile }
6970
: allFiles.SelectFileNamesByExtension(".sln");
7071

71-
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
72+
// If DLL reference paths are specified on the command-line, use those to discover
73+
// assemblies. Otherwise (the default), query the git CLI to determine which DLL files
74+
// are tracked as part of the repository.
75+
this.git = new Git(this.progressMonitor);
76+
var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
7277

7378
// Find DLLs in the .Net / Asp.Net Framework
7479
if (options.ScanNetFrameworkDlls)
@@ -98,13 +103,9 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
98103
progressMonitor.MissingNuGet();
99104
}
100105

101-
// TODO: remove the below when the required SDK is installed
102-
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
103-
{
104-
Restore(solutions);
105-
Restore(allProjects);
106-
DownloadMissingPackages(allFiles);
107-
}
106+
Restore(solutions);
107+
Restore(allProjects);
108+
DownloadMissingPackages(allFiles);
108109
}
109110

110111
assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
@@ -157,7 +158,6 @@ private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
157158
{
158159
progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files.");
159160

160-
// TODO: use SDK specified in global.json
161161
var sdk = new Sdk(dotnet).GetNewestSdk();
162162
if (sdk != null)
163163
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ public interface IDependencyOptions
5151
/// The number of threads to use.
5252
/// </summary>
5353
int Threads { get; }
54+
55+
/// <summary>
56+
/// The path to the local ".dotnet" directory, if any.
57+
/// </summary>
58+
string? DotNetPath { get; }
5459
}
5560

5661
public class DependencyOptions : IDependencyOptions
@@ -73,5 +78,7 @@ public bool ExcludesFile(string path) =>
7378
Excludes.Any(path.Contains);
7479

7580
public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
81+
82+
public string? DotNetPath { get; set; } = null;
7683
}
7784
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Diagnostics;
4+
using System.IO;
45
using Semmle.Util;
56

67
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -10,12 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
1011
/// </summary>
1112
internal class DotNet : IDotNet
1213
{
13-
private const string dotnet = "dotnet";
1414
private readonly ProgressMonitor progressMonitor;
15+
private readonly string dotnet;
1516

16-
public DotNet(ProgressMonitor progressMonitor)
17+
public DotNet(IDependencyOptions options, ProgressMonitor progressMonitor)
1718
{
1819
this.progressMonitor = progressMonitor;
20+
this.dotnet = Path.Combine(options.DotNetPath ?? string.Empty, "dotnet");
1921
Info();
2022
}
2123

@@ -29,7 +31,7 @@ private void Info()
2931
}
3032
}
3133

32-
private static ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
34+
private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
3335
new ProcessStartInfo(dotnet, args)
3436
{
3537
UseShellExecute = false,
@@ -39,7 +41,7 @@ private static ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectSt
3941
private bool RunCommand(string args)
4042
{
4143
progressMonitor.RunningProcess($"{dotnet} {args}");
42-
using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false));
44+
using var proc = Process.Start(this.MakeDotnetStartInfo(args, redirectStandardOutput: false));
4345
proc?.WaitForExit();
4446
var exitCode = proc?.ExitCode ?? -1;
4547
if (exitCode != 0)
@@ -77,7 +79,7 @@ public bool AddPackage(string folder, string package)
7779
private IList<string> GetListed(string args, string artifact)
7880
{
7981
progressMonitor.RunningProcess($"{dotnet} {args}");
80-
var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true);
82+
var pi = this.MakeDotnetStartInfo(args, redirectStandardOutput: true);
8183
var exitCode = pi.ReadOutput(out var artifacts);
8284
if (exitCode != 0)
8385
{

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics;
4+
5+
namespace Semmle.Extraction.CSharp.DependencyFetching
6+
{
7+
/// <summary>
8+
/// Utilities for querying information from the git CLI.
9+
/// </summary>
10+
internal class Git
11+
{
12+
private readonly ProgressMonitor progressMonitor;
13+
private const string git = "git";
14+
15+
public Git(ProgressMonitor progressMonitor)
16+
{
17+
this.progressMonitor = progressMonitor;
18+
}
19+
20+
/// <summary>
21+
/// Lists all files matching <paramref name="pattern"/> which are tracked in the
22+
/// current git repository.
23+
/// </summary>
24+
/// <param name="pattern">The file pattern.</param>
25+
/// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
26+
/// <exception cref="Exception"></exception>
27+
public List<string> ListFiles(string pattern)
28+
{
29+
var results = new List<string>();
30+
var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
31+
32+
progressMonitor.RunningProcess($"{git} {args}");
33+
var pi = new ProcessStartInfo(git, args)
34+
{
35+
UseShellExecute = false,
36+
RedirectStandardOutput = true
37+
};
38+
39+
using var p = new Process() { StartInfo = pi };
40+
p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
41+
{
42+
if (!string.IsNullOrWhiteSpace(e.Data))
43+
{
44+
results.Add(e.Data);
45+
}
46+
});
47+
p.Start();
48+
p.BeginOutputReadLine();
49+
p.WaitForExit();
50+
51+
if (p.ExitCode != 0)
52+
{
53+
progressMonitor.CommandFailed(git, args, p.ExitCode);
54+
throw new Exception($"{git} {args} failed");
55+
}
56+
57+
return results;
58+
}
59+
60+
}
61+
}

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ public override bool HandleOption(string key, string value)
5050
case "references":
5151
dependencies.DllDirs.Add(value);
5252
return true;
53+
case "dotnet":
54+
dependencies.DotNetPath = value;
55+
return true;
5356
default:
5457
return base.HandleOption(key, value);
5558
}

0 commit comments

Comments
 (0)