Skip to content

Commit 822c56b

Browse files
vweijstersvweijsters
authored
Merge pull request #2664 from sharwell/document-testing
Update StyleCopTester
2 parents 3c635f8 + e5c8925 commit 822c56b

5 files changed

Lines changed: 161 additions & 14 deletions

File tree

‎NuGet.config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<configuration>
33
<packageSources>
4+
<add key="roslyn" value="https://dotnet.myget.org/F/roslyn/api/v3/index.json" />
45
<add key="symreader-converter" value="https://dotnet.myget.org/F/symreader-converter/api/v3/index.json" />
56
</packageSources>
67
</configuration>

‎StyleCop.Analyzers/StyleCopTester/App.config‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎StyleCop.Analyzers/StyleCopTester/Program.cs‎

Lines changed: 140 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ namespace StyleCopTester
1111
using System.Linq;
1212
using System.Reflection;
1313
using System.Text;
14+
using System.Text.RegularExpressions;
1415
using System.Threading;
1516
using System.Threading.Tasks;
1617
using System.Windows.Threading;
18+
using Microsoft.Build.Locator;
1719
using Microsoft.CodeAnalysis;
1820
using Microsoft.CodeAnalysis.CodeActions;
1921
using Microsoft.CodeAnalysis.CodeFixes;
@@ -82,6 +84,8 @@ private static async Task MainAsync(string[] args, CancellationToken cancellatio
8284
}
8385
}
8486

87+
MSBuildLocator.RegisterDefaults();
88+
8589
Stopwatch stopwatch = Stopwatch.StartNew();
8690
var analyzers = GetAllAnalyzers();
8791

@@ -93,9 +97,17 @@ private static async Task MainAsync(string[] args, CancellationToken cancellatio
9397
return;
9498
}
9599

100+
var properties = new Dictionary<string, string>
101+
{
102+
// This property ensures that XAML files will be compiled in the current AppDomain
103+
// rather than a separate one. Any tasks isolated in AppDomains or tasks that create
104+
// AppDomains will likely not work due to https://github.com/Microsoft/MSBuildLocator/issues/16.
105+
{ "AlwaysCompileMarkupFilesInSeparateDomain", bool.FalseString },
106+
};
107+
96108
MSBuildWorkspace workspace = MSBuildWorkspace.Create();
97109
string solutionPath = args.SingleOrDefault(i => !i.StartsWith("/", StringComparison.Ordinal));
98-
Solution solution = await workspace.OpenSolutionAsync(solutionPath, cancellationToken).ConfigureAwait(false);
110+
Solution solution = await workspace.OpenSolutionAsync(solutionPath, cancellationToken: cancellationToken).ConfigureAwait(false);
99111

100112
Console.WriteLine($"Loaded solution in {stopwatch.ElapsedMilliseconds}ms");
101113

@@ -122,6 +134,80 @@ private static async Task MainAsync(string[] args, CancellationToken cancellatio
122134

123135
Console.WriteLine($"Found {allDiagnostics.Length} diagnostics in {stopwatch.ElapsedMilliseconds}ms");
124136

137+
bool testDocuments = args.Contains("/editperf") || args.Any(arg => arg.StartsWith("/editperf:"));
138+
if (testDocuments)
139+
{
140+
Func<string, bool> documentMatch = _ => true;
141+
string matchArg = args.FirstOrDefault(arg => arg.StartsWith("/editperf:"));
142+
if (matchArg != null)
143+
{
144+
Regex expression = new Regex(matchArg.Substring("/editperf:".Length), RegexOptions.Compiled | RegexOptions.IgnoreCase);
145+
documentMatch = documentPath => expression.IsMatch(documentPath);
146+
}
147+
148+
int iterations = 10;
149+
string iterationsArg = args.FirstOrDefault(arg => arg.StartsWith("/edititer:"));
150+
if (iterationsArg != null)
151+
{
152+
iterations = int.Parse(iterationsArg.Substring("/edititer:".Length));
153+
}
154+
155+
var projectPerformance = new Dictionary<ProjectId, double>();
156+
var documentPerformance = new Dictionary<DocumentId, DocumentAnalyzerPerformance>();
157+
foreach (var projectId in solution.ProjectIds)
158+
{
159+
Project project = solution.GetProject(projectId);
160+
if (project.Language != LanguageNames.CSharp)
161+
{
162+
continue;
163+
}
164+
165+
foreach (var documentId in project.DocumentIds)
166+
{
167+
var document = project.GetDocument(documentId);
168+
if (!documentMatch(document.FilePath))
169+
{
170+
continue;
171+
}
172+
173+
var currentDocumentPerformance = await TestDocumentPerformanceAsync(analyzers, project, documentId, iterations, force, cancellationToken).ConfigureAwait(false);
174+
Console.WriteLine($"{document.FilePath ?? document.Name}: {currentDocumentPerformance.EditsPerSecond:0.00}");
175+
documentPerformance.Add(documentId, currentDocumentPerformance);
176+
}
177+
178+
double sumOfDocumentAverages = documentPerformance.Where(x => x.Key.ProjectId == projectId).Sum(x => x.Value.EditsPerSecond);
179+
double documentCount = documentPerformance.Where(x => x.Key.ProjectId == projectId).Count();
180+
if (documentCount > 0)
181+
{
182+
projectPerformance[project.Id] = sumOfDocumentAverages / documentCount;
183+
}
184+
}
185+
186+
var slowestFiles = documentPerformance.OrderBy(pair => pair.Value.EditsPerSecond).GroupBy(pair => pair.Key.ProjectId);
187+
Console.WriteLine("Slowest files in each project:");
188+
foreach (var projectGroup in slowestFiles)
189+
{
190+
Console.WriteLine($" {solution.GetProject(projectGroup.Key).Name}");
191+
foreach (var pair in projectGroup.Take(5))
192+
{
193+
var document = solution.GetDocument(pair.Key);
194+
Console.WriteLine($" {document.FilePath ?? document.Name}: {pair.Value.EditsPerSecond:0.00}");
195+
}
196+
}
197+
198+
foreach (var projectId in solution.ProjectIds)
199+
{
200+
double averageEditsInProject;
201+
if (!projectPerformance.TryGetValue(projectId, out averageEditsInProject))
202+
{
203+
continue;
204+
}
205+
206+
Project project = solution.GetProject(projectId);
207+
Console.WriteLine($"{project.Name} ({project.DocumentIds.Count} documents): {averageEditsInProject:0.00} edits per second");
208+
}
209+
}
210+
125211
foreach (var group in allDiagnostics.GroupBy(i => i.Id).OrderBy(i => i.Key, StringComparer.OrdinalIgnoreCase))
126212
{
127213
Console.WriteLine($" {group.Key}: {group.Count()} instances");
@@ -155,6 +241,44 @@ private static async Task MainAsync(string[] args, CancellationToken cancellatio
155241
}
156242
}
157243

244+
private static async Task<DocumentAnalyzerPerformance> TestDocumentPerformanceAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, Project project, DocumentId documentId, int iterations, bool force, CancellationToken cancellationToken)
245+
{
246+
var supportedDiagnosticsSpecificOptions = new Dictionary<string, ReportDiagnostic>();
247+
if (force)
248+
{
249+
foreach (var analyzer in analyzers)
250+
{
251+
foreach (var diagnostic in analyzer.SupportedDiagnostics)
252+
{
253+
// make sure the analyzers we are testing are enabled
254+
supportedDiagnosticsSpecificOptions[diagnostic.Id] = ReportDiagnostic.Default;
255+
}
256+
}
257+
}
258+
259+
// Report exceptions during the analysis process as errors
260+
supportedDiagnosticsSpecificOptions.Add("AD0001", ReportDiagnostic.Error);
261+
262+
// update the project compilation options
263+
var modifiedSpecificDiagnosticOptions = supportedDiagnosticsSpecificOptions.ToImmutableDictionary().SetItems(project.CompilationOptions.SpecificDiagnosticOptions);
264+
var modifiedCompilationOptions = project.CompilationOptions.WithSpecificDiagnosticOptions(modifiedSpecificDiagnosticOptions);
265+
266+
var stopwatch = Stopwatch.StartNew();
267+
for (int i = 0; i < iterations; i++)
268+
{
269+
var processedProject = project.WithCompilationOptions(modifiedCompilationOptions);
270+
271+
Compilation compilation = await processedProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
272+
CompilationWithAnalyzers compilationWithAnalyzers = compilation.WithAnalyzers(analyzers, new CompilationWithAnalyzersOptions(new AnalyzerOptions(ImmutableArray.Create<AdditionalText>()), null, true, false));
273+
274+
SyntaxTree tree = await project.GetDocument(documentId).GetSyntaxTreeAsync(cancellationToken).ConfigureAwait(false);
275+
await compilationWithAnalyzers.GetAnalyzerSyntaxDiagnosticsAsync(tree, cancellationToken).ConfigureAwait(false);
276+
await compilationWithAnalyzers.GetAnalyzerSemanticDiagnosticsAsync(compilation.GetSemanticModel(tree), null, cancellationToken).ConfigureAwait(false);
277+
}
278+
279+
return new DocumentAnalyzerPerformance(iterations / stopwatch.Elapsed.TotalSeconds);
280+
}
281+
158282
private static void WriteDiagnosticResults(ImmutableArray<Tuple<ProjectId, Diagnostic>> diagnostics, string fileName)
159283
{
160284
var orderedDiagnostics =
@@ -495,6 +619,21 @@ private static void PrintHelp()
495619
Console.WriteLine("/id:<id> Enable analyzer with diagnostic ID <id> (when this is specified, only this analyzer is enabled)");
496620
Console.WriteLine("/apply Write code fix changes back to disk");
497621
Console.WriteLine("/force Force an analyzer to be enabled, regardless of the configured rule set(s) for the solution");
622+
Console.WriteLine("/editperf[:<match>] Test the incremental performance of analyzers to simulate the behavior of editing files. If <match> is specified, only files matching this regular expression are evaluated for editor performance.");
623+
Console.WriteLine("/edititer:<iterations> Specifies the number of iterations to use for testing documents with /editperf. When this is not specified, the default value is 10.");
624+
}
625+
626+
private struct DocumentAnalyzerPerformance
627+
{
628+
public DocumentAnalyzerPerformance(double editsPerSecond)
629+
{
630+
this.EditsPerSecond = editsPerSecond;
631+
}
632+
633+
public double EditsPerSecond
634+
{
635+
get;
636+
}
498637
}
499638
}
500639
}

‎StyleCop.Analyzers/StyleCopTester/Properties/launchSettings.json‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
11
{
22
"profiles": {
33
"StyleCopAnalyzers.sln /stats": {
4+
"commandName": "Project",
45
"commandLineArgs": "..\\..\\StyleCopAnalyzers.sln /stats",
5-
"workingDirectory": "$(MSBuildProjectDirectory)"
6+
"workingDirectory": "$(MSBuildProjectDirectory)",
7+
"environmentVariables": {
8+
}
9+
},
10+
"StyleCopAnalyzers.sln /stats /editperf": {
11+
"commandName": "Project",
12+
"commandLineArgs": "..\\..\\StyleCopAnalyzers.sln /stats /editperf",
13+
"workingDirectory": "$(MSBuildProjectDirectory)",
14+
"environmentVariables": {
15+
}
616
}
717
}
818
}

‎StyleCop.Analyzers/StyleCopTester/StyleCopTester.csproj‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33

44
<PropertyGroup>
55
<OutputType>Exe</OutputType>
6-
<TargetFrameworks>net452</TargetFrameworks>
6+
<TargetFrameworks>net46</TargetFrameworks>
7+
8+
<!-- Automatically generate the necessary assembly binding redirects -->
9+
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
710
</PropertyGroup>
811

912
<PropertyGroup>
@@ -20,16 +23,16 @@
2023
</ItemGroup>
2124

2225
<ItemGroup>
23-
<PackageReference Include="Microsoft.CodeAnalysis" Version="1.2.1" />
26+
<PackageReference Include="Microsoft.Build.Locator" Version="1.0.13" />
27+
<PackageReference Include="Microsoft.CodeAnalysis.Common" Version=" 2.9.0-beta4-62830-01" />
28+
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version=" 2.9.0-beta4-62830-01" />
29+
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version=" 2.9.0-beta4-62830-01" />
30+
<PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version=" 2.9.0-beta4-62830-01" />
2431
</ItemGroup>
2532

2633
<ItemGroup>
2734
<ProjectReference Include="..\StyleCop.Analyzers.CodeFixes\StyleCop.Analyzers.CodeFixes.csproj" />
2835
<ProjectReference Include="..\StyleCop.Analyzers\StyleCop.Analyzers.csproj" />
2936
</ItemGroup>
3037

31-
<ItemGroup>
32-
<None Include="App.config" />
33-
</ItemGroup>
34-
3538
</Project>

0 commit comments

Comments
 (0)