Skip to content

Commit d391039

Browse files
sharwellsharwell
committed
Merge pull request #1970 from sharwell/churn-testing
Churn testing
2 parents 2fcf04f + ffc2bc7 commit d391039

5 files changed

Lines changed: 186 additions & 2 deletions

File tree

‎StyleCop.Analyzers/ChurnTesting/RunChurnTest.ps1‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
$NuGet = '..\..\.nuget\NuGet.exe'
2+
&$NuGet update -Self -Verbosity quiet
3+
4+
# Make sure the project binaries are up-to-date
5+
&$NuGet restore ..\..\StyleCopAnalyzers.sln
6+
Push-Location
7+
cd ..\..\build
8+
.\build.ps1 -Incremental
9+
Pop-Location
10+
11+
&$NuGet install Microsoft.CodeAnalysis -Version 1.1.0 -OutputDirectory ..\..\packages -Verbosity quiet
12+
&$NuGet install Microsoft.CodeAnalysis -Version 1.1.1 -OutputDirectory ..\..\packages -Verbosity quiet
13+
14+
# Create folders for the testing tool
15+
Remove-Item .\bin\StyleCopTester-Roslyn.1.0.0 -Force -Recurse -ErrorAction Ignore
16+
Remove-Item .\bin\StyleCopTester-Roslyn.1.1.0 -Force -Recurse -ErrorAction Ignore
17+
Remove-Item .\bin\StyleCopTester-Roslyn.1.1.1 -Force -Recurse -ErrorAction Ignore
18+
19+
New-Item .\bin\StyleCopTester-Roslyn.1.0.0 -ItemType Directory
20+
New-Item .\bin\StyleCopTester-Roslyn.1.1.0 -ItemType Directory
21+
New-Item .\bin\StyleCopTester-Roslyn.1.1.1 -ItemType Directory
22+
23+
Copy-Item ..\StyleCopTester\bin\Release\* .\bin\StyleCopTester-Roslyn.1.0.0
24+
Copy-Item ..\StyleCopTester\bin\Release\* .\bin\StyleCopTester-Roslyn.1.1.0
25+
Copy-Item ..\StyleCopTester\bin\Release\* .\bin\StyleCopTester-Roslyn.1.1.1
26+
27+
Copy-Item .\StyleCopTester-Roslyn.1.0.exe.config .\bin\StyleCopTester-Roslyn.1.0.0\StyleCopTester.exe.config
28+
Copy-Item .\StyleCopTester-Roslyn.1.1.exe.config .\bin\StyleCopTester-Roslyn.1.1.0\StyleCopTester.exe.config
29+
Copy-Item .\StyleCopTester-Roslyn.1.1.exe.config .\bin\StyleCopTester-Roslyn.1.1.1\StyleCopTester.exe.config
30+
31+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.Common.1.1.0\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
32+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.Workspaces.Common.1.1.0\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
33+
Copy-Item ..\..\packages\System.Collections.Immutable.1.1.37\lib\dotnet\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
34+
Copy-Item ..\..\packages\System.Reflection.Metadata.1.1.0\lib\dotnet5.2\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
35+
Copy-Item ..\..\packages\Microsoft.Composition.1.0.27\lib\portable-net45+win8+wp8+wpa81\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
36+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.CSharp.1.1.0\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
37+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.CSharp.Workspaces.1.1.0\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
38+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.VisualBasic.1.1.0\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
39+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.VisualBasic.Workspaces.1.1.0\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.0
40+
41+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.Common.1.1.1\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
42+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.Workspaces.Common.1.1.1\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
43+
Copy-Item ..\..\packages\System.Collections.Immutable.1.1.37\lib\dotnet\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
44+
Copy-Item ..\..\packages\System.Reflection.Metadata.1.1.0\lib\dotnet5.2\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
45+
Copy-Item ..\..\packages\Microsoft.Composition.1.0.27\lib\portable-net45+win8+wp8+wpa81\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
46+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.CSharp.1.1.1\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
47+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.CSharp.Workspaces.1.1.1\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
48+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.VisualBasic.1.1.1\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
49+
Copy-Item ..\..\packages\Microsoft.CodeAnalysis.VisualBasic.Workspaces.1.1.1\lib\net45\*.dll .\bin\StyleCopTester-Roslyn.1.1.1
50+
51+
#
52+
# Testing DartVS/DartVS@6f54d1d2
53+
#
54+
55+
git clone https://github.com/DartVS/DartVS.git bin\DartVS
56+
Push-Location
57+
cd bin\DartVS
58+
git checkout 6f54d1d2bf6a16aaac5a6add7e073716e35e21ba
59+
Pop-Location
60+
61+
&$NuGet restore bin\DartVS\DanTup.DartVS.sln -Verbosity quiet
62+
63+
.\bin\StyleCopTester-Roslyn.1.0.0\StyleCopTester.exe bin\DartVS\DanTup.DartVS.sln /all /log:bin\DartVS-1.0.0.txt | Out-Null
64+
.\bin\StyleCopTester-Roslyn.1.1.0\StyleCopTester.exe bin\DartVS\DanTup.DartVS.sln /all /log:bin\DartVS-1.1.0.txt | Out-Null
65+
.\bin\StyleCopTester-Roslyn.1.1.1\StyleCopTester.exe bin\DartVS\DanTup.DartVS.sln /all /log:bin\DartVS-1.1.1.txt | Out-Null
66+
67+
#
68+
# Testing JamesNK/Newtonsoft.Json@48786adc
69+
#
70+
71+
git clone https://github.com/JamesNK/Newtonsoft.Json.git bin\Newtonsoft.Json
72+
Push-Location
73+
cd bin\Newtonsoft.Json
74+
git checkout 48786adc5bf9e9bcaea52147f09d6022ae14082c
75+
Pop-Location
76+
77+
&$NuGet restore bin\Newtonsoft.Json\Src\Newtonsoft.Json.Portable.sln -Verbosity quiet
78+
79+
.\bin\StyleCopTester-Roslyn.1.0.0\StyleCopTester.exe bin\Newtonsoft.Json\Src\Newtonsoft.Json.Portable.sln /all /log:bin\Newtonsoft.Json-1.0.0.txt | Out-Null
80+
.\bin\StyleCopTester-Roslyn.1.1.0\StyleCopTester.exe bin\Newtonsoft.Json\Src\Newtonsoft.Json.Portable.sln /all /log:bin\Newtonsoft.Json-1.1.0.txt | Out-Null
81+
.\bin\StyleCopTester-Roslyn.1.1.1\StyleCopTester.exe bin\Newtonsoft.Json\Src\Newtonsoft.Json.Portable.sln /all /log:bin\Newtonsoft.Json-1.1.1.txt | Out-Null

‎StyleCop.Analyzers/ChurnTesting/StyleCopTester-Roslyn.1.0.exe.config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
5+
</startup>
6+
</configuration>

‎StyleCop.Analyzers/ChurnTesting/StyleCopTester-Roslyn.1.1.exe.config‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
5+
</startup>
6+
7+
<runtime>
8+
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
9+
<dependentAssembly>
10+
<assemblyIdentity name="System.Collections.Immutable" publicKeyToken="b03f5f7f11d50a3a" culture="neutral"/>
11+
<bindingRedirect oldVersion="1.0.27.0-1.1.65535.65535" newVersion="1.1.37.0"/>
12+
</dependentAssembly>
13+
<dependentAssembly>
14+
<assemblyIdentity name="System.Reflection.Metadata" publicKeyToken="b03f5f7f11d50a3a" culture="neutral"/>
15+
<bindingRedirect oldVersion="1.0.0.0-1.0.99.0" newVersion="1.1.0.0"/>
16+
</dependentAssembly>
17+
<dependentAssembly>
18+
<assemblyIdentity name="Microsoft.CodeAnalysis.Workspaces.Desktop" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
19+
<bindingRedirect oldVersion="0.7.0.0-1.1.0.0" newVersion="1.1.0.0"/>
20+
</dependentAssembly>
21+
<dependentAssembly>
22+
<assemblyIdentity name="Microsoft.CodeAnalysis.Workspaces" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
23+
<bindingRedirect oldVersion="0.7.0.0-1.1.0.0" newVersion="1.1.0.0"/>
24+
</dependentAssembly>
25+
<dependentAssembly>
26+
<assemblyIdentity name="Microsoft.CodeAnalysis.CSharp" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
27+
<bindingRedirect oldVersion="0.7.0.0-1.1.0.0" newVersion="1.1.0.0"/>
28+
</dependentAssembly>
29+
<dependentAssembly>
30+
<assemblyIdentity name="Microsoft.CodeAnalysis.CSharp.Workspaces" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
31+
<bindingRedirect oldVersion="0.7.0.0-1.1.0.0" newVersion="1.1.0.0"/>
32+
</dependentAssembly>
33+
<dependentAssembly>
34+
<assemblyIdentity name="Microsoft.CodeAnalysis.VisualBasic" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
35+
<bindingRedirect oldVersion="0.7.0.0-1.1.0.0" newVersion="1.1.0.0"/>
36+
</dependentAssembly>
37+
<dependentAssembly>
38+
<assemblyIdentity name="Microsoft.CodeAnalysis" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
39+
<bindingRedirect oldVersion="0.7.0.0-1.1.0.0" newVersion="1.1.0.0"/>
40+
</dependentAssembly>
41+
</assemblyBinding>
42+
</runtime>
43+
</configuration>

‎StyleCop.Analyzers/StyleCopTester/Program.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ namespace StyleCopTester
1010
using System.Diagnostics;
1111
using System.Linq;
1212
using System.Reflection;
13+
using System.Text;
1314
using System.Threading;
1415
using System.Threading.Tasks;
1516
using System.Windows.Threading;
@@ -18,6 +19,8 @@ namespace StyleCopTester
1819
using Microsoft.CodeAnalysis.CodeFixes;
1920
using Microsoft.CodeAnalysis.Diagnostics;
2021
using Microsoft.CodeAnalysis.MSBuild;
22+
using File = System.IO.File;
23+
using Path = System.IO.Path;
2124

2225
/// <summary>
2326
/// StyleCopTester is a tool that will analyze a solution, find diagnostics in it and will print out the number of
@@ -116,6 +119,13 @@ private static async Task MainAsync(string[] args, CancellationToken cancellatio
116119
}
117120
}
118121

122+
string logArgument = args.FirstOrDefault(x => x.StartsWith("/log:"));
123+
if (logArgument != null)
124+
{
125+
string fileName = logArgument.Substring(logArgument.IndexOf(':') + 1);
126+
WriteDiagnosticResults(diagnostics.SelectMany(i => i.Value.Select(j => Tuple.Create(i.Key, j))).ToImmutableArray(), fileName);
127+
}
128+
119129
if (args.Contains("/codefixes"))
120130
{
121131
await TestCodeFixesAsync(stopwatch, solution, allDiagnostics, cancellationToken).ConfigureAwait(true);
@@ -128,6 +138,38 @@ private static async Task MainAsync(string[] args, CancellationToken cancellatio
128138
}
129139
}
130140

141+
private static void WriteDiagnosticResults(ImmutableArray<Tuple<ProjectId, Diagnostic>> diagnostics, string fileName)
142+
{
143+
var orderedDiagnostics =
144+
diagnostics
145+
.OrderBy(i => i.Item2.Id)
146+
.ThenBy(i => i.Item2.Location.SourceTree?.FilePath, StringComparer.OrdinalIgnoreCase)
147+
.ThenBy(i => i.Item2.Location.SourceSpan.Start)
148+
.ThenBy(i => i.Item2.Location.SourceSpan.End);
149+
150+
var uniqueLines = new HashSet<string>();
151+
StringBuilder completeOutput = new StringBuilder();
152+
StringBuilder uniqueOutput = new StringBuilder();
153+
foreach (var diagnostic in orderedDiagnostics)
154+
{
155+
string message = diagnostic.Item2.ToString();
156+
string uniqueMessage = $"{diagnostic.Item1}: {diagnostic.Item2}";
157+
completeOutput.AppendLine(message);
158+
if (uniqueLines.Add(uniqueMessage))
159+
{
160+
uniqueOutput.AppendLine(message);
161+
}
162+
}
163+
164+
string directoryName = Path.GetDirectoryName(fileName);
165+
string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
166+
string extension = Path.GetExtension(fileName);
167+
string uniqueFileName = Path.Combine(directoryName, $"{fileNameWithoutExtension}-Unique{extension}");
168+
169+
File.WriteAllText(fileName, completeOutput.ToString(), Encoding.UTF8);
170+
File.WriteAllText(uniqueFileName, uniqueOutput.ToString(), Encoding.UTF8);
171+
}
172+
131173
private static async Task TestFixAllAsync(Stopwatch stopwatch, Solution solution, ImmutableDictionary<ProjectId, ImmutableArray<Diagnostic>> diagnostics, CancellationToken cancellationToken)
132174
{
133175
Console.WriteLine("Calculating fixes");

‎build/build.ps1‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ param (
33
[string]$VisualStudioVersion = '14.0',
44
[switch]$SkipKeyCheck,
55
[string]$Verbosity = 'minimal',
6-
[string]$Logger
6+
[string]$Logger,
7+
[switch]$Incremental
78
)
89

910
# build the solution
@@ -63,12 +64,23 @@ If ($Logger) {
6364
$LoggerArgument = "/logger:$Logger"
6465
}
6566

66-
&$msbuild '/nologo' '/m' '/nr:false' '/t:rebuild' $LoggerArgument "/verbosity:$Verbosity" "/p:Configuration=$BuildConfig" "/p:VisualStudioVersion=$VisualStudioVersion" "/p:KeyConfiguration=$KeyConfiguration" $SolutionPath
67+
If ($Incremental) {
68+
$Target = 'build'
69+
} Else {
70+
$Target = 'rebuild'
71+
}
72+
73+
&$msbuild '/nologo' '/m' '/nr:false' "/t:$Target" $LoggerArgument "/verbosity:$Verbosity" "/p:Configuration=$BuildConfig" "/p:VisualStudioVersion=$VisualStudioVersion" "/p:KeyConfiguration=$KeyConfiguration" $SolutionPath
6774
If (-not $?) {
6875
$host.ui.WriteErrorLine('Build failed, aborting!')
6976
exit $LASTEXITCODE
7077
}
7178

79+
if (-not $Incremental) {
80+
# Skip NuGet validation and copying packages to the output directory
81+
exit 0
82+
}
83+
7284
# By default, do not create a NuGet package unless the expected strong name key files were used
7385
if (-not $SkipKeyCheck) {
7486
. .\keys.ps1

0 commit comments

Comments
 (0)