Skip to content

Commit c870b0d

Browse files
tamasvajktamasvajk
committed
Add more logging to the file filtering
1 parent 694be29 commit c870b0d

2 files changed

Lines changed: 87 additions & 22 deletions

File tree

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
5959
continue;
6060
}
6161

62-
pathFilters.Add(new PathFilter(new Regex(new FilePattern(filterText).RegexPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
62+
var regex = new FilePattern(filterText).RegexPattern;
63+
progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
64+
pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
6365
}
6466

6567
var fileIndex = files.ToDictionary(f => f, f => new FileInclusion(FileUtils.ConvertToUnix(f.FullName.ToLowerInvariant()).Replace(rootFolder, string.Empty).TrimStart('/'), pathFilters.All(f => !f.Include)));
@@ -75,7 +77,20 @@ public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
7577
}
7678
}
7779

78-
return fileIndex.Where(f => f.Value.Include).Select(f => f.Key);
80+
var included = new List<FileInfo>();
81+
foreach (var file in fileIndex)
82+
{
83+
if (file.Value.Include)
84+
{
85+
included.Add(file.Key);
86+
}
87+
else
88+
{
89+
progressMonitor.Log(Severity.Info, $"Excluding '{file.Key.FullName}'");
90+
}
91+
}
92+
93+
return included;
7994
}
8095
}
8196
}

‎csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs‎

Lines changed: 70 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,15 @@ public class FilePathFilterTest
1212
{
1313
private class ProgressMonitorStub : IProgressMonitor
1414
{
15-
public void Log(Severity severity, string message) { }
15+
public List<string> Messages { get; } = [];
16+
17+
public void Log(Severity severity, string message)
18+
{
19+
Messages.Add(message);
20+
}
1621
}
1722

18-
private static (FilePathFilter TestSubject, IEnumerable<FileInfo> Files) TestSetup()
23+
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup()
1924
{
2025
return TestSetup("/a/b",
2126
[
@@ -27,13 +32,15 @@ private static (FilePathFilter TestSubject, IEnumerable<FileInfo> Files) TestSet
2732
]);
2833
}
2934

30-
private static (FilePathFilter TestSubject, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
35+
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
3136
{
3237
root = GetPlatformSpecifixPath(root);
3338
paths = GetPlatformSpecifixPaths(paths);
3439

35-
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), new ProgressMonitorStub());
36-
return (filePathFilter, paths.Select(p => new FileInfo(p)));
40+
var progressMonitor = new ProgressMonitorStub();
41+
42+
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor);
43+
return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p)));
3744
}
3845

3946
private static string GetPlatformSpecifixPath(string file)
@@ -52,27 +59,28 @@ private static IEnumerable<FileInfo> GetExpected(IEnumerable<string> files)
5259
return files.Select(f => new FileInfo(f));
5360
}
5461

55-
private static void AssertEquivalence(IEnumerable<FileInfo>? expected, IEnumerable<FileInfo>? actual)
62+
private static void AssertFileInfoEquivalence(IEnumerable<FileInfo>? expected, IEnumerable<FileInfo>? actual)
5663
{
5764
Assert.Equivalent(expected?.Select(f => f.FullName), actual?.Select(f => f.FullName), strict: true);
5865
}
5966

6067
[Fact]
6168
public void TestNoFilter()
6269
{
63-
(var testSubject, var files) = TestSetup();
70+
(var testSubject, var progressMonitor, var files) = TestSetup();
6471

6572
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
6673

6774
var filtered = testSubject.Filter(files);
6875

69-
AssertEquivalence(files, filtered);
76+
AssertFileInfoEquivalence(files, filtered);
77+
Assert.Equivalent(Array.Empty<string>(), progressMonitor.Messages, strict: true);
7078
}
7179

7280
[Fact]
7381
public void TestFiltersWithOnlyInclude()
7482
{
75-
(var testSubject, var files) = TestSetup();
83+
(var testSubject, var progressMonitor, var files) = TestSetup();
7684

7785
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
7886
include:c/d
@@ -89,13 +97,20 @@ public void TestFiltersWithOnlyInclude()
8997
"/a/b/c/x/y/i.cs"
9098
]);
9199

92-
AssertEquivalence(expected, filtered);
100+
AssertFileInfoEquivalence(expected, filtered);
101+
102+
var expectedRegexMessages = new[]
103+
{
104+
"Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'",
105+
"Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'"
106+
};
107+
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
93108
}
94109

95110
[Fact]
96111
public void TestFiltersWithOnlyExclude()
97112
{
98-
(var testSubject, var files) = TestSetup();
113+
(var testSubject, var progressMonitor, var files) = TestSetup();
99114

100115
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
101116
exclude:c/d/e
@@ -109,13 +124,19 @@ public void TestFiltersWithOnlyExclude()
109124
"/a/b/c/x/z/i.cs"
110125
]);
111126

112-
AssertEquivalence(expected, filtered);
127+
AssertFileInfoEquivalence(expected, filtered);
128+
129+
var expectedRegexMessages = new[]
130+
{
131+
"Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'"
132+
};
133+
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
113134
}
114135

115136
[Fact]
116137
public void TestFiltersWithIncludeExclude()
117138
{
118-
(var testSubject, var files) = TestSetup();
139+
(var testSubject, var progressMonitor, var files) = TestSetup();
119140

120141
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
121142
include:c/x
@@ -129,13 +150,20 @@ public void TestFiltersWithIncludeExclude()
129150
"/a/b/c/x/y/i.cs"
130151
]);
131152

132-
AssertEquivalence(expected, filtered);
153+
AssertFileInfoEquivalence(expected, filtered);
154+
155+
var expectedRegexMessages = new[]
156+
{
157+
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
158+
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
159+
};
160+
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
133161
}
134162

135163
[Fact]
136164
public void TestFiltersWithIncludeExcludeExcludeFirst()
137165
{
138-
(var testSubject, var files) = TestSetup();
166+
(var testSubject, var progressMonitor, var files) = TestSetup();
139167

140168
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
141169
exclude:c/x/z
@@ -149,13 +177,20 @@ public void TestFiltersWithIncludeExcludeExcludeFirst()
149177
"/a/b/c/x/y/i.cs"
150178
]);
151179

152-
AssertEquivalence(expected, filtered);
180+
AssertFileInfoEquivalence(expected, filtered);
181+
182+
var expectedRegexMessages = new[]
183+
{
184+
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
185+
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
186+
};
187+
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
153188
}
154189

155190
[Fact]
156191
public void TestFiltersWithIncludeExcludeComplexPatterns1()
157192
{
158-
(var testSubject, var files) = TestSetup();
193+
(var testSubject, var progressMonitor, var files) = TestSetup();
159194

160195
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
161196
include:c/**/i.*
@@ -173,13 +208,21 @@ public void TestFiltersWithIncludeExcludeComplexPatterns1()
173208
"/a/b/c/x/y/i.cs"
174209
]);
175210

176-
AssertEquivalence(expected, filtered);
211+
AssertFileInfoEquivalence(expected, filtered);
212+
213+
var expectedRegexMessages = new[]
214+
{
215+
"Filtering in files matching '^c/.*/i\\.[^/]*.*'. Original glob filter: 'include:c/**/i.*'",
216+
"Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'",
217+
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
218+
};
219+
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
177220
}
178221

179222
[Fact]
180223
public void TestFiltersWithIncludeExcludeComplexPatterns2()
181224
{
182-
(var testSubject, var files) = TestSetup();
225+
(var testSubject, var progressMonitor, var files) = TestSetup();
183226

184227
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
185228
include:**/i.*
@@ -193,7 +236,14 @@ public void TestFiltersWithIncludeExcludeComplexPatterns2()
193236
"/a/b/c/x/y/i.cs"
194237
]);
195238

196-
AssertEquivalence(expected, filtered);
239+
AssertFileInfoEquivalence(expected, filtered);
240+
241+
var expectedRegexMessages = new[]
242+
{
243+
"Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'",
244+
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
245+
};
246+
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
197247
}
198248
}
199249
}

0 commit comments

Comments
 (0)