Merge pull request #15131 from tamasvajk/standalone/file-name

C#: Exclude not existing or problematic files from standalone extraction

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -72,7 +72,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.progressMonitor.FindingFiles(srcDir);
 
 
-            var allFiles = GetAllFiles();
+            var allFiles = GetAllFiles().ToList();
             var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
             var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
             var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
@@ -439,6 +439,25 @@ private IEnumerable<FileInfo> GetAllFiles()
                 files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
             }
 
+            files = files.Where(f =>
+            {
+                try
+                {
+                    if (f.Exists)
+                    {
+                        return true;
+                    }
+
+                    progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
+                    return false;
+                }
+                catch (Exception ex)
+                {
+                    progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
+                    return false;
+                }
+            });
+
             files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
             return files;
         }

csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/Program.cs

@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}

csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/global.json

@@ -0,0 +1,5 @@
+{
+    "sdk": {
+        "version": "8.0.100"
+    }
+}

csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/test.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+  </PropertyGroup>
+
+</Project>

csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/test.py

@@ -0,0 +1,8 @@
+from create_database_utils import *
+
+path = b'\xd2abcd.cs'
+
+with open(path, 'w') as file:
+  file.write('class X { }\n')
+
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])