Skip to content

Commit 8473678

Browse files
MathiasVPMathiasVP
committed
PS: Support --file-list option in the extractor.
1 parent 9ba4ffd commit 8473678

5 files changed

Lines changed: 68 additions & 30 deletions

File tree

‎powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Options.cs‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
using Semmle.Util.Logging;
21
using System.Collections.Generic;
32
using System.IO;
43
using System.Linq;
54
using Semmle.Util;
5+
using Semmle.Util.Logging;
66

77
namespace Semmle.Extraction.PowerShell.Standalone
88
{
@@ -36,22 +36,31 @@ public override bool HandleOption(string key, string value)
3636
case "exclude":
3737
Excludes.Add(value);
3838
return true;
39+
case "file-list":
40+
Files = File.ReadAllLines(value).Select(f => new FileInfo(f)).ToArray();
41+
return true;
3942
default:
4043
return base.HandleOption(key, value);
4144
}
4245
}
4346

4447
public override bool HandleArgument(string arg)
4548
{
46-
SrcDir = arg;
47-
if (!new FileInfo(SrcDir).Exists)
49+
if (!new FileInfo(arg).Exists)
4850
{
49-
var di = new DirectoryInfo(SrcDir);
51+
var di = new DirectoryInfo(arg);
5052
if (!di.Exists)
5153
{
52-
System.Console.WriteLine("Error: The file or directory {0} does not exist", di.FullName);
54+
System.Console.WriteLine(
55+
"Error: The file or directory {0} does not exist",
56+
di.FullName
57+
);
5358
Errors = true;
5459
}
60+
else
61+
{
62+
Files = di.GetFiles("*.*", SearchOption.AllDirectories);
63+
}
5564
}
5665
return true;
5766
}
@@ -70,12 +79,17 @@ public override void InvalidArgument(string argument)
7079
/// <summary>
7180
/// Files/patterns to exclude.
7281
/// </summary>
73-
public IList<string> Excludes { get; } = new List<string>() { "node_modules", "bower_components" };
82+
public IList<string> Excludes { get; } =
83+
new List<string>() { "node_modules", "bower_components" };
7484

7585
/// <summary>
7686
/// The directory or file containing the source code;
7787
/// </summary>
78-
public string SrcDir { get; set; } = Directory.GetCurrentDirectory();
88+
public FileInfo[] Files { get; set; } =
89+
new DirectoryInfo(Directory.GetCurrentDirectory()).GetFiles(
90+
"*.*",
91+
SearchOption.AllDirectories
92+
);
7993

8094
/// <summary>
8195
/// Whether the extraction phase should be skipped (dry-run).
@@ -109,19 +123,20 @@ public bool ExcludesFile(string path)
109123
/// </summary>
110124
public static void ShowHelp(System.IO.TextWriter output)
111125
{
112-
output.WriteLine("PowerShell# standalone extractor\n\nExtracts PowerShell scripts in the current directory.\n");
126+
output.WriteLine(
127+
"PowerShell# standalone extractor\n\nExtracts PowerShell scripts in the current directory.\n"
128+
);
113129
output.WriteLine("Additional options:\n");
114130
output.WriteLine(" <path> Use the provided path instead.");
115-
output.WriteLine(" --exclude:xxx Exclude a file or directory (can be specified multiple times)");
131+
output.WriteLine(
132+
" --exclude:xxx Exclude a file or directory (can be specified multiple times)"
133+
);
116134
output.WriteLine(" --dry-run Stop before extraction");
117135
output.WriteLine(" --threads:nnn Specify number of threads (default=CPU cores)");
118136
output.WriteLine(" --verbose Produce more output");
119-
120137
}
121138

122-
private Options()
123-
{
124-
}
139+
private Options() { }
125140

126141
public static Options Create(string[] args)
127142
{

‎powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Program.cs‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
using System;
22
using System.Collections.Generic;
3-
using Semmle.Util.Logging;
43
using System.IO;
54
using System.Linq;
65
using Semmle.Extraction.PowerShell;
6+
using Semmle.Util.Logging;
77

88
namespace Semmle.Extraction.PowerShell.Standalone
99
{
10-
1110
/// <summary>
1211
/// One independent run of the extractor.
1312
/// </summary>
@@ -45,9 +44,13 @@ public static int Main(string[] args)
4544
var start = DateTime.Now;
4645

4746
output.Log(Severity.Info, "Running PowerShell standalone extractor");
48-
var di = new DirectoryInfo(options.SrcDir);
49-
var sourceFiles = di.GetFiles("*.*", SearchOption.AllDirectories)
50-
.Where(d => options.Extensions.Contains(d.Extension,StringComparer.InvariantCultureIgnoreCase))
47+
var sourceFiles = options
48+
.Files.Where(d =>
49+
options.Extensions.Contains(
50+
d.Extension,
51+
StringComparer.InvariantCultureIgnoreCase
52+
)
53+
)
5154
.Select(d => d.FullName)
5255
.Where(d => !options.ExcludesFile(d))
5356
.ToArray();
@@ -64,7 +67,10 @@ public static int Main(string[] args)
6467

6568
if (!options.SkipExtraction)
6669
{
67-
using var fileLogger = new FileLogger(options.Verbosity, PowerShell.Extractor.Extractor.GetPowerShellLogPath());
70+
using var fileLogger = new FileLogger(
71+
options.Verbosity,
72+
PowerShell.Extractor.Extractor.GetPowerShellLogPath()
73+
);
6874

6975
output.Log(Severity.Info, "");
7076
output.Log(Severity.Info, "Extracting...");
@@ -73,7 +79,8 @@ public static int Main(string[] args)
7379
sourceFiles,
7480
new ExtractionProgress(output),
7581
fileLogger,
76-
options);
82+
options
83+
);
7784
output.Log(Severity.Info, $"Extraction completed in {DateTime.Now - start}");
7885
}
7986

@@ -89,14 +96,27 @@ public ExtractionProgress(ILogger output)
8996

9097
private readonly ILogger logger;
9198

92-
public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
99+
public void Analysed(
100+
int item,
101+
int total,
102+
string source,
103+
string output,
104+
TimeSpan time,
105+
AnalysisAction action
106+
)
93107
{
94-
logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
108+
logger.Log(
109+
Severity.Info,
110+
"[{0}/{1}] {2} ({3})",
111+
item,
112+
total,
113+
source,
95114
action == AnalysisAction.Extracted
96115
? time.ToString()
97116
: action == AnalysisAction.Excluded
98117
? "excluded"
99-
: "up to date");
118+
: "up to date"
119+
);
100120
}
101121
}
102122
}

powershell/tools/index.cmd renamed to powershell/tools/index-files.cmd

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ if not defined CODEQL_POWERSHELL_EXTRACTOR (
44
set CODEQL_POWERSHELL_EXTRACTOR=Semmle.Extraction.PowerShell.Standalone.exe
55
)
66

7-
type NUL && "%CODEQL_EXTRACTOR_POWERSHELL_ROOT%/tools/%CODEQL_PLATFORM%/%CODEQL_POWERSHELL_EXTRACTOR%"
7+
type NUL && "%CODEQL_EXTRACTOR_POWERSHELL_ROOT%/tools/%CODEQL_PLATFORM%/%CODEQL_POWERSHELL_EXTRACTOR%" ^
8+
--file-list "%1"
89

910
exit /b %ERRORLEVEL%

powershell/tools/index.sh renamed to powershell/tools/index-files.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ if [[ -z "${CODEQL_EXTRACTOR_POWERSHELL_ROOT}" ]]; then
44
export CODEQL_EXTRACTOR_POWERSHELL_ROOT="Semmle.Extraction.PowerShell.Standalone.exe"
55
fi
66

7-
"$CODEQL_EXTRACTOR_POWERSHELL_ROOT/tools/$CODEQL_PLATFORM/$CODEQL_POWERSHELL_EXTRACTOR"
7+
"$CODEQL_EXTRACTOR_POWERSHELL_ROOT/tools/$CODEQL_PLATFORM/$CODEQL_POWERSHELL_EXTRACTOR" --file-list "%1"

‎powershell/tools/qltest.cmd‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
@echo off
22

3-
if not defined CODEQL_POWERSHELL_EXTRACTOR (
4-
set CODEQL_POWERSHELL_EXTRACTOR=Semmle.Extraction.PowerShell.Standalone.exe
5-
)
6-
7-
type NUL && "%CODEQL_EXTRACTOR_POWERSHELL_ROOT%/tools/%CODEQL_PLATFORM%/%CODEQL_POWERSHELL_EXTRACTOR%"
3+
type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
4+
--prune=**/*.testproj ^
5+
--include-extension=.ps1 ^
6+
--size-limit=5m ^
7+
--language=powershell ^
8+
--working-dir=. ^
9+
"%CODEQL_EXTRACTOR_POWERSHELL_WIP_DATABASE%"
810

911
exit /b %ERRORLEVEL%

0 commit comments

Comments
 (0)