Skip to content

Commit 6e14111

Browse files
redsun82redsun82
committed
Just: use --all-checks and --codeql special flags, and relativize flags in forwarder
1 parent d7d7cf9 commit 6e14111

8 files changed

Lines changed: 112 additions & 46 deletions

File tree

‎java/justfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import '../misc/just/lib.just'
2+
3+
build: (_build "java")

‎java/ql/justfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import "../../misc/just/lib.just"
2+
3+
[no-cd]
4+
format *ARGS=".": (_ql_format ARGS)
5+
6+
consistency_queries := source_dir() / "consistency-queries"

‎java/ql/test/justfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import "../justfile"
2+
3+
base_flags := """\
4+
CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \
5+
"""
6+
7+
all_checks := default_db_checks + """\
8+
--check-undefined-labels \
9+
--check-repeated-labels \
10+
--check-redefined-labels \
11+
--check-use-before-definition \
12+
--consistency-queries=""" + consistency_queries
13+
14+
[no-cd]
15+
test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS)

‎misc/just/codeql-test-run.ts‎

Lines changed: 63 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as child_process from "child_process";
22
import * as path from "path";
33
import * as os from "os";
4+
import * as fs from "fs";
45

56
function invoke(
67
invocation: string[],
@@ -28,65 +29,68 @@ type Args = {
2829
tests: string[];
2930
flags: string[];
3031
env: string[];
31-
build: boolean;
32-
testing_level: number;
32+
codeql: string;
33+
all: boolean;
3334
};
3435

3536
function parseArgs(args: Args, argv: string) {
36-
argv.split(/(?<!\\) /).forEach((arg) => {
37-
if (arg === "--no-build") {
38-
args.build = false;
39-
} else if (arg.startsWith("-")) {
40-
args.flags.push(arg);
41-
} else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
42-
args.env.push(arg);
43-
} else if (/^\++$/.test(arg)) {
44-
args.testing_level = Math.max(args.testing_level, arg.length);
45-
} else if (arg !== "") {
46-
args.tests.push(arg);
47-
}
48-
});
37+
argv.split(/(?<!\\) /)
38+
.map((arg) => arg.replace("\\ ", " "))
39+
.forEach((arg) => {
40+
if (arg.startsWith("--codeql=")) {
41+
args.codeql = arg.split("=")[1];
42+
} else if (arg === "+" || arg === "--all-checks") {
43+
args.all = true;
44+
} else if (arg.startsWith("-")) {
45+
args.flags.push(arg);
46+
} else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
47+
args.env.push(arg);
48+
} else if (arg !== "") {
49+
args.tests.push(arg);
50+
}
51+
});
4952
}
5053

5154
function codeqlTestRun(argv: string[]): number {
52-
const [language, extra_args, ...plus] = argv;
53-
let codeql = process.env["SEMMLE_CODE"]
54-
? path.join(
55-
process.env["SEMMLE_CODE"],
56-
"target",
57-
"intree",
58-
`codeql-${language}`,
59-
"codeql",
60-
)
61-
: "codeql";
55+
const semmle_code = process.env["SEMMLE_CODE"];
56+
const [language, base_args, all_args, extra_args] = argv;
6257
const ram_per_thread = process.platform === "linux" ? 3000 : 2048;
6358
const cpus = os.cpus().length;
6459
let args: Args = {
6560
tests: [],
6661
flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`],
6762
env: [],
68-
build: true,
69-
testing_level: 0,
63+
codeql: semmle_code ? "build" : "host",
64+
all: false,
7065
};
66+
parseArgs(args, base_args);
7167
parseArgs(args, extra_args);
72-
for (let i = 0; i < Math.min(plus.length, args.testing_level); i++) {
73-
parseArgs(args, plus[i]);
68+
if (args.all) {
69+
parseArgs(args, all_args);
70+
}
71+
if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) {
72+
console.error(
73+
"Using `--codeql=build` or `--codeql=built` requires working with the internal repository",
74+
);
75+
return 1;
7476
}
7577
if (args.tests.length === 0) {
7678
args.tests.push(".");
7779
}
78-
if (args.build && process.env["SEMMLE_CODE"]) {
79-
// If SEMMLE_CODE is set, we are in the semmle-code repo, so we build the codeql binary.
80-
// Otherwise, we use codeql from PATH.
80+
if (args.codeql === "build") {
8181
if (
8282
invoke(["python3", "build", `target/intree/codeql-${language}`], {
83-
cwd: process.env["SEMMLE_CODE"],
83+
cwd: semmle_code,
8484
}) !== 0
8585
) {
8686
return 1;
8787
}
8888
}
89-
process.env["CODEQL_CONFIG_FILE"] ||= "."; // disable the default implicit config file, but keep an explicit one
89+
if (args.codeql !== "host") {
90+
// disable the default implicit config file, but keep an explicit one
91+
// this is the same behavior wrt to `--codeql` as the integration test runner
92+
process.env["CODEQL_CONFIG_FILE"] ||= ".";
93+
}
9094
// Set and unset environment variables
9195
args.env.forEach((envVar) => {
9296
const [key, value] = envVar.split("=", 2);
@@ -101,6 +105,31 @@ function codeqlTestRun(argv: string[]): number {
101105
process.exit(1);
102106
}
103107
});
108+
let codeql;
109+
if (args.codeql === "built" || args.codeql === "build") {
110+
codeql = path.join(
111+
semmle_code!,
112+
"target",
113+
"intree",
114+
`codeql-${language}`,
115+
"codeql",
116+
);
117+
} else if (args.codeql === "host") {
118+
codeql = "codeql";
119+
} else {
120+
codeql = args.codeql;
121+
if (fs.lstatSync(codeql).isDirectory()) {
122+
codeql = path.join(codeql, "codeql");
123+
if (process.platform === "win32") {
124+
codeql += ".exe";
125+
}
126+
}
127+
if (!fs.existsSync(codeql)) {
128+
console.error(`CodeQL executable not found: ${codeql}`);
129+
return 1;
130+
}
131+
}
132+
104133
return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {
105134
log_prefix: args.env.join(" "),
106135
});

‎misc/just/forward-command.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ function forwardCommand(args: string[]): number {
3636
}
3737
process.env[envVariable] = "true";
3838
const cmdArgs = args.slice(1);
39-
// non-positional arguments are flags, repeated + (used by language tests) or environment variable settings
40-
const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/;
39+
// non-positional arguments are flags, + (used by language tests) or environment variable settings
40+
const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/;
4141
const flags = cmdArgs.filter((arg) => is_non_positional.test(arg));
4242
const positionalArgs = cmdArgs.filter(
4343
(arg) => !is_non_positional.test(arg),
@@ -55,11 +55,26 @@ function forwardCommand(args: string[]): number {
5555
if (relativeArgs.length === 1 && relativeArgs[0] === ".") {
5656
relativeArgs = [];
5757
}
58+
let relativeFlags = flags.map((arg) => {
59+
// this might break in specific corner cases, but is good enough for most uses
60+
// workaround if this doesn't work is to not use the forwarder (call just directly in the relevant directory)
61+
if (arg.includes("=") && arg.includes(path.sep)) {
62+
let [flags, flag_arg] = arg.split("=", 2);
63+
flag_arg = flag_arg
64+
.split(path.delimiter)
65+
.map((p) =>
66+
path.isAbsolute(p) ? p : path.relative(commonPath, p),
67+
)
68+
.join(path.delimiter);
69+
return `${flags}=${flag_arg}`;
70+
}
71+
return arg;
72+
});
5873

5974
const invocation = [
6075
process.env["JUST_EXECUTABLE"] || "just",
6176
cmd,
62-
...flags,
77+
...relativeFlags,
6378
...relativeArgs,
6479
];
6580
console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`);

‎misc/just/lib.just‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ default_db_checks := """\
3838
"""
3939

4040
[no-cd, positional-arguments, no-exit-message]
41-
@_codeql_test LANGUAGE +ARGS:
41+
@_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS:
4242
{{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@"
4343

4444

‎rust/ql/justfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@ import "../../misc/just/lib.just"
33
[no-cd]
44
format *ARGS=".": (_ql_format ARGS)
55

6-
db_checks := default_db_checks
7-
8-
consistency := "--consistency-queries=" + source_dir() / "consistency-queries"
6+
all_checks := default_db_checks + """\
7+
--consistency-queries=""" + source_dir() / "consistency-queries"
98

109
[no-cd]
11-
test *ARGS=".": (_codeql_test "rust" ARGS db_checks consistency)
10+
test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS)

‎swift/ql/justfile‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@ import "../../misc/just/lib.just"
33
[no-cd]
44
format *ARGS=".": (_ql_format ARGS)
55

6-
db_checks := default_db_checks + """\
6+
all_checks := default_db_checks + """\
77
--check-repeated-labels \
88
--check-redefined-labels \
99
--check-use-before-definition \
10-
"""
10+
--consistency-queries=""" + source_dir() / "consistency-queries"
1111

12-
consistency := "--consistency-queries=" + source_dir() / "consistency-queries"
1312

1413
[no-cd]
15-
test *ARGS=".": (_codeql_test "swift" ARGS db_checks consistency)
14+
test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS)

0 commit comments

Comments
 (0)