Skip to content

Commit 74b763b

Browse files
keiravillekodekeiravillekode
committed
upgrade to interface version 2
https://exercism.org/docs/building/tooling/test-runners/interface
1 parent 712ff5c commit 74b763b

File tree

8 files changed

+216
-26
lines changed

8 files changed

+216
-26
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/tests/**/results.json
22
/tests/**/test
33
/tests/**/test.c
4+
/tests/**/lib
45
/tests/**/*.actual
56
/tests/**/*.expected

‎bin/run.sh‎

Lines changed: 131 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
# $3: path to output directory
1010

1111
# Output:
12-
# Writes the test results to a results.json file in the passed-in output directory.
13-
# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
12+
# Writes a v2 results.json to the output directory, per
13+
# https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
1414

1515
# Example:
1616
# ./bin/run.sh two-fer path/to/solution/folder/ path/to/output/directory/
@@ -30,22 +30,141 @@ results_file="${output_dir}/results.json"
3030
mkdir -p "${output_dir}"
3131

3232
# Symlink in the external packages
33-
ln -s "/opt/test-runner/lib" "${solution_dir}/lib"
34-
35-
ls -al /opt/test-runner/lib
33+
ln -sfn "/opt/test-runner/lib" "${solution_dir}/lib"
3634

3735
echo "${slug}: testing..."
3836

39-
# Run the tests for the provided implementation file and redirect stdout and
40-
# stderr to capture it
4137
test_output=$(futhark test "${solution_dir}/test.fut" 2>&1)
38+
exit_code=$?
39+
40+
# Strip absolute paths from output so results are portable
41+
test_output=$(printf '%s' "${test_output}" | sed "s#${solution_dir}/\{0,1\}##g")
42+
43+
# ---------- Extract test names and inputs from test.fut ----------
44+
# Each test block: description comment, "-- ==", optional "-- entry:",
45+
# "-- input { ... }" or "-- output { ... }" or "-- error: ...".
46+
test_info_json=$(jq -Rs '
47+
def parse_entry: capture("^-- entry:\\s*(?<e>\\S+)") | .e;
48+
def parse_input: capture("^-- input\\s+(?<i>.*)") | .i;
4249
43-
# Write the results.json file based on the exit code of the command that was
44-
# just executed that tested the implementation file
45-
if [ $? -eq 0 ]; then
46-
jq -n '{version: 1, status: "pass"}' > ${results_file}
50+
split("\n") |
51+
reduce .[] as $line (
52+
{state: "idle", comment: "", entry: "main", input: "", tests: []};
53+
if .state == "idle" then
54+
if ($line | startswith("-- ==")) then
55+
.state = "header"
56+
elif ($line | startswith("-- ")) then
57+
.comment = ($line | ltrimstr("-- "))
58+
else . end
59+
elif .state == "header" then
60+
if ($line | startswith("-- entry:")) then
61+
.entry = ($line | parse_entry)
62+
elif ($line | startswith("-- input")) then
63+
.input = ($line | parse_input)
64+
| .state = "body"
65+
elif ($line | startswith("-- output")) then
66+
.state = "body"
67+
elif ($line | startswith("-- error:")) then
68+
.state = "body"
69+
else . end
70+
elif .state == "body" then
71+
if ($line | test("^\\s*$")) then
72+
.tests += [{name: .comment, entry: .entry, input: .input}]
73+
| .comment = "" | .entry = "main" | .input = "" | .state = "idle"
74+
elif ($line | startswith("--")) then
75+
.
76+
else
77+
.tests += [{name: .comment, entry: .entry, input: .input}]
78+
| .comment = "" | .entry = "main" | .input = "" | .state = "idle"
79+
end
80+
else . end
81+
) | if .state == "body" then
82+
.tests += [{name: .comment, entry: .entry, input: .input}]
83+
else . end
84+
| .tests
85+
' "${solution_dir}/test.fut")
86+
87+
# ---------- Handle results based on exit code ----------
88+
if [ ${exit_code} -eq 0 ]; then
89+
# All tests passed
90+
jq -n --argjson tests "${test_info_json}" '
91+
{version: 2, status: "pass", tests: [
92+
$tests[] | {name: .name, status: "pass"}
93+
]}
94+
' > "${results_file}"
95+
elif printf '%s' "${test_output}" | grep -q "^Error at"; then
96+
# Compilation error — no per-test results
97+
jq -n --arg message "${test_output}" \
98+
'{version: 2, status: "error", message: $message}' > "${results_file}"
4799
else
48-
jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
100+
# Test failures — parse output to identify which tests failed.
101+
# futhark test only prints failing tests:
102+
# Entry point: main; dataset: #0 ("2100i32"):
103+
# test.fut.main.0.actual and test.fut.main.0.expected do not match:
104+
# Value #0: expected False, got True
105+
# Match failures to test blocks by entry point and input values.
106+
tests_json=$(jq -n \
107+
--argjson info "${test_info_json}" \
108+
--arg output "${test_output}" '
109+
110+
def entry_point: capture("Entry point: (?<e>[^;]+)") | .e;
111+
def dataset_label: capture("\\(\"(?<l>[^\"]+)\"\\)") | .l // "";
112+
113+
def parse_failures:
114+
split("\n") |
115+
reduce .[] as $line (
116+
{current: null, failures: []};
117+
if (($line | contains("Entry point:")) and ($line | contains("dataset:"))) then
118+
(if .current then .failures += [.current] else . end)
119+
| .current = {
120+
entry: ($line | entry_point),
121+
label: ($line | dataset_label),
122+
message: ""
123+
}
124+
elif .current then
125+
.current.message += (
126+
if .current.message != "" then "\n" + $line else $line end
127+
)
128+
else . end
129+
) | if .current then .failures += [.current] else . end
130+
| .failures
131+
| map(.message |= sub("\n+$"; ""));
132+
133+
def strip_braces: ltrimstr("{") | rtrimstr("}");
134+
135+
# "{ 2100 }" -> ["2100"]; "{ [30, 50] }" -> ["[30,", "50]"]
136+
def input_values:
137+
strip_braces | split(" ") | map(select(length > 0));
138+
139+
def match_failure(failures):
140+
. as $test |
141+
if ($test.input | length) == 0 then null
142+
else
143+
($test.input | input_values) as $tokens |
144+
[failures[] | select(
145+
.entry == $test.entry and
146+
(.label as $lbl | $tokens | all(inside($lbl)))
147+
)] | first // null
148+
end;
149+
150+
($output | parse_failures) as $failures |
151+
[
152+
$info[] |
153+
(match_failure($failures)) as $fail |
154+
if $fail then
155+
{name: .name, status: "fail", message: $fail.message}
156+
else
157+
{name: .name, status: "pass"}
158+
end
159+
]
160+
')
161+
162+
overall=$(printf '%s' "${tests_json}" | jq -r '
163+
if any(.[]; .status == "fail") then "fail" else "pass" end
164+
')
165+
166+
jq -n --arg status "${overall}" --argjson tests "${tests_json}" \
167+
'{version: 2, status: $status, tests: $tests}' > "${results_file}"
49168
fi
50169

51170
echo "${slug}: done"

‎tests/all-fail/expected_results.json‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
{
2-
"version": 1,
2+
"version": 2,
33
"status": "fail",
4-
"message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/all-fail/test.fut:\nCompiling with --backend=c:\nRunning compiled program:\nRunning /opt/test-runner/tests/all-fail/test:\nEntry point: main; dataset: #0 (\"2015i32\"):\n/opt/test-runner/tests/all-fail/test.fut.main.0.actual and /opt/test-runner/tests/all-fail/test.fut.main.0.expected do not match:\nValue #0: expected False, got True\nEntry point: main; dataset: #0 (\"1960i32\"):\n/opt/test-runner/tests/all-fail/test.fut.main.0.actual and /opt/test-runner/tests/all-fail/test.fut.main.0.expected do not match:\nValue #0: expected True, got False\nEntry point: main; dataset: #0 (\"2100i32\"):\n/opt/test-runner/tests/all-fail/test.fut.main.0.actual and /opt/test-runner/tests/all-fail/test.fut.main.0.expected do not match:\nValue #0: expected False, got True\n0/1 passed."
4+
"tests": [
5+
{
6+
"name": "Year not divisible by 4 in common year",
7+
"status": "fail",
8+
"message": "test.fut.main.0.actual and test.fut.main.0.expected do not match:\nValue #0: expected False, got True"
9+
},
10+
{
11+
"name": "Year divisible by 4 and 5 is still a leap year",
12+
"status": "fail",
13+
"message": "test.fut.main.0.actual and test.fut.main.0.expected do not match:\nValue #0: expected True, got False"
14+
},
15+
{
16+
"name": "Year divisible by 100, not divisible by 400 in common year",
17+
"status": "fail",
18+
"message": "test.fut.main.0.actual and test.fut.main.0.expected do not match:\nValue #0: expected False, got True\n0/1 passed."
19+
}
20+
]
521
}

‎tests/empty-file/expected_results.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": 1,
3-
"status": "fail",
4-
"message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/empty-file/test.fut:\nCompiling with --backend=c:\nError at /opt/test-runner/tests/empty-file/test.fut:19:3-10:\nUnknown name \"is_leap\"\nIf you find this error message confusing, uninformative, or wrong, please open an issue:\n https://github.com/diku-dk/futhark/issues\n\n0/1 passed."
2+
"version": 2,
3+
"status": "error",
4+
"message": "0 failed, 0 passed, 1 to go.\ntest.fut:\nCompiling with --backend=c:\nError at test.fut:19:3-10:\nUnknown name \"is_leap\"\nIf you find this error message confusing, uninformative, or wrong, please open an issue:\n https://github.com/diku-dk/futhark/issues\n\n0/1 passed."
55
}

‎tests/external-package/expected_results.json‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,30 @@
11
{
2-
"version": 1,
3-
"status": "pass"
2+
"version": 2,
3+
"status": "pass",
4+
"tests": [
5+
{
6+
"name": "List of scores",
7+
"status": "pass"
8+
},
9+
{
10+
"name": "Latest score",
11+
"status": "pass"
12+
},
13+
{
14+
"name": "Personal best",
15+
"status": "pass"
16+
},
17+
{
18+
"name": "Personal top three from a list of scores",
19+
"status": "pass"
20+
},
21+
{
22+
"name": "Personal top highest to lowest",
23+
"status": "pass"
24+
},
25+
{
26+
"name": "Personal top when there is a tie",
27+
"status": "pass"
28+
}
29+
]
430
}

‎tests/partial-fail/expected_results.json‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
{
2-
"version": 1,
2+
"version": 2,
33
"status": "fail",
4-
"message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/partial-fail/test.fut:\nCompiling with --backend=c:\nRunning compiled program:\nRunning /opt/test-runner/tests/partial-fail/test:\nEntry point: main; dataset: #0 (\"2100i32\"):\n/opt/test-runner/tests/partial-fail/test.fut.main.0.actual and /opt/test-runner/tests/partial-fail/test.fut.main.0.expected do not match:\nValue #0: expected False, got True\n0/1 passed."
4+
"tests": [
5+
{
6+
"name": "Year not divisible by 4 in common year",
7+
"status": "pass"
8+
},
9+
{
10+
"name": "Year divisible by 4 and 5 is still a leap year",
11+
"status": "pass"
12+
},
13+
{
14+
"name": "Year divisible by 100, not divisible by 400 in common year",
15+
"status": "fail",
16+
"message": "test.fut.main.0.actual and test.fut.main.0.expected do not match:\nValue #0: expected False, got True\n0/1 passed."
17+
}
18+
]
519
}

‎tests/success/expected_results.json‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,18 @@
11
{
2-
"version": 1,
3-
"status": "pass"
2+
"version": 2,
3+
"status": "pass",
4+
"tests": [
5+
{
6+
"name": "Year not divisible by 4 in common year",
7+
"status": "pass"
8+
},
9+
{
10+
"name": "Year divisible by 4 and 5 is still a leap year",
11+
"status": "pass"
12+
},
13+
{
14+
"name": "Year divisible by 100, not divisible by 400 in common year",
15+
"status": "pass"
16+
}
17+
]
418
}

‎tests/syntax-error/expected_results.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": 1,
3-
"status": "fail",
4-
"message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/syntax-error/test.fut:\nCompiling with --backend=c:\nError at /opt/test-runner/tests/syntax-error/syntax_error.fut:1:12-13:\nUnexpected token: '#'\nExpected one of the following:\n\nIf you find this error message confusing, uninformative, or wrong, please open an issue:\n https://github.com/diku-dk/futhark/issues\n\n0/1 passed."
2+
"version": 2,
3+
"status": "error",
4+
"message": "0 failed, 0 passed, 1 to go.\ntest.fut:\nCompiling with --backend=c:\nError at syntax_error.fut:1:12-13:\nUnexpected token: '#'\nExpected one of the following:\n\nIf you find this error message confusing, uninformative, or wrong, please open an issue:\n https://github.com/diku-dk/futhark/issues\n\n0/1 passed."
55
}

0 commit comments

Comments
 (0)