Skip to content

Commit 6ac9fcd

Browse files
edelaunaedelauna
committed
fix(smoke): keep startup timeouts in isolated controller
1 parent ca8b033 commit 6ac9fcd

8 files changed

Lines changed: 592 additions & 384 deletions

File tree

‎scripts/solheim-smoke-workflow.test.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ describe("Project A provider smoke workflow", () => {
5656
it("bounds runtime and serializes the single provider instance", () => {
5757
assert.match(workflow, /cancel-in-progress: false/)
5858
assert.match(workflow, /timeout-minutes: 8/)
59+
assert.match(workflow, /^ {8}timeout-minutes: 25$/m)
5960
})
6061
it("uploads only the verdict and video, never host logs or storage", () => {
6162
const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0]

‎scripts/solheim-smoke/README.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@ The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8
1818
groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary
1919
workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is
2020
supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage
21-
can contain the configured credential while the session runs; it is removed at teardown and never uploaded.
21+
can contain the configured credential while the session runs. It is removed when no child remains alive;
22+
if termination fails, storage cleanup is skipped and `gates.childExited` reports that separately from the
23+
original smoke failure. Storage is never uploaded.
2224
This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only.
2325

2426
Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit,
25-
with 30-second per-stage bounds in the extension. The live CI step has an eight-minute outer timeout.
27+
in the controller. The extension awaits startup and configuration persistence without racing shared-profile
28+
writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than
29+
trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout.
2630
Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty
2731
completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass.
2832
The driver never approves dialogs or tool execution; it closes the task during teardown.

‎scripts/solheim-smoke/config.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ export function buildChildEnvironment(
6464
XDG_CONFIG_HOME: path.join(isolatedHome, ".config"),
6565
XDG_CACHE_HOME: path.join(isolatedHome, ".cache"),
6666
ROO_CODE_IPC_SOCKET_PATH: socketPath,
67-
ROO_CODE_TASK_START_STAGE_TIMEOUT_MS: "30000",
6867
}
6968
}
7069

‎scripts/solheim-smoke/driver.mts‎

Lines changed: 101 additions & 161 deletions
Original file line numberDiff line numberDiff line change
@@ -1,181 +1,121 @@
11
import fs from "node:fs/promises"
22
import os from "node:os"
33
import path from "node:path"
4-
import { randomUUID } from "node:crypto"
54
import { fileURLToPath } from "node:url"
6-
import { spawn, type ChildProcess } from "node:child_process"
5+
import { spawn } from "node:child_process"
76
import { downloadAndUnzipVSCode } from "@vscode/test-electron"
87
import { IpcClient } from "@roo-code/ipc"
9-
import { IpcMessageType, TaskCommandName, type TaskEvent } from "@roo-code/types"
10-
import {
11-
buildChildEnvironment,
12-
buildLaunchArgs,
13-
buildSmokeConfiguration,
14-
SMOKE_PROMPT,
15-
SMOKE_TIMEOUT_MS,
16-
SOLHEIM_BASE_URL,
17-
SOLHEIM_MODEL,
18-
} from "./config.ts"
19-
import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts"
20-
import { createTaskStartWaiter } from "./task-start.ts"
8+
import { IpcMessageType } from "@roo-code/types"
9+
import { buildChildEnvironment, buildLaunchArgs } from "./config.ts"
10+
import { runSmoke, smokeExitCode, type SmokeRuntime } from "./run.ts"
2111

2212
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")
2313
const out = process.env.SOLHEIM_SMOKE_OUT_DIR
2414
if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory")
2515
await fs.mkdir(out, { recursive: true })
26-
const started = Date.now()
27-
const evidence: SmokeEvidence = {
28-
providerResponseSeen: false,
29-
completionSeen: false,
30-
unexpectedTool: false,
31-
aborted: false,
32-
}
33-
let code: SmokeCode = "SETUP_FAILED"
34-
let activated = false
35-
let accepted = false
36-
let sessionLost = false
37-
let child: ChildProcess | undefined
38-
let client: IpcClient | undefined
3916
let temp: string | undefined
40-
let exited = false
41-
let waiter: ReturnType<typeof createTaskStartWaiter> | undefined
42-
const pause = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
43-
async function until(check: () => boolean, timeoutMs: number, stopOnSessionLoss = true): Promise<boolean> {
44-
const deadline = Date.now() + timeoutMs
45-
while (!check() && (!stopOnSessionLoss || !sessionLost) && Date.now() < deadline) await pause(100)
46-
return check()
47-
}
48-
function signalChild(signal: NodeJS.Signals): void {
49-
if (!child?.pid || exited) return
50-
try {
51-
process.kill(-child.pid, signal)
52-
} catch {
53-
child.kill(signal)
54-
}
55-
}
56-
function interrupted(): void {
57-
sessionLost = true
58-
waiter?.onExit()
59-
signalChild("SIGTERM")
60-
}
61-
process.once("SIGINT", interrupted)
62-
process.once("SIGTERM", interrupted)
63-
64-
try {
65-
const apiKey = process.env.SOLHEIM_API_KEY
66-
if (!apiKey) throw new Error("Provider credential missing")
67-
const extension = path.join(root, "src")
68-
await fs.access(path.join(extension, "package.json"))
69-
temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-"))
70-
const workspace = path.join(temp, "workspace")
71-
const home = path.join(temp, "home")
72-
await Promise.all([workspace, home].map((folder) => fs.mkdir(folder)))
73-
const socket = path.join(temp, "ipc.sock")
74-
const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as {
75-
devDependencies: Record<string, string>
76-
}
77-
const executable = await downloadAndUnzipVSCode({
78-
version: pkg.devDependencies["@types/vscode"] ?? "1.100.0",
79-
cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"),
80-
})
81-
const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension)
82-
child = process.env.DISPLAY
83-
? spawn(executable, args, {
84-
detached: true,
85-
env: buildChildEnvironment(process.env, home, socket),
86-
stdio: "ignore",
87-
})
88-
: spawn("xvfb-run", ["-a", executable, ...args], {
17+
let socket = ""
18+
const runtime: SmokeRuntime = {
19+
async launch() {
20+
const extension = path.join(root, "src")
21+
await fs.access(path.join(extension, "package.json"))
22+
temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-"))
23+
const workspace = path.join(temp, "workspace")
24+
const isolatedHome = path.join(temp, "home")
25+
await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder)))
26+
socket = path.join(temp, "ipc.sock")
27+
const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as {
28+
devDependencies: Record<string, string>
29+
}
30+
const executable = await downloadAndUnzipVSCode({
31+
version: pkg.devDependencies["@types/vscode"] ?? "1.100.0",
32+
cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"),
33+
})
34+
const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension)
35+
const child = spawn(
36+
process.env.DISPLAY ? executable : "xvfb-run",
37+
process.env.DISPLAY ? args : ["-a", executable, ...args],
38+
{
8939
detached: true,
90-
env: buildChildEnvironment(process.env, home, socket),
40+
env: buildChildEnvironment(process.env, isolatedHome, socket),
9141
stdio: "ignore",
92-
})
93-
child.once("error", () => {
94-
sessionLost = true
95-
exited = true
96-
waiter?.onExit()
97-
})
98-
child.once("close", () => {
99-
sessionLost = true
100-
exited = true
101-
waiter?.onExit()
102-
})
103-
client = new IpcClient(socket, () => {})
104-
client.on(IpcMessageType.Disconnect, () => {
105-
sessionLost = true
106-
waiter?.onDisconnect()
107-
})
108-
let taskId: string | undefined
109-
client.on(IpcMessageType.TaskEvent, (event: TaskEvent) => {
110-
const id = waiter?.onTaskEvent(event)
111-
if (id && taskId === undefined) taskId = id
112-
collectEvidence(evidence, event, taskId)
113-
})
114-
activated = await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS)
115-
if (!activated) code = "ACTIVATION_FAILED"
116-
else {
117-
const requestId = randomUUID()
118-
waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 })
119-
client.sendCommand({
120-
commandName: TaskCommandName.StartNewTask,
121-
data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) },
122-
})
123-
try {
124-
taskId = await waiter.result
125-
accepted = true
126-
} catch {
127-
code = "TASK_START_FAILED"
42+
},
43+
)
44+
let exited = false
45+
let onExit = () => {}
46+
const closed = () => {
47+
exited = true
48+
onExit()
12849
}
129-
if (accepted) {
130-
await until(
131-
() =>
132-
evidence.unexpectedTool ||
133-
evidence.aborted ||
134-
(evidence.providerResponseSeen && evidence.completionSeen),
135-
SMOKE_TIMEOUT_MS,
136-
)
137-
code = smokeCode(evidence, sessionLost)
50+
child.once("error", () => {
51+
// Spawn failure has no live PID; a failed kill must not claim exit.
52+
if (!child.pid) closed()
53+
else onExit()
54+
})
55+
child.once("close", closed)
56+
return {
57+
get exited() {
58+
return exited
59+
},
60+
onExit(listener) {
61+
onExit = listener
62+
if (exited) listener()
63+
},
64+
signal(signal) {
65+
if (!child.pid || exited) return
66+
try {
67+
process.kill(-child.pid, signal)
68+
} catch {
69+
child.kill(signal)
70+
}
71+
},
13872
}
139-
}
140-
} catch {
141-
/* Keep failures fixed and credential-free, never print thrown provider/host text. */
142-
} finally {
143-
// Teardown is independently bounded, including spawn/activation failures.
144-
if (client?.isReady) client.sendCommand({ commandName: TaskCommandName.CloseTask })
145-
if (child && !exited) {
146-
await until(() => exited, 5_000, false)
147-
if (!exited) {
148-
signalChild("SIGTERM")
149-
await until(() => exited, 5_000, false)
73+
},
74+
connect() {
75+
const client = new IpcClient(socket, () => {})
76+
return {
77+
get isReady() {
78+
return client.isReady
79+
},
80+
onTaskEvent(listener) {
81+
client.on(IpcMessageType.TaskEvent, listener)
82+
},
83+
onDisconnect(listener) {
84+
client.on(IpcMessageType.Disconnect, listener)
85+
},
86+
sendCommand(command) {
87+
client.sendCommand(command)
88+
},
89+
disconnect() {
90+
client.disconnect()
91+
},
15092
}
151-
if (!exited) {
152-
signalChild("SIGKILL")
153-
await until(() => exited, 5_000, false)
93+
},
94+
async cleanup() {
95+
if (temp) await fs.rm(temp, { recursive: true, force: true })
96+
},
97+
async writeVerdict(verdict) {
98+
// Atomic, whitelisted metadata only; no provider text or host logs.
99+
const pending = path.join(out, `verdict.json.${process.pid}.tmp`)
100+
await fs.writeFile(pending, JSON.stringify(verdict, null, 2))
101+
await fs.rename(pending, path.join(out, "verdict.json"))
102+
},
103+
onInterrupt(listener) {
104+
process.once("SIGINT", listener)
105+
process.once("SIGTERM", listener)
106+
return () => {
107+
process.removeListener("SIGINT", listener)
108+
process.removeListener("SIGTERM", listener)
154109
}
155-
if (!exited && code === "OK") code = "TEARDOWN_FAILED"
156-
}
157-
client?.disconnect()
158-
if (temp && (!child || exited)) await fs.rm(temp, { recursive: true, force: true })
159-
}
160-
// Whitelisted metadata only: model answers and host logs are never saved or uploaded.
161-
const verdict = {
162-
kind: "provider-smoke",
163-
model: SOLHEIM_MODEL,
164-
provider: SOLHEIM_BASE_URL,
165-
passed: code === "OK",
166-
code,
167-
gates: {
168-
extensionActivated: activated,
169-
ipcConnected: activated,
170-
taskAccepted: accepted,
171-
providerResponseSeen: evidence.providerResponseSeen,
172-
completionSeen: evidence.completionSeen,
173110
},
174-
durationMs: Date.now() - started,
111+
now: Date.now,
112+
pause: (ms) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
113+
}
114+
try {
115+
const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "")
116+
console.log(`Solheim provider smoke: ${verdict.code}`)
117+
process.exit(smokeExitCode(verdict))
118+
} catch {
119+
console.log("Solheim provider smoke: SETUP_FAILED")
120+
process.exit(1)
175121
}
176-
// Write then rename, so a reader sees the old or the new verdict and never a partial one.
177-
const verdictTemp = path.join(out, `verdict.json.${process.pid}.tmp`)
178-
await fs.writeFile(verdictTemp, JSON.stringify(verdict, null, 2))
179-
await fs.rename(verdictTemp, path.join(out, "verdict.json"))
180-
console.log(`Solheim provider smoke: ${code}`)
181-
process.exit(code === "OK" ? 0 : 1)

0 commit comments

Comments
 (0)