From 3d2f042f9d0f1d7e3097364c76803f3c49deb494 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 5 Oct 2026 02:57:24 +0000 Subject: [PATCH 01/11] feat(ci): add recorded Solheim provider smoke test --- .github/workflows/code-qa.yml | 6 + .github/workflows/solheim-provider-smoke.yml | 60 ++++ package.json | 7 + packages/types/src/__tests__/events.test.ts | 234 ++++++++++++++ packages/types/src/__tests__/ipc.test.ts | 55 ++++ packages/types/src/events.ts | 57 ++++ packages/types/src/ipc.ts | 4 +- pnpm-lock.yaml | 9 + scripts/solheim-smoke-workflow.test.mjs | 139 ++++++++ scripts/solheim-smoke/README.md | 56 ++++ scripts/solheim-smoke/config.test.ts | 65 ++++ scripts/solheim-smoke/config.ts | 85 +++++ scripts/solheim-smoke/driver.mts | 181 +++++++++++ scripts/solheim-smoke/evidence.test.ts | 91 ++++++ scripts/solheim-smoke/evidence.ts | 62 ++++ scripts/solheim-smoke/package.json | 4 + scripts/solheim-smoke/record.sh | 44 +++ scripts/solheim-smoke/task-start.test.ts | 215 +++++++++++++ scripts/solheim-smoke/task-start.ts | 96 ++++++ scripts/solheim-smoke/tsconfig.json | 17 + .../__tests__/api-start-task-logging.spec.ts | 296 ++++++++++++++++++ src/extension/api.ts | 176 +++++++++-- 22 files changed, 1933 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/solheim-provider-smoke.yml create mode 100644 packages/types/src/__tests__/events.test.ts create mode 100644 scripts/solheim-smoke-workflow.test.mjs create mode 100644 scripts/solheim-smoke/README.md create mode 100644 scripts/solheim-smoke/config.test.ts create mode 100644 scripts/solheim-smoke/config.ts create mode 100644 scripts/solheim-smoke/driver.mts create mode 100644 scripts/solheim-smoke/evidence.test.ts create mode 100644 scripts/solheim-smoke/evidence.ts create mode 100644 scripts/solheim-smoke/package.json create mode 100644 scripts/solheim-smoke/record.sh create mode 100644 scripts/solheim-smoke/task-start.test.ts create mode 100644 scripts/solheim-smoke/task-start.ts create mode 100644 scripts/solheim-smoke/tsconfig.json create mode 100644 src/extension/__tests__/api-start-task-logging.spec.ts diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index f4508f270e..e8cf511450 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -98,6 +98,12 @@ jobs: run: pnpm check-types - name: Validate Code QA workflow run: pnpm test:code-qa-ci + - name: Validate Solheim provider smoke workflow + run: pnpm test:solheim-smoke-ci + - name: Test Solheim provider smoke units + run: pnpm test:solheim-smoke:unit + - name: Check Solheim provider smoke types + run: pnpm solheim-smoke:check-types - name: Model-check task lifecycle protocols run: pnpm lifecycle:model-check - name: Validate MCP OAuth integration diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml new file mode 100644 index 0000000000..7f61107186 --- /dev/null +++ b/.github/workflows/solheim-provider-smoke.yml @@ -0,0 +1,60 @@ +name: Solheim provider smoke + +# Project A: infrastructure validation only. No PR diff, review judgment, router, +# deterministic review rule, or GitHub posting. Only trusted main-branch code runs. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: solheim-provider-smoke + cancel-in-progress: false + +jobs: + smoke: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + # Retain the existing protected environment and secret names for compatibility. + # Repository policy must restrict deployments to main and require approval. + environment: final-vscode-review-smoke + steps: + - name: Checkout exact trusted main revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup-node-pnpm + with: + install-args: "--frozen-lockfile" + + - name: Build trusted extension and webview + run: | + pnpm -w bundle + pnpm --filter @roo-code/vscode-webview build + + - name: Install display dependencies + run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 + + - name: Run Solheim provider smoke + id: provider + timeout-minutes: 8 + env: + SOLHEIM_API_KEY: ${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }} + SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke + run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh + + - name: Upload provider smoke verdict and recording + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: solheim-provider-smoke-${{ github.sha }} + path: | + ${{ runner.temp }}/solheim-provider-smoke/verdict.json + ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 + if-no-files-found: error + retention-days: 7 diff --git a/package.json b/package.json index 4ca0b672c1..0fcde2ca98 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,10 @@ "check-types": "turbo check-types --log-order grouped --output-logs new-only", "test": "turbo test --log-order grouped --output-logs new-only", "test:code-qa-ci": "node --test scripts/code-qa-workflow.test.mjs", + "solheim:smoke": "node --import tsx scripts/solheim-smoke/driver.mts", + "test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs", + "test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts", + "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", @@ -38,6 +42,9 @@ "devDependencies": { "@changesets/cli": "2.31.1", "@roo-code/config-typescript": "workspace:^", + "@roo-code/ipc": "workspace:^", + "@roo-code/types": "workspace:^", + "@vscode/test-electron": "2.5.2", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", "@types/node": "22.20.1", diff --git a/packages/types/src/__tests__/events.test.ts b/packages/types/src/__tests__/events.test.ts new file mode 100644 index 0000000000..3525e75eac --- /dev/null +++ b/packages/types/src/__tests__/events.test.ts @@ -0,0 +1,234 @@ +import { + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + TASK_START_STAGES, + taskEventSchema, + taskStartResponseSchema, +} from "../events.js" + +const requestId = "final-smoke-0123abcdef456789" + +describe("taskStartResponseSchema", () => { + it("accepts a success response with the accepted task ID", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true, taskId: "task-1" }) + expect(result.success).toBe(true) + + if (result.success) { + expect(result.data).toEqual({ requestId, success: true, taskId: "task-1" }) + } + }) + + it("accepts the fixed failure response", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.success).toBe(false) + expect(result.data.stage).toBeUndefined() + } + }) + + it("accepts a failure response carrying each allowed stage", () => { + for (const stage of TASK_START_STAGES) { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.stage).toBe(stage) + } + } + }) + + it("rejects a failure response carrying an unknown stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "secretStore", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a non-string stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: { name: "taskCreation" }, + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response carrying a stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + stage: "taskCreation", + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response without a taskId", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying an unfixed error code", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: "provider_auth_failed", + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a raw error message", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: "openai rejected the key sk-secret-123 at https://api.example.com", + }) + expect(result.success).toBe(false) + }) + + it("rejects an empty requestId", () => { + const result = taskStartResponseSchema.safeParse({ requestId: "", success: true, taskId: "task-1" }) + expect(result.success).toBe(false) + }) + + it("rejects a success response with an extra top-level field", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + providerError: "openai rejected the key sk-secret-123", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response with an extra field on the error object", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }) + expect(result.success).toBe(false) + }) +}) + +describe("taskEventSchema TaskStartResponse", () => { + it("routes a success response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1" }], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response with a stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("rejects a failure payload with an unknown stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "keyring", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with a stage plus an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a success payload with an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1", stack: "at run (api.ts:1:1)" }], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with an extra error field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) +}) diff --git a/packages/types/src/__tests__/ipc.test.ts b/packages/types/src/__tests__/ipc.test.ts index a843354a55..daf9b3cb1d 100644 --- a/packages/types/src/__tests__/ipc.test.ts +++ b/packages/types/src/__tests__/ipc.test.ts @@ -117,4 +117,59 @@ describe("IPC Types", () => { expect(result.success).toBe(false) }) }) + + describe("StartNewTask requestId", () => { + const legacyCommand = { + commandName: TaskCommandName.StartNewTask, + data: { + configuration: {}, + text: "review the diff", + }, + } + + it("should accept a legacy StartNewTask without requestId", () => { + const result = taskCommandSchema.safeParse(legacyCommand) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBeUndefined() + } + }) + + it("should accept and preserve a valid requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "final-smoke-0123abcdef456789" }, + }) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBe("final-smoke-0123abcdef456789") + } + }) + + it("should reject an empty requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "" }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId longer than 128 characters", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "a".repeat(129) }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId with characters outside the allowed set", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "id with spaces" }, + }) + expect(result.success).toBe(false) + }) + }) }) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 20f7f7e71e..204256980d 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -51,8 +51,59 @@ export enum RooCodeEventName { CommandsResponse = "commandsResponse", ModesResponse = "modesResponse", ModelsResponse = "modelsResponse", + /** Correlated reply to a StartNewTask command that carried a requestId. */ + TaskStartResponse = "taskStartResponse", } +/** + * TaskStartResponse + */ + +// The requestId is generated by the command sender and copied into the reply, +// so its shape is bounded and restricted to characters that cannot smuggle +// log structure. +export const taskStartRequestIdSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9._:-]+$/) + +// The failure values are fixed constants. The response never carries raw +// thrown messages, prompts, images, provider settings, secrets, URLs, or +// stack traces. Both variants reject unknown keys, so an extra or +// secret-bearing field fails validation instead of being stripped. +export const TASK_START_FAILURE_ERROR_CODE = "task_start_failed" +export const TASK_START_FAILURE_ERROR_MESSAGE = "Task start failed" + +// Fixed stage vocabulary for a failed correlated start. The stage names the +// startup phase that stopped responding; it carries no error content. +export const TASK_START_STAGES = ["eviction", "settings", "taskCreation", "unknown"] as const + +export const taskStartStageSchema = z.enum(TASK_START_STAGES) + +export type TaskStartStage = (typeof TASK_START_STAGES)[number] + +export const taskStartResponseSchema = z.discriminatedUnion("success", [ + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(true), + taskId: z.string().min(1), + }) + .strict(), + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(false), + errorCode: z.literal(TASK_START_FAILURE_ERROR_CODE), + errorMessage: z.literal(TASK_START_FAILURE_ERROR_MESSAGE), + stage: taskStartStageSchema.optional(), + }) + .strict(), +]) + +export type TaskStartResponse = z.infer + /** * RooCodeEvents */ @@ -125,6 +176,7 @@ export const rooCodeEventsSchema = z.object({ ]), [RooCodeEventName.ModesResponse]: z.tuple([z.array(z.object({ slug: z.string(), name: z.string() }))]), [RooCodeEventName.ModelsResponse]: z.tuple([z.record(z.string(), modelInfoSchema)]), + [RooCodeEventName.TaskStartResponse]: z.tuple([taskStartResponseSchema]), }) export type RooCodeEvents = z.infer @@ -270,6 +322,11 @@ export const taskEventSchema = z.discriminatedUnion("eventName", [ payload: rooCodeEventsSchema.shape[RooCodeEventName.ModelsResponse], taskId: z.number().optional(), }), + z.object({ + eventName: z.literal(RooCodeEventName.TaskStartResponse), + payload: rooCodeEventsSchema.shape[RooCodeEventName.TaskStartResponse], + taskId: z.number().optional(), + }), ]) export type TaskEvent = z.infer diff --git a/packages/types/src/ipc.ts b/packages/types/src/ipc.ts index fea040af0b..e7df762069 100644 --- a/packages/types/src/ipc.ts +++ b/packages/types/src/ipc.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { type TaskEvent, taskEventSchema } from "./events.js" +import { type TaskEvent, taskEventSchema, taskStartRequestIdSchema } from "./events.js" import { rooCodeSettingsSchema } from "./global-settings.js" /** @@ -64,6 +64,8 @@ export const taskCommandSchema = z.discriminatedUnion("commandName", [ text: z.string(), images: z.array(z.string()).optional(), newTab: z.boolean().optional(), + // Senders without a requestId keep the fire-and-forget contract. + requestId: taskStartRequestIdSchema.optional(), }), }), z.object({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db4358d6e7..58d45a98b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,12 @@ importers: '@roo-code/config-typescript': specifier: workspace:^ version: link:packages/config-typescript + '@roo-code/ipc': + specifier: workspace:^ + version: link:packages/ipc + '@roo-code/types': + specifier: workspace:^ + version: link:packages/types '@stryker-mutator/core': specifier: 10.0.0 version: 10.0.0(@types/node@22.20.1) @@ -44,6 +50,9 @@ importers: '@types/node': specifier: 22.20.1 version: 22.20.1 + '@vscode/test-electron': + specifier: 2.5.2 + version: 2.5.2 '@vscode/vsce': specifier: 3.9.2 version: 3.9.2 diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs new file mode 100644 index 0000000000..1239cfe5a5 --- /dev/null +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -0,0 +1,139 @@ +import assert from "node:assert/strict" +import fs from "node:fs" +import { URL } from "node:url" +import { describe, it } from "node:test" +import { spawnSync } from "node:child_process" +import os from "node:os" +import path from "node:path" +const workflow = fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8") +const codeQa = fs.readFileSync(new URL("../.github/workflows/code-qa.yml", import.meta.url), "utf8") +const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8")) +const driver = fs.readFileSync(new URL("./solheim-smoke/driver.mts", import.meta.url), "utf8") +const recorder = fs.readFileSync(new URL("./solheim-smoke/record.sh", import.meta.url), "utf8") +const smokePkg = JSON.parse(fs.readFileSync(new URL("./solheim-smoke/package.json", import.meta.url), "utf8")) + +describe("Project A provider smoke workflow", () => { + it("runs smoke TypeScript as ESM without requiring compiled workspace packages", () => { + assert.equal(smokePkg.type, "module") + assert.equal(smokePkg.private, true) + }) + it("runs only by manual dispatch against the exact trusted main revision", () => { + assert.match(workflow, /workflow_dispatch:/) + assert.match(workflow, /if: github.ref == 'refs\/heads\/main'/) + assert.match(workflow, /ref: \$\{\{ github.sha \}\}/) + assert.ok(!workflow.includes("pull_request_review:")) + assert.ok(!workflow.includes("pull_request_target:")) + assert.match(workflow, /persist-credentials: false/) + assert.match(workflow, /environment: final-vscode-review-smoke/) + }) + it("has one credential-bearing step, no GitHub write permissions, and no posting", () => { + assert.equal((workflow.match(/secrets\./g) ?? []).length, 1) + const before = workflow.slice(0, workflow.indexOf("- name: Run Solheim provider smoke")) + assert.ok(!before.includes("secrets.")) + assert.match(workflow, /permissions:\n\s+contents: read/) + assert.ok(!/\bwrite\b/.test(workflow)) + assert.ok(!workflow.includes("github-script")) + assert.ok(!workflow.includes("post-review")) + }) + it("has one smoke job and no review, routing, shadow, or deterministic lanes", () => { + const jobs = workflow.split("jobs:\n")[1] ?? "" + assert.deepEqual( + [...jobs.matchAll(/^ {4}([a-z-]+):$/gm)].map((match) => match[1]), + ["smoke"], + ) + for (const name of [ + "theme-review", + "probe-dispatch", + "rules-review", + "merge-report", + "bounded-code-review", + "model-qualification", + ]) + assert.ok(!workflow.includes(name)) + assert.ok(!driver.includes("model-qualification")) + assert.ok(!driver.includes("AcknowledgeCompletion")) + }) + it("bounds runtime and serializes the single provider instance", () => { + assert.match(workflow, /cancel-in-progress: false/) + assert.match(workflow, /timeout-minutes: 8/) + }) + it("uploads only the verdict and video, never host logs or storage", () => { + const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] + assert.deepEqual( + paths + ?.trim() + .split("\n") + .map((line) => line.trim()), + [ + "${{ runner.temp }}/solheim-provider-smoke/verdict.json", + "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", + ], + ) + assert.match(workflow, /!cancelled\(\)/) + assert.match(workflow, /if-no-files-found: error/) + assert.ok(!driver.includes("console.error(")) + assert.ok(!driver.includes("stderrLog")) + }) + it("records a fresh display with credential-free ffmpeg and bounded teardown", () => { + assert.match(workflow, /install -y xvfb ffmpeg/) + assert.match(workflow, /xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp"/) + assert.match(recorder, /env -i "PATH=\$PATH" "DISPLAY=\$DISPLAY"/) + assert.match(recorder, /-t 480 -fs 125829120/) + assert.match(recorder, /frag_keyframe\+empty_moov/) + assert.match(recorder, /trap finish EXIT/) + assert.match(recorder, /attempt < 50/) + assert.ok(!recorder.includes("SOLHEIM_API_KEY")) + assert.match(workflow, /retention-days: 7/) + }) + it("keeps Code QA and package commands focused on smoke contracts", () => { + for (const command of ["test:solheim-smoke:unit", "test:solheim-smoke-ci", "solheim-smoke:check-types"]) + assert.ok(codeQa.includes(command)) + assert.ok(!codeQa.includes("model-qualification")) + assert.ok(!Object.keys(pkg.scripts).some((name) => name.includes("model-qualification"))) + assert.ok(!codeQa.includes("final-smoke")) + assert.equal(pkg.scripts["solheim:smoke"], "node --import tsx scripts/solheim-smoke/driver.mts") + assert.ok(!fs.existsSync(new URL("../.github/workflows/final-vscode-review-smoke.yml", import.meta.url))) + }) +}) + +// Runs record.sh with stub ffmpeg and ffprobe, so no display or codec is needed. +function runRecorder(args, { ffmpegWrites = true, ffprobeOk = true } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = path.join(dir, "bin") + fs.mkdirSync(bin) + const stub = (name, body) => + fs.writeFileSync(path.join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 }) + // The stub ffmpeg writes the last argument, stops on INT, and exits. + stub( + "ffmpeg", + `${ffmpegWrites ? 'for a; do :; done; echo data > "$a"' : ":"}\ntrap 'exit 0' INT\nwhile true; do sleep 0.05; done`, + ) + stub("ffprobe", ffprobeOk ? "exit 0" : "exit 1") + const result = spawnSync("bash", [new URL("./solheim-smoke/record.sh", import.meta.url).pathname, ...args], { + env: { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":99", SOLHEIM_SMOKE_OUT_DIR: path.join(dir, "out") }, + encoding: "utf8", + timeout: 20_000, + }) + fs.rmSync(dir, { recursive: true, force: true }) + return result +} + +describe("record.sh exit and validation behavior", () => { + it("passes when the command succeeds and the recording is valid", () => { + assert.equal(runRecorder(["true"]).status, 0) + }) + it("keeps the exit code of a failing command", () => { + assert.equal(runRecorder(["bash", "-c", "exit 7"]).status, 7) + }) + it("fails when the recording is missing", () => { + const result = runRecorder(["true"], { ffmpegWrites: false }) + assert.equal(result.status, 1) + assert.match(result.stdout, /recording failed/) + }) + it("fails when ffprobe rejects the recording", () => { + assert.equal(runRecorder(["true"], { ffprobeOk: false }).status, 1) + }) + it("keeps a failing command code when the recording is also invalid", () => { + assert.equal(runRecorder(["bash", "-c", "exit 7"], { ffprobeOk: false }).status, 7) + }) +}) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md new file mode 100644 index 0000000000..35e7189138 --- /dev/null +++ b/scripts/solheim-smoke/README.md @@ -0,0 +1,56 @@ +# Project A: Solheim provider smoke + +Infrastructure validation only: a real VS Code extension activates, accepts a correlated IPC task, +receives output from Solheim's OpenAI-compatible provider, and produces a completion result. +It does not read a PR, assess an answer's meaning, select checks, find defects, or post to GitHub. + +The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to +`refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the +single credential-bearing step. There is one job, no PR input or approval gate, no review/posting job, +and `contents: read` is the only GitHub permission. This deliberately validates the provider integration +on main, not arbitrary PR artifacts. It is not a required PR review check. + +The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOKE_OPENAI_API_KEY` +retain their existing names to avoid an unnecessary repository-settings migration. Restrict its deployment +policy to main and require approval. No environment or secret settings have been changed by this cleanup. + +The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool +groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary +workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is +supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage +can contain the configured credential while the session runs; it is removed at teardown and never uploaded. +This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only. + +Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit, +with 30-second per-stage bounds in the extension. The live CI step has an eight-minute outer timeout. +Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. +The driver never approves dialogs or tool execution; it closes the task during teardown. + +The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names) +and `smoke.mp4`, a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video +contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict, +it is not text-redacted. No settings screen is opened, and no host logs, storage, task configuration or +credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt interruption. +Ordinary success and failure runs upload both artifacts; cancellation may prevent finalization/upload. + +Checks, without a provider key or network calls: + +```sh +pnpm test:solheim-smoke:unit +pnpm test:solheim-smoke-ci +pnpm solheim-smoke:check-types +``` + +For an explicitly authorized local live run, build the extension/webview and supply `SOLHEIM_API_KEY` +through the environment, never an argument. Set `SOLHEIM_SMOKE_OUT_DIR` to a scratch directory and run +`pnpm solheim:smoke`. Linux needs Xvfb or an existing display. Do not run an untrusted extension with a key. +To record locally, install ffmpeg and run `xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" +bash scripts/solheim-smoke/record.sh` instead. Do not attach the recorder to your existing desktop. + +The new simplified driver/workflow has not yet had a real-provider CI run. Earlier question-lane runs +established the underlying integration, not this refactor's live status. Review qualification stopped +after the frozen Stage 1B pilot produced zero incremental verified defects. Its research harness and +receipts are archived outside this smoke-only change; the chronology, results, caveats and preservation +details are recorded in [tracking issue #1896](https://github.com/Zoo-Code-Org/Zoo-Code/issues/1896). diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts new file mode 100644 index 0000000000..345062f0d3 --- /dev/null +++ b/scripts/solheim-smoke/config.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + buildChildEnvironment, + buildLaunchArgs, + buildSmokeConfiguration, + SOLHEIM_BASE_URL, + SOLHEIM_MODEL, + SMOKE_PROMPT, +} from "./config.ts" + +describe("Solheim provider smoke configuration", () => { + it("uses one fixed provider and a small output budget", () => { + const config = buildSmokeConfiguration("test-credential") + assert.equal(config.openAiBaseUrl, SOLHEIM_BASE_URL) + assert.equal(config.openAiModelId, SOLHEIM_MODEL) + assert.equal(config.openAiApiKey, "test-credential") + assert.equal(config.modelMaxTokens, 512) + assert.deepEqual(JSON.parse(config.openAiExtraBody ?? ""), { chat_template_kwargs: { enable_thinking: false } }) + }) + it("is a no-tools infrastructure task, with no delegation or reviewer instructions", () => { + const config = buildSmokeConfiguration("test-credential") + assert.deepEqual(config.customModes?.[0]?.groups, []) + assert.equal(config.mode, "provider-smoke") + for (const name of [ + "alwaysAllowWrite", + "alwaysAllowExecute", + "alwaysAllowMcp", + "alwaysAllowSubtasks", + "alwaysAllowReadOnly", + ] as const) + assert.equal(config[name], false) + assert.ok(config.disabledTools?.includes("new_task")) + assert.ok(config.disabledTools?.includes("ask_followup_question")) + assert.match(SMOKE_PROMPT, /not a code review/) + }) + it("passes only allowlisted host variables into the isolated extension host", () => { + const env = buildChildEnvironment( + { + PATH: "/bin", + DISPLAY: ":1", + SOLHEIM_API_KEY: "secret", + GITHUB_TOKEN: "token", + NODE_OPTIONS: "--require=bad.js", + XDG_CONFIG_HOME: "/real-home", + EXTRA_CREDENTIAL: "other", + }, + "/tmp/isolated", + "/tmp/ipc", + ) + assert.equal(env.PATH, "/bin") + assert.equal(env.DISPLAY, ":1") + assert.equal(env.HOME, "/tmp/isolated") + assert.equal(env.XDG_CONFIG_HOME, "/tmp/isolated/.config") + for (const name of ["SOLHEIM_API_KEY", "GITHUB_TOKEN", "NODE_OPTIONS", "EXTRA_CREDENTIAL"]) + assert.equal(env[name], undefined) + }) + it("uses isolated storage and a keyring-free launch", () => { + const args = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.equal(args[0], "/empty-workspace") + assert.ok(args.includes("--password-store=basic")) + assert.ok(args.includes("--extensionDevelopmentPath=/extension")) + assert.ok(args.includes("--user-data-dir=/data")) + }) +}) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts new file mode 100644 index 0000000000..69c5d6aa71 --- /dev/null +++ b/scripts/solheim-smoke/config.ts @@ -0,0 +1,85 @@ +import path from "node:path" +import type { RooCodeSettings } from "@roo-code/types" + +export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" +export const SOLHEIM_MODEL = "qwen3.8-27b" +export const SMOKE_TIMEOUT_MS = 120_000 +export const SMOKE_PROMPT = + "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." + +export function buildSmokeConfiguration(apiKey: string): RooCodeSettings { + return { + apiProvider: "openai", + openAiBaseUrl: SOLHEIM_BASE_URL, + openAiApiKey: apiKey, + openAiModelId: SOLHEIM_MODEL, + openAiR1FormatEnabled: true, + openAiExtraBody: JSON.stringify({ chat_template_kwargs: { enable_thinking: false } }), + modelMaxTokens: 512, + autoCondenseContext: false, + mode: "provider-smoke", + customModes: [ + { + slug: "provider-smoke", + name: "Provider smoke", + roleDefinition: "Complete one connectivity smoke task. You are not a reviewer.", + groups: [], + }, + ], + disabledTools: [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", + ], + autoApprovalEnabled: true, + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + } +} + +// Inherit only host-launch necessities, never the credential-bearing parent environment. +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, + socketPath: string, +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of ["PATH", "DISPLAY", "XAUTHORITY", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT", "WINDIR"]) + if (env[name]) child[name] = env[name] + return { + ...child, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), + ROO_CODE_IPC_SOCKET_PATH: socketPath, + ROO_CODE_TASK_START_STAGE_TIMEOUT_MS: "30000", + } +} + +export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { + return [ + workspace, + `--user-data-dir=${userData}`, + `--extensions-dir=${extensions}`, + `--extensionDevelopmentPath=${extension}`, + "--no-sandbox", + "--disable-gpu-sandbox", + "--password-store=basic", + "--disable-updates", + "--skip-welcome", + "--skip-release-notes", + "--disable-workspace-trust", + ] +} diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts new file mode 100644 index 0000000000..2a44a262c7 --- /dev/null +++ b/scripts/solheim-smoke/driver.mts @@ -0,0 +1,181 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { randomUUID } from "node:crypto" +import { fileURLToPath } from "node:url" +import { spawn, type ChildProcess } from "node:child_process" +import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { IpcClient } from "@roo-code/ipc" +import { IpcMessageType, TaskCommandName, type TaskEvent } from "@roo-code/types" +import { + buildChildEnvironment, + buildLaunchArgs, + buildSmokeConfiguration, + SMOKE_PROMPT, + SMOKE_TIMEOUT_MS, + SOLHEIM_BASE_URL, + SOLHEIM_MODEL, +} from "./config.ts" +import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" +import { createTaskStartWaiter } from "./task-start.ts" + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const out = process.env.SOLHEIM_SMOKE_OUT_DIR +if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") +await fs.mkdir(out, { recursive: true }) +const started = Date.now() +const evidence: SmokeEvidence = { + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, +} +let code: SmokeCode = "SETUP_FAILED" +let activated = false +let accepted = false +let sessionLost = false +let child: ChildProcess | undefined +let client: IpcClient | undefined +let temp: string | undefined +let exited = false +let waiter: ReturnType | undefined +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +async function until(check: () => boolean, timeoutMs: number, stopOnSessionLoss = true): Promise { + const deadline = Date.now() + timeoutMs + while (!check() && (!stopOnSessionLoss || !sessionLost) && Date.now() < deadline) await pause(100) + return check() +} +function signalChild(signal: NodeJS.Signals): void { + if (!child?.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } +} +function interrupted(): void { + sessionLost = true + waiter?.onExit() + signalChild("SIGTERM") +} +process.once("SIGINT", interrupted) +process.once("SIGTERM", interrupted) + +try { + const apiKey = process.env.SOLHEIM_API_KEY + if (!apiKey) throw new Error("Provider credential missing") + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) + const workspace = path.join(temp, "workspace") + const home = path.join(temp, "home") + await Promise.all([workspace, home].map((folder) => fs.mkdir(folder))) + const socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) + child = process.env.DISPLAY + ? spawn(executable, args, { + detached: true, + env: buildChildEnvironment(process.env, home, socket), + stdio: "ignore", + }) + : spawn("xvfb-run", ["-a", executable, ...args], { + detached: true, + env: buildChildEnvironment(process.env, home, socket), + stdio: "ignore", + }) + child.once("error", () => { + sessionLost = true + exited = true + waiter?.onExit() + }) + child.once("close", () => { + sessionLost = true + exited = true + waiter?.onExit() + }) + client = new IpcClient(socket, () => {}) + client.on(IpcMessageType.Disconnect, () => { + sessionLost = true + waiter?.onDisconnect() + }) + let taskId: string | undefined + client.on(IpcMessageType.TaskEvent, (event: TaskEvent) => { + const id = waiter?.onTaskEvent(event) + if (id && taskId === undefined) taskId = id + collectEvidence(evidence, event, taskId) + }) + activated = await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) + if (!activated) code = "ACTIVATION_FAILED" + else { + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, + }) + try { + taskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + } + if (accepted) { + await until( + () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + SMOKE_TIMEOUT_MS, + ) + code = smokeCode(evidence, sessionLost) + } + } +} catch { + /* Keep failures fixed and credential-free, never print thrown provider/host text. */ +} finally { + // Teardown is independently bounded, including spawn/activation failures. + if (client?.isReady) client.sendCommand({ commandName: TaskCommandName.CloseTask }) + if (child && !exited) { + await until(() => exited, 5_000, false) + if (!exited) { + signalChild("SIGTERM") + await until(() => exited, 5_000, false) + } + if (!exited) { + signalChild("SIGKILL") + await until(() => exited, 5_000, false) + } + if (!exited && code === "OK") code = "TEARDOWN_FAILED" + } + client?.disconnect() + if (temp && (!child || exited)) await fs.rm(temp, { recursive: true, force: true }) +} +// Whitelisted metadata only: model answers and host logs are never saved or uploaded. +const verdict = { + kind: "provider-smoke", + model: SOLHEIM_MODEL, + provider: SOLHEIM_BASE_URL, + passed: code === "OK", + code, + gates: { + extensionActivated: activated, + ipcConnected: activated, + taskAccepted: accepted, + providerResponseSeen: evidence.providerResponseSeen, + completionSeen: evidence.completionSeen, + }, + durationMs: Date.now() - started, +} +// Write then rename, so a reader sees the old or the new verdict and never a partial one. +const verdictTemp = path.join(out, `verdict.json.${process.pid}.tmp`) +await fs.writeFile(verdictTemp, JSON.stringify(verdict, null, 2)) +await fs.rename(verdictTemp, path.join(out, "verdict.json")) +console.log(`Solheim provider smoke: ${code}`) +process.exit(code === "OK" ? 0 : 1) diff --git a/scripts/solheim-smoke/evidence.test.ts b/scripts/solheim-smoke/evidence.test.ts new file mode 100644 index 0000000000..b3be3faf4f --- /dev/null +++ b/scripts/solheim-smoke/evidence.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, type TaskEvent } from "@roo-code/types" +import { collectEvidence, smokeCode, type SmokeEvidence } from "./evidence.ts" +const fresh = (): SmokeEvidence => ({ + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, +}) +const usage = ( + taskId: string, + totalTokensOut: number, +): Extract => ({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [taskId, { totalTokensIn: 1, totalTokensOut, totalCost: 0, contextTokens: 1 }, {}], +}) +const message = (taskId: string, value: Record) => ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId, action: "created", message: { ts: 1, ...value } }], +}) + +describe("provider smoke evidence", () => { + it("rejects malformed and uncorrelated events, including activity before task acceptance", () => { + const state = fresh() + collectEvidence(state, usage("root", 5), undefined) + collectEvidence(state, usage("other", 5), "root") + collectEvidence( + state, + { eventName: RooCodeEventName.TaskTokenUsageUpdated, payload: ["root", { totalTokensOut: "5" }, {}] }, + "root", + ) + assert.deepEqual(state, fresh()) + }) + it("requires actual output usage; echoed task text is not provider evidence", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "say", say: "text", text: "task prompt" }), "root") + collectEvidence(state, usage("root", 0), "root") + assert.equal(state.providerResponseSeen, false) + collectEvidence(state, usage("root", 5), "root") + assert.equal(state.providerResponseSeen, true) + }) + it("accepts any non-empty completed response, without reviewing its meaning", () => { + const state = fresh() + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion", partial: true }), + "root", + ) + assert.equal(state.completionSeen, false) + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion" }), + "root", + ) + assert.equal(state.completionSeen, true) + assert.ok(!JSON.stringify(state).includes("any completion")) + }) + it("allows the final completion dialog but rejects other tools", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "ask", ask: "completion_result" }), "root") + assert.equal(state.completionSeen, false) + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("other", { type: "ask", ask: "command", text: "ignored" }), "root") + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("root", { type: "ask", ask: "command", text: "never stored" }), "root") + assert.equal(state.unexpectedTool, true) + const noAsk = fresh() + const attempted = usage("root", 5) + attempted.payload[2] = { execute_command: { attempts: 1, failures: 1 } } + collectEvidence(noAsk, attempted, "root") + assert.equal(noAsk.unexpectedTool, true) + }) + it("cannot pass on activation alone or ignore a lost session", () => { + assert.equal(smokeCode(fresh(), false), "PROVIDER_TIMEOUT") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true }, false), "COMPLETION_MISSING") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, false), "OK") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, true), "SESSION_LOST") + assert.equal( + smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true, unexpectedTool: true }, false), + "UNEXPECTED_TOOL", + ) + }) + it("rejects delegation and aborts on the accepted root", () => { + const state = fresh() + collectEvidence(state, { eventName: RooCodeEventName.TaskDelegated, payload: ["root", "child"] }, "root") + assert.equal(state.unexpectedTool, true) + collectEvidence(state, { eventName: RooCodeEventName.TaskAborted, payload: ["root"] }, "root") + assert.equal(smokeCode(state, false), "SESSION_LOST") + }) +}) diff --git a/scripts/solheim-smoke/evidence.ts b/scripts/solheim-smoke/evidence.ts new file mode 100644 index 0000000000..3e436ca8c4 --- /dev/null +++ b/scripts/solheim-smoke/evidence.ts @@ -0,0 +1,62 @@ +import { RooCodeEventName, taskEventSchema, type TaskEvent } from "@roo-code/types" + +export type SmokeEvidence = { + providerResponseSeen: boolean + completionSeen: boolean + unexpectedTool: boolean + aborted: boolean +} + +// Only schema-valid events for the accepted root task can satisfy smoke gates. +// No model text, credentials, tool arguments, or task IDs are persisted. +export function collectEvidence(state: SmokeEvidence, raw: unknown, rootTaskId: string | undefined): void { + if (!rootTaskId) return + const parsed = taskEventSchema.safeParse(raw) + if (!parsed.success) return + const event: TaskEvent = parsed.data + if (event.eventName === RooCodeEventName.Message) { + const entry = event.payload[0] + if (entry.taskId !== rootTaskId || entry.message.partial === true) return + const message = entry.message + if (message.type === "say" && message.say === "completion_result" && message.text?.trim()) + state.completionSeen = true + if (message.type === "ask" && message.ask !== "completion_result") state.unexpectedTool = true + } else if (event.payload[0] === rootTaskId) { + if ( + event.eventName === RooCodeEventName.TaskTokenUsageUpdated || + event.eventName === RooCodeEventName.TaskCompleted + ) { + if ( + Object.entries(event.payload[2]).some( + ([tool, usage]) => tool !== "attempt_completion" && usage.attempts > 0, + ) + ) + state.unexpectedTool = true + } + if (event.eventName === RooCodeEventName.TaskTokenUsageUpdated && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskCompleted && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskAborted) state.aborted = true + if (event.eventName === RooCodeEventName.TaskDelegated || event.eventName === RooCodeEventName.TaskSpawned) + state.unexpectedTool = true + } +} + +export type SmokeCode = + | "OK" + | "SETUP_FAILED" + | "ACTIVATION_FAILED" + | "TASK_START_FAILED" + | "SESSION_LOST" + | "UNEXPECTED_TOOL" + | "PROVIDER_TIMEOUT" + | "COMPLETION_MISSING" + | "TEARDOWN_FAILED" +export function smokeCode(state: SmokeEvidence, sessionLost: boolean): SmokeCode { + if (sessionLost || state.aborted) return "SESSION_LOST" + if (state.unexpectedTool) return "UNEXPECTED_TOOL" + if (!state.providerResponseSeen) return "PROVIDER_TIMEOUT" + if (!state.completionSeen) return "COMPLETION_MISSING" + return "OK" +} diff --git a/scripts/solheim-smoke/package.json b/scripts/solheim-smoke/package.json new file mode 100644 index 0000000000..c4fb2812b9 --- /dev/null +++ b/scripts/solheim-smoke/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh new file mode 100644 index 0000000000..f69e17b17f --- /dev/null +++ b/scripts/solheim-smoke/record.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Run inside a fresh xvfb-run display, never record an existing desktop. +set -euo pipefail +: "${DISPLAY:?Run this wrapper under xvfb-run}" +: "${SOLHEIM_SMOKE_OUT_DIR:?Set a scratch output directory}" +mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" +recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" + +# The recorder needs display access, not provider/GitHub credentials. Fragmented +# MP4 keeps completed fragments readable if the action is forcibly interrupted. +env -i "PATH=$PATH" "DISPLAY=$DISPLAY" "XAUTHORITY=${XAUTHORITY:-}" \ + ffmpeg -nostdin -hide_banner -loglevel error -y \ + -f x11grab -video_size 1280x720 -framerate 12 -i "$DISPLAY" \ + -an -c:v libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p -g 24 \ + -t 480 -fs 125829120 -movflags +frag_keyframe+empty_moov \ + "$recording" >/dev/null 2>&1 & +recorder_pid=$! + +finish() { + result=$? + trap - EXIT + kill -INT "$recorder_pid" 2>/dev/null || true + # A hung recorder must not hold the action open indefinitely. + for ((attempt = 0; attempt < 50; attempt++)); do + kill -0 "$recorder_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$recorder_pid" 2>/dev/null || true + wait "$recorder_pid" 2>/dev/null || true + if [[ ! -s "$recording" ]] || ! env -i "PATH=$PATH" \ + ffprobe -v error "$recording" >/dev/null 2>&1; then + echo "Solheim smoke recording failed" + [[ "$result" -ne 0 ]] || result=1 + fi + exit "$result" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +sleep 0.5 +kill -0 "$recorder_pid" 2>/dev/null || exit 1 + +# Arguments are used for credential-free recorder tests; CI supplies none. +if [[ "$#" -gt 0 ]]; then "$@"; else pnpm solheim:smoke; fi diff --git a/scripts/solheim-smoke/task-start.test.ts b/scripts/solheim-smoke/task-start.test.ts new file mode 100644 index 0000000000..58ebb91fac --- /dev/null +++ b/scripts/solheim-smoke/task-start.test.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { RooCodeEventName, TASK_START_FAILURE_ERROR_CODE, TASK_START_FAILURE_ERROR_MESSAGE } from "@roo-code/types" + +import { createTaskStartWaiter } from "./task-start.ts" + +const REQUEST_ID = "final-smoke-0123abcdef456789" + +const startResponse = (payload: unknown): unknown => ({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [payload], +}) + +const successPayload = (requestId = REQUEST_ID): unknown => ({ + requestId, + success: true, + taskId: "accepted-task-1", +}) + +const failurePayload = (requestId = REQUEST_ID): Record => ({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, +}) + +const settledOutcome = async (promise: Promise): Promise<"resolved" | "rejected" | "pending"> => { + const probe = promise.then( + () => "resolved" as const, + () => "rejected" as const, + ) + return Promise.race([probe, new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 10))]) +} + +describe("createTaskStartWaiter", () => { + it("resolves with the accepted task ID on a matching success response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload())) + + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("rejects with the fixed sanitized failure values on a matching failure response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal(error.message, `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE}`) + return true + }) + }) + + it("rejects with the stage in the sanitized error on a failure response carrying a stage", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "taskCreation", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal( + error.message, + `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE} (stage=taskCreation)`, + ) + return true + }) + }) + + it("rejects with the stage unknown in the sanitized error when the response names it", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "unknown", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /\(stage=unknown\)$/) + return true + }) + }) + + it("ignores a failure response carrying an unknown stage value", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "secretStore", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("ignores responses for a different requestId", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload("some-other-request"))) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores malformed and non-start events", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskCreated, payload: ["task-x"] }) + waiter.onTaskEvent(startResponse({ requestId: REQUEST_ID, success: true })) + waiter.onTaskEvent(startResponse("not-an-object")) + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskStartResponse }) + waiter.onTaskEvent(undefined) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores a success response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: true, + taskId: "accepted-task-1", + providerError: "key sk-secret-123 rejected", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("cannot settle on a failure response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("settles only once: later responses cannot change the outcome", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + + assert.equal(waiter.onTaskEvent(startResponse(successPayload())), undefined) + waiter.onDisconnect() + waiter.onExit() + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("rejects on timeout", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /task start response missing after 10ms/) + assert.match(error.message, new RegExp(REQUEST_ID)) + return true + }) + }) + + it("rejects on IPC disconnect", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onDisconnect() + + await assert.rejects(waiter.result, /IPC disconnected before the task start response arrived/) + }) + + it("rejects on VS Code exit", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onExit() + + await assert.rejects(waiter.result, /VS Code exited before the task start response arrived/) + }) + + it("stops the timeout once settled", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + waiter.onTaskEvent(startResponse(successPayload())) + await waiter.result + + // A pending timer must not reject after success. Wait past the + // original deadline; a leak would reject the already-settled promise + // handler chain here. + await new Promise((resolve) => setTimeout(resolve, 30)) + assert.equal(await waiter.result, "accepted-task-1") + }) +}) diff --git a/scripts/solheim-smoke/task-start.ts b/scripts/solheim-smoke/task-start.ts new file mode 100644 index 0000000000..2aafaaea57 --- /dev/null +++ b/scripts/solheim-smoke/task-start.ts @@ -0,0 +1,96 @@ +import { RooCodeEventName, taskStartResponseSchema } from "@roo-code/types" + +export type TaskStartWaiter = { + onTaskEvent: (event: unknown) => string | undefined + onDisconnect: () => void + onExit: () => void + result: Promise +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null + +// Correlates one StartNewTask request with its sanitized response, similar to +// JSON-RPC: the sender generates the requestId and the reply copies it back. +// The waiter settles exactly once. A matched success resolves with the +// accepted task ID. A matched failure, a timeout, an IPC disconnect, or a +// VS Code exit rejects. Unmatched or malformed responses never settle it. +export function createTaskStartWaiter({ + requestId, + timeoutMs, +}: { + requestId: string + timeoutMs: number +}): TaskStartWaiter { + let resolveResult!: (taskId: string) => void + let rejectResult!: (error: Error) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + + let settled = false + let timer: NodeJS.Timeout | undefined + + const settle = (finish: () => void): void => { + if (settled) { + return + } + settled = true + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + finish() + } + + const rejectWith = (message: string): void => { + settle(() => rejectResult(new Error(message))) + } + + timer = setTimeout(() => { + rejectWith(`task start response missing after ${timeoutMs}ms (requestId=${requestId})`) + }, timeoutMs) + + return { + onTaskEvent(event: unknown): string | undefined { + if (settled) return undefined + if (!isRecord(event) || event["eventName"] !== RooCodeEventName.TaskStartResponse) { + return undefined + } + + const payload = event["payload"] + if (!Array.isArray(payload) || payload.length === 0) { + return undefined + } + + const parsed = taskStartResponseSchema.safeParse(payload[0]) + if (!parsed.success || parsed.data.requestId !== requestId) { + return undefined + } + + const response = parsed.data + if (response.success) { + const acceptedTaskId = response.taskId + settle(() => resolveResult(acceptedTaskId)) + return acceptedTaskId + } + + const { errorCode, errorMessage, stage } = response + // The stage is a fixed sanitized token from the response schema; it + // names the startup phase that stopped responding. + const stageDetail = stage === undefined ? "" : ` (stage=${stage})` + rejectWith(`${errorCode}: ${errorMessage}${stageDetail}`) + return undefined + }, + + onDisconnect(): void { + rejectWith("IPC disconnected before the task start response arrived") + }, + + onExit(): void { + rejectWith("VS Code exited before the task start response arrived") + }, + + result, + } +} diff --git a/scripts/solheim-smoke/tsconfig.json b/scripts/solheim-smoke/tsconfig.json new file mode 100644 index 0000000000..55bddebe10 --- /dev/null +++ b/scripts/solheim-smoke/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "lib": ["ES2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./*.ts", "./*.mts"] +} diff --git a/src/extension/__tests__/api-start-task-logging.spec.ts b/src/extension/__tests__/api-start-task-logging.spec.ts new file mode 100644 index 0000000000..a4fe87f22c --- /dev/null +++ b/src/extension/__tests__/api-start-task-logging.spec.ts @@ -0,0 +1,296 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import * as vscode from "vscode" + +import { + TaskCommandName, + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + taskStartResponseSchema, + providerIdentifiers, + type RooCodeSettings, +} from "@roo-code/types" + +import { API } from "../api" +import { Package } from "../../shared/package" +import type { ClineProvider } from "../../core/webview/ClineProvider" +import { makeClineProviderFactory } from "../../test-utils/provider" + +const { commandHandlers, sentMessages } = vi.hoisted(() => ({ + commandHandlers: [] as Array<(clientId: string, command: unknown) => Promise>, + sentMessages: [] as Array<{ clientId: string; message: unknown }>, +})) + +vi.mock("@roo-code/ipc", () => ({ + IpcServer: class { + listen(): void {} + on(_messageType: unknown, handler: (clientId: string, command: unknown) => Promise): void { + commandHandlers.push(handler) + } + send(clientId: string, message: unknown): void { + sentMessages.push({ clientId, message }) + } + }, +})) + +type SentTaskEvent = { data: { eventName: RooCodeEventName; payload: unknown[] } } + +const sentTaskEvents = (): SentTaskEvent[] => sentMessages.map(({ message }) => message as SentTaskEvent) + +const buildStartCommand = (overrides?: { requestId?: string; text?: string }) => ({ + commandName: TaskCommandName.StartNewTask, + data: { + configuration: { + apiProvider: providerIdentifiers.openai, + openAiApiKey: "opaque-solheim-secret-0123456789abcdef", + openAiModelId: "qwen3.8-27b", + openAiBaseUrl: "https://api.solheim.ai/v1", + autoApprovalEnabled: true, + } as RooCodeSettings, + text: overrides?.text ?? "provider connectivity smoke", + requestId: overrides?.requestId, + }, +}) + +function withStageTimeoutOverride(ms: string): () => void { + const previous = process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS + process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = ms + return () => { + if (previous === undefined) { + delete process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS + } else { + process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = previous + } + } +} + +function buildApi(createTaskImpl?: () => Promise<{ taskId: string }>, postStateToWebviewImpl?: () => Promise) { + const outputChannel = { appendLine: vi.fn() } as unknown as vscode.OutputChannel + const provider = { + context: {}, + on: vi.fn(), + contextProxy: { setValues: vi.fn().mockResolvedValue(undefined) }, + providerSettingsManager: { saveConfig: vi.fn().mockResolvedValue("default"), setModeConfig: vi.fn() }, + postStateToWebview: postStateToWebviewImpl + ? vi.fn().mockImplementation(postStateToWebviewImpl) + : vi.fn().mockResolvedValue(undefined), + evictCurrentTask: vi.fn().mockResolvedValue(undefined), + postMessageToWebview: vi.fn().mockResolvedValue(undefined), + removeClineFromStack: vi.fn().mockResolvedValue(undefined), + createTask: createTaskImpl + ? vi.fn().mockImplementation(createTaskImpl) + : vi.fn().mockResolvedValue({ taskId: "task-1" }), + } as unknown as ClineProvider + + const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}) + const executeCommand = vi.spyOn(vscode.commands, "executeCommand").mockResolvedValue(undefined) + const api = new API(outputChannel, provider, makeClineProviderFactory(), "/tmp/final-smoke-test.sock", true) + + const collectLogs = () => + [ + ...(outputChannel.appendLine as ReturnType).mock.calls.flat().map(String), + ...consoleSpy.mock.calls.flat().map((arg) => JSON.stringify(arg)), + ].join("\n") + + return { api, provider, executeCommand, collectLogs } +} + +describe("API StartNewTask IPC", () => { + afterEach(() => { + commandHandlers.length = 0 + sentMessages.length = 0 + vi.restoreAllMocks() + }) + + it("never logs the API key, base URL, or prompt text from the StartNewTask command", async () => { + const { collectLogs } = buildApi() + + const handler = commandHandlers.at(-1) + expect(handler).toBeDefined() + + await handler!("client-1", buildStartCommand()) + + const logged = collectLogs() + + expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(logged).not.toContain('"openAiApiKey":"') + // Configuration is omitted entirely, even nominally non-secret fields. + expect(logged).not.toContain("openAiModelId") + expect(logged).not.toContain("qwen3.8-27b") + expect(logged).not.toContain("openAiBaseUrl") + expect(logged).not.toContain("openAiApiKey") + // Prompt content is metadata only: length, never text. + expect(logged).not.toContain("provider connectivity smoke") + expect(logged).toContain("promptLength") + }) + + it("omits the sidebar focus for an IPC start, so an unresolved focus command cannot block it", async () => { + const { executeCommand } = buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-legacy" })) + + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) + + it("answers a correlated start with exactly one sanitized success response", async () => { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-1" })) + + expect(sentMessages).toHaveLength(1) + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success) { + expect(parsed.data).toEqual({ requestId: "req-1", success: true, taskId: "task-1" }) + } + }) + + it("answers a failed correlated start with exactly one fixed safe failure response", async () => { + const rawError = new Error( + "openai rejected key opaque-solheim-secret-0123456789abcdef at https://api.solheim.ai/v1 for prompt review the pull request diff", + ) + buildApi(() => Promise.reject(rawError)) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-2" })) + + expect(sentMessages).toHaveLength(1) + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success && !parsed.data.success) { + expect(parsed.data.errorCode).toBe(TASK_START_FAILURE_ERROR_CODE) + expect(parsed.data.errorMessage).toBe(TASK_START_FAILURE_ERROR_MESSAGE) + expect(parsed.data.stage).toBe("unknown") + } + + const sent = JSON.stringify(sentMessages) + expect(sent).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(sent).not.toContain("api.solheim.ai") + expect(sent).not.toContain("review the pull request diff") + }) + + it("bounds a hung task creation stage and answers with exactly one sanitized stage failure", async () => { + const restore = withStageTimeoutOverride("100") + try { + const { collectLogs } = buildApi(() => new Promise(() => {})) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-timeout-task" })) + + expect(sentMessages).toHaveLength(1) + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success && !parsed.data.success) { + expect(parsed.data).toEqual({ + requestId: "req-timeout-task", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + }) + } + + const logged = collectLogs() + expect(logged).toContain("taskCreation") + expect(logged).toContain("req-timeout-task") + expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") + } finally { + restore() + } + }) + + it("removes a late task and refuses a retry until the timed-out creation settles", async () => { + const restore = withStageTimeoutOverride("50") + try { + let finishCreation: (task: { taskId: string }) => void = () => {} + const { provider } = buildApi( + () => + new Promise((resolve) => { + finishCreation = resolve + }), + ) + const handler = commandHandlers.at(-1)! + + await handler("client-1", buildStartCommand({ requestId: "req-first" })) + await handler("client-1", buildStartCommand({ requestId: "req-retry" })) + + expect(provider.createTask).toHaveBeenCalledTimes(1) + const retry = taskStartResponseSchema.parse(sentTaskEvents()[1]?.data.payload[0]) + expect(retry.success).toBe(false) + + finishCreation({ taskId: "late-task" }) + await vi.waitFor(() => expect(provider.removeClineFromStack).toHaveBeenCalledTimes(1)) + + await handler("client-1", buildStartCommand({ requestId: "req-after" })) + expect(provider.createTask).toHaveBeenCalledTimes(2) + } finally { + restore() + } + }) + + it("bounds a hung settings stage and names it in the failure response", async () => { + const restore = withStageTimeoutOverride("100") + try { + buildApi(undefined, () => new Promise(() => {})) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-timeout-settings" })) + + expect(sentMessages).toHaveLength(1) + const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success && !parsed.data.success) { + expect(parsed.data.stage).toBe("settings") + } + } finally { + restore() + } + }) + + it("still answers with one success response when every stage is fast under a small bound", async () => { + const restore = withStageTimeoutOverride("100") + try { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-fast" })) + + expect(sentMessages).toHaveLength(1) + const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) + expect(parsed.success).toBe(true) + if (parsed.success) { + expect(parsed.data).toEqual({ requestId: "req-fast", success: true, taskId: "task-1" }) + } + } finally { + restore() + } + }) + + it("sends no start response for legacy IPC starts without requestId", async () => { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand()) + + expect(sentMessages).toHaveLength(0) + }) + + it("keeps the sidebar focus for direct API callers that do not opt out", async () => { + const { api, executeCommand } = buildApi() + + await api.startNewTask({ configuration: {} as RooCodeSettings, text: "direct caller" }) + + expect(executeCommand).toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) +}) diff --git a/src/extension/api.ts b/src/extension/api.ts index 6a8ed8dfdd..30ff68838e 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -13,6 +13,8 @@ import { type ProviderSettings, type ProviderSettingsEntry, type TaskEvent, + type TaskStartResponse, + type TaskStartStage, type CreateTaskOptions, type TaskApiConversationHistorySequence, type WebviewThemeFixture, @@ -21,7 +23,10 @@ import { isSecretStateKey, IpcOrigin, IpcMessageType, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, } from "@roo-code/types" + import { IpcServer } from "@roo-code/ipc" import { Package } from "../shared/package" @@ -33,9 +38,51 @@ import { Terminal } from "../integrations/terminal/Terminal" import { TerminalRegistry } from "../integrations/terminal/TerminalRegistry" import { getCommands } from "../services/command/commands" +// The default bound is per stage, not per start, so one slow stage cannot +// hide behind the budget of another. Secret storage writes through VS Code +// secret storage can hang indefinitely under Xvfb without a keyring service. +const DEFAULT_TASK_START_STAGE_TIMEOUT_MS = 60_000 + +function readTaskStartStageTimeoutMs(env: NodeJS.ProcessEnv): number { + const parsed = Number(env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS) + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TASK_START_STAGE_TIMEOUT_MS +} + +class TaskStartStageTimeoutError extends Error { + constructor(readonly stage: TaskStartStage) { + super(`task start stage timed out: ${stage}`) + } +} + +// Races the stage against its bound. Losing the race stops the wait only. +// The operation keeps running, so onAbandon receives it for later cleanup. +async function withStageTimeout( + stage: TaskStartStage, + timeoutMs: number, + operation: () => T | Thenable, + onAbandon?: (running: Promise) => void, +): Promise { + let timer: NodeJS.Timeout | undefined + const running = Promise.resolve(operation()) + try { + return await Promise.race([ + running, + new Promise((_, reject) => { + timer = setTimeout(() => { + onAbandon?.(running) + reject(new TaskStartStageTimeoutError(stage)) + }, timeoutMs) + }), + ]) + } finally { + clearTimeout(timer) + } +} + export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel private readonly sidebarProvider: ClineProvider + private abandonedStarts = 0 private readonly ipc?: IpcServer private readonly log: (...args: unknown[]) => void private logfile?: string @@ -81,12 +128,49 @@ export class API extends EventEmitter implements RooCodeAPI { } switch (command.commandName) { - case TaskCommandName.StartNewTask: - this.log( - `[API] StartNewTask -> ${command.data.text}, ${JSON.stringify(command.data.configuration)}`, - ) - await this.startNewTask(command.data) + case TaskCommandName.StartNewTask: { + // Neither prompt nor settings values enter the log. + this.log("[API] StartNewTask", { + promptLength: command.data.text.length, + imageCount: command.data.images?.length ?? 0, + newTab: command.data.newTab === true, + requestId: command.data.requestId, + }) + + const { requestId } = command.data + + if (requestId === undefined) { + await this.startNewTask(command.data, { focusSidebar: false }) + break + } + + // Exactly one sanitized, client-scoped response per + // correlated start. The reply never echoes the failure. + // Each startup stage is bounded so a hung await (for + // example secret storage under a missing keyring) + // still produces the reply. + let response: TaskStartResponse + try { + const taskId = await this.startNewTask(command.data, { + focusSidebar: false, + stageTimeoutMs: readTaskStartStageTimeoutMs(process.env), + }) + response = { requestId, success: true, taskId } + } catch (error) { + const stage: TaskStartStage = + error instanceof TaskStartStageTimeoutError ? error.stage : "unknown" + this.log("[API] StartNewTask failed", { requestId, stage }) + response = { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage, + } + } + sendResponse(RooCodeEventName.TaskStartResponse, [response]) break + } case TaskCommandName.CancelTask: this.log(`[API] CancelTask`) await this.cancelCurrentTask() @@ -169,35 +253,73 @@ export class API extends EventEmitter implements RooCodeAPI { return super.emit(eventName, ...args) } - public async startNewTask({ - configuration, - text, - images, - newTab, - }: { - configuration: RooCodeSettings - text?: string - images?: string[] - newTab?: boolean - }) { + public async startNewTask( + { + configuration, + text, + images, + newTab, + }: { + configuration: RooCodeSettings + text?: string + images?: string[] + newTab?: boolean + }, + startOptions?: { focusSidebar?: boolean; stageTimeoutMs?: number }, + ) { + // Stage names for a bounded correlated start. The settings persistence + // named in the startup evidence (createTask -> setValues -> + // storeSecret) runs inside the taskCreation stage at this layer, so a + // taskCreation timeout covers it. + // A timed-out stage keeps running. Count it, and reject a new bounded + // start until it settles. A late task from the taskCreation stage is + // removed, so it cannot run after the failure reply. + if (startOptions?.stageTimeoutMs !== undefined && this.abandonedStarts > 0) { + throw new Error("an earlier task start is still running") + } + const boundStage = ( + stage: TaskStartStage, + operation: () => T | Thenable, + onLate?: (late: Awaited) => unknown, + ): Promise => + startOptions?.stageTimeoutMs === undefined + ? Promise.resolve(operation()) + : withStageTimeout(stage, startOptions.stageTimeoutMs, operation, (running) => { + this.abandonedStarts++ + void running + .then((late) => onLate?.(late as Awaited)) + .catch(() => {}) + .finally(() => this.abandonedStarts--) + }) + let provider: ClineProvider if (newTab) { - await vscode.commands.executeCommand("workbench.action.files.revert") - await vscode.commands.executeCommand("workbench.action.closeAllEditors") + await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.files.revert")) + await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.closeAllEditors")) - provider = await this.providerFactory.createInNewTab() + provider = await boundStage("unknown", () => this.providerFactory.createInNewTab()) this.registerListeners(provider) } else { - await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) + // IPC callers omit focus: executeCommand cannot be cancelled and a + // stalled webview launch would block the whole start. + if (startOptions?.focusSidebar ?? true) { + await boundStage("unknown", () => + vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`), + ) + } provider = this.sidebarProvider } - await provider.evictCurrentTask() - await provider.postStateToWebview() - await provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }) - await provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }) + await boundStage("eviction", () => provider.evictCurrentTask()) + await boundStage("settings", () => provider.postStateToWebview()) + await boundStage("settings", () => + provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }), + ) + await boundStage("settings", () => + provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }), + ) // API-driven tasks never stop to ask the user, so the hard stop is // effectively unlimited. The soft repetition tier is disabled too: @@ -208,7 +330,11 @@ export class API extends EventEmitter implements RooCodeAPI { toolRepetitionSoftLimit: 0, } - const task = await provider.createTask(text, images, undefined, options, configuration) + const task = await boundStage( + "taskCreation", + () => provider.createTask(text, images, undefined, options, configuration), + (late) => late && provider.removeClineFromStack(), + ) if (!task) { throw new Error("Failed to create task due to policy restrictions") From 748479560da819c8306ff7fc8e6e09706bc5e043 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 6 Oct 2026 01:08:18 +0000 Subject: [PATCH 02/11] fix(smoke): keep startup timeouts in isolated controller --- packages/ipc/package.json | 1 + packages/ipc/src/__tests__/ipc-server.test.ts | 65 +++++ packages/ipc/src/ipc-server.ts | 9 +- scripts/solheim-smoke-workflow.test.mjs | 1 + scripts/solheim-smoke/README.md | 8 +- scripts/solheim-smoke/config.ts | 1 - scripts/solheim-smoke/driver.mts | 262 +++++++----------- scripts/solheim-smoke/run.test.ts | 245 ++++++++++++++++ scripts/solheim-smoke/run.ts | 167 +++++++++++ .../__tests__/api-start-task-logging.spec.ts | 180 ++++-------- src/extension/api.ts | 112 +------- 11 files changed, 664 insertions(+), 387 deletions(-) create mode 100644 packages/ipc/src/__tests__/ipc-server.test.ts create mode 100644 scripts/solheim-smoke/run.test.ts create mode 100644 scripts/solheim-smoke/run.ts diff --git a/packages/ipc/package.json b/packages/ipc/package.json index 374c4240b6..4dd505bb18 100644 --- a/packages/ipc/package.json +++ b/packages/ipc/package.json @@ -5,6 +5,7 @@ "type": "module", "exports": "./src/index.ts", "scripts": { + "test": "node --import tsx --test src/__tests__/*.test.ts", "lint": "eslint src --ext=ts --max-warnings=0", "check-types": "tsc --noEmit", "clean": "rimraf .turbo" diff --git a/packages/ipc/src/__tests__/ipc-server.test.ts b/packages/ipc/src/__tests__/ipc-server.test.ts new file mode 100644 index 0000000000..3419ebf982 --- /dev/null +++ b/packages/ipc/src/__tests__/ipc-server.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { IpcMessageType, IpcOrigin, TaskCommandName, ipcMessageSchema } from "@roo-code/types" +import { IpcServer } from "../ipc-server.js" + +function fixture() { + const logs: unknown[][] = [] + const commands: unknown[] = [] + const server = new IpcServer("/unused-test.sock", (...args) => logs.push(args)) + server.on(IpcMessageType.TaskCommand, (clientId, command) => commands.push({ clientId, command })) + return { logs, commands, receive: (data: unknown) => server["onMessage"](data) } +} +const key = "dummy-api-key-not-for-public-logging" +const command = (requestId: string) => ({ + type: IpcMessageType.TaskCommand, + origin: IpcOrigin.Client, + clientId: "client-1", + data: { + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: "private prompt", configuration: { apiProvider: "openai", openAiApiKey: key } }, + }, +}) + +describe("IPC rejection logging", () => { + it("rejects an invalid request ID without logging its key, prompt or schema issue values", () => { + const f = fixture() + f.receive(command(`invalid request containing ${key}`)) + assert.deepEqual(f.commands, []) + assert.equal(f.logs.length, 1) + assert.equal(f.logs[0]?.[0], "[server#onMessage] invalid payload") + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.ok(!JSON.stringify(f.logs).includes("private prompt")) + assert.ok(!JSON.stringify(f.logs).includes("invalid request containing")) + }) + it("does not log credential-shaped primitive input or dynamic issue paths", () => { + const f = fixture() + const valid = command("valid-id") + const malformedHeaders = { + ...valid, + data: { + ...valid.data, + data: { + ...valid.data.data, + configuration: { ...valid.data.data.configuration, openAiHeaders: { [key]: null } }, + }, + }, + } + const rejected = ipcMessageSchema.safeParse(malformedHeaders) + assert.ok(!rejected.success) + assert.ok(rejected.error.issues.some((issue) => issue.path.includes(key))) + for (const data of [key, null, { type: key }, malformedHeaders]) { + f.receive(data) + } + assert.equal(f.logs.length, 4) + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.deepEqual(f.commands, []) + }) + it("still dispatches a valid correlated command to its client without logging its settings", () => { + const f = fixture() + f.receive(command("valid-id")) + assert.equal(f.commands.length, 1) + assert.deepEqual(f.commands[0], { clientId: "client-1", command: command("valid-id").data }) + assert.deepEqual(f.logs, []) + }) +}) diff --git a/packages/ipc/src/ipc-server.ts b/packages/ipc/src/ipc-server.ts index 200e241014..26dd626a9e 100644 --- a/packages/ipc/src/ipc-server.ts +++ b/packages/ipc/src/ipc-server.ts @@ -76,16 +76,15 @@ export class IpcServer extends EventEmitter implements RooCodeI private onMessage(data: unknown) { if (typeof data !== "object") { - this.log(`[server#onMessage] invalid data -> ${JSON.stringify(data)}`) + this.log("[server#onMessage] invalid data") return } const result = ipcMessageSchema.safeParse(data) if (!result.success) { - this.log( - `[server#onMessage] invalid payload -> ${JSON.stringify(result.error.issues)} -> ${JSON.stringify(data)}`, - ) + // Both the input and Zod issue values/paths may contain credentials. + this.log("[server#onMessage] invalid payload", { issueCount: result.error.issues.length }) return } @@ -98,7 +97,7 @@ export class IpcServer extends EventEmitter implements RooCodeI this.emit(IpcMessageType.TaskCommand, payload.clientId, payload.data) break default: - this.log(`[server#onMessage] unhandled payload: ${JSON.stringify(payload)}`) + this.log("[server#onMessage] unhandled payload") break } } diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 1239cfe5a5..0bab69c438 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -56,6 +56,7 @@ describe("Project A provider smoke workflow", () => { it("bounds runtime and serializes the single provider instance", () => { assert.match(workflow, /cancel-in-progress: false/) assert.match(workflow, /timeout-minutes: 8/) + assert.match(workflow, /^ {8}timeout-minutes: 25$/m) }) it("uploads only the verdict and video, never host logs or storage", () => { const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index 35e7189138..219ce3341e 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -18,11 +18,15 @@ The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8 groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage -can contain the configured credential while the session runs; it is removed at teardown and never uploaded. +can contain the configured credential while the session runs. It is removed when no child remains alive; +if termination fails, storage cleanup is skipped and `gates.childExited` reports that separately from the +original smoke failure. Storage is never uploaded. This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only. Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit, -with 30-second per-stage bounds in the extension. The live CI step has an eight-minute outer timeout. +in the controller. The extension awaits startup and configuration persistence without racing shared-profile +writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than +trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts index 69c5d6aa71..7b19187cdc 100644 --- a/scripts/solheim-smoke/config.ts +++ b/scripts/solheim-smoke/config.ts @@ -64,7 +64,6 @@ export function buildChildEnvironment( XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), ROO_CODE_IPC_SOCKET_PATH: socketPath, - ROO_CODE_TASK_START_STAGE_TIMEOUT_MS: "30000", } } diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts index 2a44a262c7..383e2db206 100644 --- a/scripts/solheim-smoke/driver.mts +++ b/scripts/solheim-smoke/driver.mts @@ -1,181 +1,121 @@ import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { randomUUID } from "node:crypto" import { fileURLToPath } from "node:url" -import { spawn, type ChildProcess } from "node:child_process" +import { spawn } from "node:child_process" import { downloadAndUnzipVSCode } from "@vscode/test-electron" import { IpcClient } from "@roo-code/ipc" -import { IpcMessageType, TaskCommandName, type TaskEvent } from "@roo-code/types" -import { - buildChildEnvironment, - buildLaunchArgs, - buildSmokeConfiguration, - SMOKE_PROMPT, - SMOKE_TIMEOUT_MS, - SOLHEIM_BASE_URL, - SOLHEIM_MODEL, -} from "./config.ts" -import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" -import { createTaskStartWaiter } from "./task-start.ts" +import { IpcMessageType } from "@roo-code/types" +import { buildChildEnvironment, buildLaunchArgs } from "./config.ts" +import { runSmoke, smokeExitCode, type SmokeRuntime } from "./run.ts" const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const out = process.env.SOLHEIM_SMOKE_OUT_DIR if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") await fs.mkdir(out, { recursive: true }) -const started = Date.now() -const evidence: SmokeEvidence = { - providerResponseSeen: false, - completionSeen: false, - unexpectedTool: false, - aborted: false, -} -let code: SmokeCode = "SETUP_FAILED" -let activated = false -let accepted = false -let sessionLost = false -let child: ChildProcess | undefined -let client: IpcClient | undefined let temp: string | undefined -let exited = false -let waiter: ReturnType | undefined -const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) -async function until(check: () => boolean, timeoutMs: number, stopOnSessionLoss = true): Promise { - const deadline = Date.now() + timeoutMs - while (!check() && (!stopOnSessionLoss || !sessionLost) && Date.now() < deadline) await pause(100) - return check() -} -function signalChild(signal: NodeJS.Signals): void { - if (!child?.pid || exited) return - try { - process.kill(-child.pid, signal) - } catch { - child.kill(signal) - } -} -function interrupted(): void { - sessionLost = true - waiter?.onExit() - signalChild("SIGTERM") -} -process.once("SIGINT", interrupted) -process.once("SIGTERM", interrupted) - -try { - const apiKey = process.env.SOLHEIM_API_KEY - if (!apiKey) throw new Error("Provider credential missing") - const extension = path.join(root, "src") - await fs.access(path.join(extension, "package.json")) - temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) - const workspace = path.join(temp, "workspace") - const home = path.join(temp, "home") - await Promise.all([workspace, home].map((folder) => fs.mkdir(folder))) - const socket = path.join(temp, "ipc.sock") - const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { - devDependencies: Record - } - const executable = await downloadAndUnzipVSCode({ - version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", - cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), - }) - const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) - child = process.env.DISPLAY - ? spawn(executable, args, { - detached: true, - env: buildChildEnvironment(process.env, home, socket), - stdio: "ignore", - }) - : spawn("xvfb-run", ["-a", executable, ...args], { +let socket = "" +const runtime: SmokeRuntime = { + async launch() { + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) + const workspace = path.join(temp, "workspace") + const isolatedHome = path.join(temp, "home") + await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder))) + socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) + const child = spawn( + process.env.DISPLAY ? executable : "xvfb-run", + process.env.DISPLAY ? args : ["-a", executable, ...args], + { detached: true, - env: buildChildEnvironment(process.env, home, socket), + env: buildChildEnvironment(process.env, isolatedHome, socket), stdio: "ignore", - }) - child.once("error", () => { - sessionLost = true - exited = true - waiter?.onExit() - }) - child.once("close", () => { - sessionLost = true - exited = true - waiter?.onExit() - }) - client = new IpcClient(socket, () => {}) - client.on(IpcMessageType.Disconnect, () => { - sessionLost = true - waiter?.onDisconnect() - }) - let taskId: string | undefined - client.on(IpcMessageType.TaskEvent, (event: TaskEvent) => { - const id = waiter?.onTaskEvent(event) - if (id && taskId === undefined) taskId = id - collectEvidence(evidence, event, taskId) - }) - activated = await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) - if (!activated) code = "ACTIVATION_FAILED" - else { - const requestId = randomUUID() - waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) - client.sendCommand({ - commandName: TaskCommandName.StartNewTask, - data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, - }) - try { - taskId = await waiter.result - accepted = true - } catch { - code = "TASK_START_FAILED" + }, + ) + let exited = false + let onExit = () => {} + const closed = () => { + exited = true + onExit() } - if (accepted) { - await until( - () => - evidence.unexpectedTool || - evidence.aborted || - (evidence.providerResponseSeen && evidence.completionSeen), - SMOKE_TIMEOUT_MS, - ) - code = smokeCode(evidence, sessionLost) + child.once("error", () => { + // Spawn failure has no live PID; a failed kill must not claim exit. + if (!child.pid) closed() + else onExit() + }) + child.once("close", closed) + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + if (exited) listener() + }, + signal(signal) { + if (!child.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } + }, } - } -} catch { - /* Keep failures fixed and credential-free, never print thrown provider/host text. */ -} finally { - // Teardown is independently bounded, including spawn/activation failures. - if (client?.isReady) client.sendCommand({ commandName: TaskCommandName.CloseTask }) - if (child && !exited) { - await until(() => exited, 5_000, false) - if (!exited) { - signalChild("SIGTERM") - await until(() => exited, 5_000, false) + }, + connect() { + const client = new IpcClient(socket, () => {}) + return { + get isReady() { + return client.isReady + }, + onTaskEvent(listener) { + client.on(IpcMessageType.TaskEvent, listener) + }, + onDisconnect(listener) { + client.on(IpcMessageType.Disconnect, listener) + }, + sendCommand(command) { + client.sendCommand(command) + }, + disconnect() { + client.disconnect() + }, } - if (!exited) { - signalChild("SIGKILL") - await until(() => exited, 5_000, false) + }, + async cleanup() { + if (temp) await fs.rm(temp, { recursive: true, force: true }) + }, + async writeVerdict(verdict) { + // Atomic, whitelisted metadata only; no provider text or host logs. + const pending = path.join(out, `verdict.json.${process.pid}.tmp`) + await fs.writeFile(pending, JSON.stringify(verdict, null, 2)) + await fs.rename(pending, path.join(out, "verdict.json")) + }, + onInterrupt(listener) { + process.once("SIGINT", listener) + process.once("SIGTERM", listener) + return () => { + process.removeListener("SIGINT", listener) + process.removeListener("SIGTERM", listener) } - if (!exited && code === "OK") code = "TEARDOWN_FAILED" - } - client?.disconnect() - if (temp && (!child || exited)) await fs.rm(temp, { recursive: true, force: true }) -} -// Whitelisted metadata only: model answers and host logs are never saved or uploaded. -const verdict = { - kind: "provider-smoke", - model: SOLHEIM_MODEL, - provider: SOLHEIM_BASE_URL, - passed: code === "OK", - code, - gates: { - extensionActivated: activated, - ipcConnected: activated, - taskAccepted: accepted, - providerResponseSeen: evidence.providerResponseSeen, - completionSeen: evidence.completionSeen, }, - durationMs: Date.now() - started, + now: Date.now, + pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +} +try { + const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") + console.log(`Solheim provider smoke: ${verdict.code}`) + process.exit(smokeExitCode(verdict)) +} catch { + console.log("Solheim provider smoke: SETUP_FAILED") + process.exit(1) } -// Write then rename, so a reader sees the old or the new verdict and never a partial one. -const verdictTemp = path.join(out, `verdict.json.${process.pid}.tmp`) -await fs.writeFile(verdictTemp, JSON.stringify(verdict, null, 2)) -await fs.rename(verdictTemp, path.join(out, "verdict.json")) -console.log(`Solheim provider smoke: ${code}`) -process.exit(code === "OK" ? 0 : 1) diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts new file mode 100644 index 0000000000..1c64ec53e4 --- /dev/null +++ b/scripts/solheim-smoke/run.test.ts @@ -0,0 +1,245 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" + +type Scenario = + | "success" + | "activation" + | "start-failure" + | "no-provider" + | "no-completion" + | "tool" + | "disconnect" + | "uncorrelated" + | "aborted" + | "setup" +function fixture( + scenario: Scenario = "success", + options: { stuck?: boolean; closeThrows?: boolean; timeout?: () => void } = {}, +) { + let now = 0 + let exited = false + let onExit = () => {} + let onDisconnect = () => {} + let onEvent: (event: unknown) => void = () => {} + let interrupt = () => {} + let cleaned = false + let disconnected = false + let unsubscribed = false + const commands: TaskCommand[] = [] + const signals: NodeJS.Signals[] = [] + const written: SmokeVerdict[] = [] + const close = () => { + exited = true + onExit() + } + const runtime: SmokeRuntime = { + async launch() { + if (scenario === "setup") throw new Error("private setup detail") + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + }, + signal(signal) { + signals.push(signal) + if (!options.stuck) close() + }, + } + }, + connect() { + return { + isReady: scenario !== "activation", + onTaskEvent(listener) { + onEvent = listener + }, + onDisconnect(listener) { + onDisconnect = listener + }, + disconnect() { + disconnected = true + }, + sendCommand(command) { + commands.push(command) + if (command.commandName === TaskCommandName.CloseTask) { + if (options.closeThrows) throw new Error("private close error") + if (!options.stuck) close() + return + } + if (command.commandName !== TaskCommandName.StartNewTask) return + if (options.timeout) { + options.timeout() + return + } + onEvent({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + scenario === "start-failure" + ? { + requestId: command.data.requestId, + success: false, + errorCode: "task_start_failed", + errorMessage: "Task start failed", + } + : { requestId: command.data.requestId, success: true, taskId: "root" }, + ], + }) + const taskId = scenario === "uncorrelated" ? "other" : "root" + if (scenario !== "no-provider") + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 1, totalTokensOut: 5, totalCost: 0, contextTokens: 1 }, + {}, + ], + }) + if (scenario !== "no-completion") + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "private synthetic answer", + }, + }, + ], + }) + if (scenario === "tool") + onEvent({ + eventName: RooCodeEventName.TaskSpawned, + payload: ["root", "child"], + }) + if (scenario === "disconnect") onDisconnect() + if (scenario === "aborted") onEvent({ eventName: RooCodeEventName.TaskAborted, payload: ["root"] }) + }, + } + }, + async cleanup() { + cleaned = true + }, + async writeVerdict(verdict) { + written.push(verdict) + }, + onInterrupt(listener) { + interrupt = listener + return () => { + unsubscribed = true + } + }, + now: () => now, + async pause(ms) { + now += ms + }, + } + return { + runtime, + commands, + signals, + written, + interrupt: () => interrupt(), + get cleaned() { + return cleaned + }, + get disconnected() { + return disconnected + }, + get unsubscribed() { + return unsubscribed + }, + } +} + +describe("provider smoke orchestration", () => { + it("correlates startup and evidence, writes a metadata-only verdict, and exits zero", async () => { + const f = fixture() + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "OK") + assert.equal(smokeExitCode(verdict), 0) + assert.deepEqual(verdict.gates, { + extensionActivated: true, + ipcConnected: true, + taskAccepted: true, + providerResponseSeen: true, + completionSeen: true, + childExited: true, + }) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.disconnected && f.unsubscribed) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.ok(!JSON.stringify(verdict).includes("private")) + }) + for (const [scenario, code] of [ + ["activation", "ACTIVATION_FAILED"], + ["start-failure", "TASK_START_FAILED"], + ["no-provider", "PROVIDER_TIMEOUT"], + ["no-completion", "COMPLETION_MISSING"], + ["tool", "UNEXPECTED_TOOL"], + ["disconnect", "SESSION_LOST"], + ["uncorrelated", "PROVIDER_TIMEOUT"], + ["aborted", "SESSION_LOST"], + ["setup", "SETUP_FAILED"], + ] as const) { + it(`fails closed for ${scenario} and still writes the verdict`, async () => { + const f = fixture(scenario) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, code) + assert.equal(verdict.passed, false) + assert.equal(smokeExitCode(verdict), 1) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.unsubscribed) + }) + } + it("bounds startup at the controller and closes its host when no reply arrives", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }) + const f = fixture("success", { timeout: () => t.mock.timers.tick(35_000) }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.taskAccepted, false) + assert.ok(f.cleaned) + }) + it("bounds TERM/KILL teardown and keeps live-host storage out of cleanup", async () => { + const f = fixture("success", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.equal(verdict.durationMs, 15_000) + assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + assert.equal(f.disconnected, true) + }) + it("preserves the original failure but reports failed teardown separately", async () => { + const f = fixture("start-failure", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + }) + it("interrupts an evidence wait, terminates only its host, and fails closed", async () => { + const f = fixture("no-completion") + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + f.interrupt() + await pause(ms) + } + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "SESSION_LOST") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.unsubscribed) + }) + it("continues host termination and writes a failing verdict after a CloseTask exception", async () => { + const f = fixture("success", { closeThrows: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + }) +}) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts new file mode 100644 index 0000000000..698216598b --- /dev/null +++ b/scripts/solheim-smoke/run.ts @@ -0,0 +1,167 @@ +import { randomUUID } from "node:crypto" +import { TaskCommandName, type TaskCommand } from "@roo-code/types" +import { buildSmokeConfiguration, SMOKE_PROMPT, SMOKE_TIMEOUT_MS, SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" +import { createTaskStartWaiter } from "./task-start.ts" + +export interface SmokeHost { + readonly exited: boolean + onExit(listener: () => void): void + signal(signal: NodeJS.Signals): void +} +export interface SmokeClient { + readonly isReady: boolean + onTaskEvent(listener: (event: unknown) => void): void + onDisconnect(listener: () => void): void + sendCommand(command: TaskCommand): void + disconnect(): void +} +export type SmokeVerdict = { + kind: "provider-smoke" + model: string + provider: string + passed: boolean + code: SmokeCode + gates: { + extensionActivated: boolean + ipcConnected: boolean + taskAccepted: boolean + providerResponseSeen: boolean + completionSeen: boolean + childExited: boolean + } + durationMs: number +} +export interface SmokeRuntime { + launch(): Promise + connect(): SmokeClient + cleanup(): Promise + writeVerdict(verdict: SmokeVerdict): Promise + onInterrupt(listener: () => void): () => void + now(): number + pause(ms: number): Promise +} + +// Only the adapter can spawn VS Code, read credentials or write files. Tests +// exercise the same controller with synthetic IPC and a deterministic clock. +export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise { + const started = runtime.now() + const evidence: SmokeEvidence = { + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, + } + let code: SmokeCode = "SETUP_FAILED" + let activated = false + let accepted = false + let sessionLost = false + let host: SmokeHost | undefined + let client: SmokeClient | undefined + let waiter: ReturnType | undefined + async function until(check: () => boolean, timeoutMs: number, stopOnLoss = true): Promise { + const deadline = runtime.now() + timeoutMs + while (!check() && (!stopOnLoss || !sessionLost) && runtime.now() < deadline) await runtime.pause(100) + } + const markLost = () => { + sessionLost = true + waiter?.onExit() + } + const unsubscribe = runtime.onInterrupt(() => { + markLost() + try { + host?.signal("SIGTERM") + } catch { + /* Teardown retries independently. */ + } + }) + try { + if (!apiKey) throw new Error("Provider credential missing") + host = await runtime.launch() + host.onExit(markLost) + client = runtime.connect() + client.onDisconnect(() => { + sessionLost = true + waiter?.onDisconnect() + }) + let taskId: string | undefined + client.onTaskEvent((event) => { + const id = waiter?.onTaskEvent(event) + if (id && taskId === undefined) taskId = id + collectEvidence(evidence, event, taskId) + }) + await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) + activated = client.isReady + if (!activated) code = "ACTIVATION_FAILED" + else { + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) + // A synchronous send failure must not leave a rejected timer promise. + void waiter.result.catch(() => {}) + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, + }) + try { + taskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + } + if (accepted) { + await until( + () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + SMOKE_TIMEOUT_MS, + ) + code = smokeCode(evidence, sessionLost) + } + } + } catch { + // Never expose thrown provider, filesystem or host messages. + } finally { + waiter?.onExit() + const attempt = async (operation: () => unknown) => { + try { + await operation() + } catch { + if (code === "OK") code = "TEARDOWN_FAILED" + } + } + if (client?.isReady) await attempt(() => client?.sendCommand({ commandName: TaskCommandName.CloseTask })) + if (host && !host.exited) { + await attempt(() => until(() => host?.exited === true, 5_000, false)) + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + if (host.exited) break + await attempt(() => host?.signal(signal)) + await attempt(() => until(() => host?.exited === true, 5_000, false)) + } + if (!host.exited && code === "OK") code = "TEARDOWN_FAILED" + } + await attempt(() => client?.disconnect()) + if (!host || host.exited) await attempt(() => runtime.cleanup()) + await attempt(unsubscribe) + } + const verdict: SmokeVerdict = { + kind: "provider-smoke", + model: SOLHEIM_MODEL, + provider: SOLHEIM_BASE_URL, + passed: code === "OK", + code, + gates: { + extensionActivated: activated, + ipcConnected: activated, + taskAccepted: accepted, + providerResponseSeen: evidence.providerResponseSeen, + completionSeen: evidence.completionSeen, + childExited: !host || host.exited, + }, + durationMs: runtime.now() - started, + } + await runtime.writeVerdict(verdict) + return verdict +} + +export const smokeExitCode = (verdict: SmokeVerdict): number => (verdict.passed ? 0 : 1) diff --git a/src/extension/__tests__/api-start-task-logging.spec.ts b/src/extension/__tests__/api-start-task-logging.spec.ts index a4fe87f22c..014991128d 100644 --- a/src/extension/__tests__/api-start-task-logging.spec.ts +++ b/src/extension/__tests__/api-start-task-logging.spec.ts @@ -52,18 +52,6 @@ const buildStartCommand = (overrides?: { requestId?: string; text?: string }) => }, }) -function withStageTimeoutOverride(ms: string): () => void { - const previous = process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS - process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = ms - return () => { - if (previous === undefined) { - delete process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS - } else { - process.env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS = previous - } - } -} - function buildApi(createTaskImpl?: () => Promise<{ taskId: string }>, postStateToWebviewImpl?: () => Promise) { const outputChannel = { appendLine: vi.fn() } as unknown as vscode.OutputChannel const provider = { @@ -99,6 +87,7 @@ describe("API StartNewTask IPC", () => { afterEach(() => { commandHandlers.length = 0 sentMessages.length = 0 + vi.useRealTimers() vi.restoreAllMocks() }) @@ -140,14 +129,15 @@ describe("API StartNewTask IPC", () => { await handler!("client-1", buildStartCommand({ requestId: "req-1" })) expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") const events = sentTaskEvents() expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) - const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success) { - expect(parsed.data).toEqual({ requestId: "req-1", success: true, taskId: "task-1" }) - } + expect(taskStartResponseSchema.parse(events[0]?.data.payload[0])).toEqual({ + requestId: "req-1", + success: true, + taskId: "task-1", + }) }) it("answers a failed correlated start with exactly one fixed safe failure response", async () => { @@ -160,16 +150,17 @@ describe("API StartNewTask IPC", () => { await handler!("client-1", buildStartCommand({ requestId: "req-2" })) expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") const events = sentTaskEvents() expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) - const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success && !parsed.data.success) { - expect(parsed.data.errorCode).toBe(TASK_START_FAILURE_ERROR_CODE) - expect(parsed.data.errorMessage).toBe(TASK_START_FAILURE_ERROR_MESSAGE) - expect(parsed.data.stage).toBe("unknown") - } + const parsed = taskStartResponseSchema.parse(events[0]?.data.payload[0]) + expect(parsed).toEqual({ + requestId: "req-2", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) const sent = JSON.stringify(sentMessages) expect(sent).not.toContain("opaque-solheim-secret-0123456789abcdef") @@ -177,113 +168,60 @@ describe("API StartNewTask IPC", () => { expect(sent).not.toContain("review the pull request diff") }) - it("bounds a hung task creation stage and answers with exactly one sanitized stage failure", async () => { - const restore = withStageTimeoutOverride("100") - try { - const { collectLogs } = buildApi(() => new Promise(() => {})) - - const handler = commandHandlers.at(-1) - await handler!("client-1", buildStartCommand({ requestId: "req-timeout-task" })) - - expect(sentMessages).toHaveLength(1) - const events = sentTaskEvents() - expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) - - const parsed = taskStartResponseSchema.safeParse(events[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success && !parsed.data.success) { - expect(parsed.data).toEqual({ - requestId: "req-timeout-task", - success: false, - errorCode: TASK_START_FAILURE_ERROR_CODE, - errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, - stage: "taskCreation", - }) - } - - const logged = collectLogs() - expect(logged).toContain("taskCreation") - expect(logged).toContain("req-timeout-task") - expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") - } finally { - restore() - } - }) - - it("removes a late task and refuses a retry until the timed-out creation settles", async () => { - const restore = withStageTimeoutOverride("50") - try { - let finishCreation: (task: { taskId: string }) => void = () => {} - const { provider } = buildApi( - () => - new Promise((resolve) => { - finishCreation = resolve - }), - ) - const handler = commandHandlers.at(-1)! - - await handler("client-1", buildStartCommand({ requestId: "req-first" })) - await handler("client-1", buildStartCommand({ requestId: "req-retry" })) - - expect(provider.createTask).toHaveBeenCalledTimes(1) - const retry = taskStartResponseSchema.parse(sentTaskEvents()[1]?.data.payload[0]) - expect(retry.success).toBe(false) - - finishCreation({ taskId: "late-task" }) - await vi.waitFor(() => expect(provider.removeClineFromStack).toHaveBeenCalledTimes(1)) - - await handler("client-1", buildStartCommand({ requestId: "req-after" })) - expect(provider.createTask).toHaveBeenCalledTimes(2) - } finally { - restore() - } - }) - - it("bounds a hung settings stage and names it in the failure response", async () => { - const restore = withStageTimeoutOverride("100") - try { - buildApi(undefined, () => new Promise(() => {})) - - const handler = commandHandlers.at(-1) - await handler!("client-1", buildStartCommand({ requestId: "req-timeout-settings" })) - - expect(sentMessages).toHaveLength(1) - const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success && !parsed.data.success) { - expect(parsed.data.stage).toBe("settings") - } - } finally { - restore() - } + it("does not race pending configuration/startup against a timeout", async () => { + vi.useFakeTimers() + let finishCreation!: (task: { taskId: string }) => void + const { provider } = buildApi( + () => + new Promise((resolve) => { + finishCreation = resolve + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-pending" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(provider.createTask).toHaveBeenCalledTimes(1) + expect(sentMessages).toHaveLength(0) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() + finishCreation({ taskId: "finished-task" }) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-pending", + success: true, + taskId: "finished-task", + }) }) - it("still answers with one success response when every stage is fast under a small bound", async () => { - const restore = withStageTimeoutOverride("100") - try { - buildApi() - - const handler = commandHandlers.at(-1) - await handler!("client-1", buildStartCommand({ requestId: "req-fast" })) - - expect(sentMessages).toHaveLength(1) - const parsed = taskStartResponseSchema.safeParse(sentTaskEvents()[0]?.data.payload[0]) - expect(parsed.success).toBe(true) - if (parsed.success) { - expect(parsed.data).toEqual({ requestId: "req-fast", success: true, taskId: "task-1" }) - } - } finally { - restore() - } + it("reports a delayed startup rejection only after it settles, without late task removal", async () => { + vi.useFakeTimers() + let rejectCreation!: (error: Error) => void + const { provider } = buildApi( + () => + new Promise((_resolve, reject) => { + rejectCreation = reject + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-reject" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(sentMessages).toHaveLength(0) + rejectCreation(new Error("private startup error")) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-reject", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() }) it("sends no start response for legacy IPC starts without requestId", async () => { - buildApi() + const { executeCommand } = buildApi() const handler = commandHandlers.at(-1) await handler!("client-1", buildStartCommand()) expect(sentMessages).toHaveLength(0) + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) }) it("keeps the sidebar focus for direct API callers that do not opt out", async () => { diff --git a/src/extension/api.ts b/src/extension/api.ts index 30ff68838e..3a33510346 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -14,7 +14,6 @@ import { type ProviderSettingsEntry, type TaskEvent, type TaskStartResponse, - type TaskStartStage, type CreateTaskOptions, type TaskApiConversationHistorySequence, type WebviewThemeFixture, @@ -38,51 +37,9 @@ import { Terminal } from "../integrations/terminal/Terminal" import { TerminalRegistry } from "../integrations/terminal/TerminalRegistry" import { getCommands } from "../services/command/commands" -// The default bound is per stage, not per start, so one slow stage cannot -// hide behind the budget of another. Secret storage writes through VS Code -// secret storage can hang indefinitely under Xvfb without a keyring service. -const DEFAULT_TASK_START_STAGE_TIMEOUT_MS = 60_000 - -function readTaskStartStageTimeoutMs(env: NodeJS.ProcessEnv): number { - const parsed = Number(env.ROO_CODE_TASK_START_STAGE_TIMEOUT_MS) - return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TASK_START_STAGE_TIMEOUT_MS -} - -class TaskStartStageTimeoutError extends Error { - constructor(readonly stage: TaskStartStage) { - super(`task start stage timed out: ${stage}`) - } -} - -// Races the stage against its bound. Losing the race stops the wait only. -// The operation keeps running, so onAbandon receives it for later cleanup. -async function withStageTimeout( - stage: TaskStartStage, - timeoutMs: number, - operation: () => T | Thenable, - onAbandon?: (running: Promise) => void, -): Promise { - let timer: NodeJS.Timeout | undefined - const running = Promise.resolve(operation()) - try { - return await Promise.race([ - running, - new Promise((_, reject) => { - timer = setTimeout(() => { - onAbandon?.(running) - reject(new TaskStartStageTimeoutError(stage)) - }, timeoutMs) - }), - ]) - } finally { - clearTimeout(timer) - } -} - export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel private readonly sidebarProvider: ClineProvider - private abandonedStarts = 0 private readonly ipc?: IpcServer private readonly log: (...args: unknown[]) => void private logfile?: string @@ -146,26 +103,22 @@ export class API extends EventEmitter implements RooCodeAPI { // Exactly one sanitized, client-scoped response per // correlated start. The reply never echoes the failure. - // Each startup stage is bounded so a hung await (for - // example secret storage under a missing keyring) - // still produces the reply. + // Await actual startup, including configuration writes. + // The smoke controller owns its timeout and terminates + // its isolated host; shared-profile writes are not raced. let response: TaskStartResponse try { const taskId = await this.startNewTask(command.data, { focusSidebar: false, - stageTimeoutMs: readTaskStartStageTimeoutMs(process.env), }) response = { requestId, success: true, taskId } - } catch (error) { - const stage: TaskStartStage = - error instanceof TaskStartStageTimeoutError ? error.stage : "unknown" - this.log("[API] StartNewTask failed", { requestId, stage }) + } catch { + this.log("[API] StartNewTask failed", { requestId }) response = { requestId, success: false, errorCode: TASK_START_FAILURE_ERROR_CODE, errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, - stage, } } sendResponse(RooCodeEventName.TaskStartResponse, [response]) @@ -265,61 +218,30 @@ export class API extends EventEmitter implements RooCodeAPI { images?: string[] newTab?: boolean }, - startOptions?: { focusSidebar?: boolean; stageTimeoutMs?: number }, + startOptions?: { focusSidebar?: boolean }, ) { - // Stage names for a bounded correlated start. The settings persistence - // named in the startup evidence (createTask -> setValues -> - // storeSecret) runs inside the taskCreation stage at this layer, so a - // taskCreation timeout covers it. - // A timed-out stage keeps running. Count it, and reject a new bounded - // start until it settles. A late task from the taskCreation stage is - // removed, so it cannot run after the failure reply. - if (startOptions?.stageTimeoutMs !== undefined && this.abandonedStarts > 0) { - throw new Error("an earlier task start is still running") - } - const boundStage = ( - stage: TaskStartStage, - operation: () => T | Thenable, - onLate?: (late: Awaited) => unknown, - ): Promise => - startOptions?.stageTimeoutMs === undefined - ? Promise.resolve(operation()) - : withStageTimeout(stage, startOptions.stageTimeoutMs, operation, (running) => { - this.abandonedStarts++ - void running - .then((late) => onLate?.(late as Awaited)) - .catch(() => {}) - .finally(() => this.abandonedStarts--) - }) - let provider: ClineProvider if (newTab) { - await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.files.revert")) - await boundStage("unknown", () => vscode.commands.executeCommand("workbench.action.closeAllEditors")) + await vscode.commands.executeCommand("workbench.action.files.revert") + await vscode.commands.executeCommand("workbench.action.closeAllEditors") - provider = await boundStage("unknown", () => this.providerFactory.createInNewTab()) + provider = await this.providerFactory.createInNewTab() this.registerListeners(provider) } else { // IPC callers omit focus: executeCommand cannot be cancelled and a // stalled webview launch would block the whole start. if (startOptions?.focusSidebar ?? true) { - await boundStage("unknown", () => - vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`), - ) + await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) } provider = this.sidebarProvider } - await boundStage("eviction", () => provider.evictCurrentTask()) - await boundStage("settings", () => provider.postStateToWebview()) - await boundStage("settings", () => - provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }), - ) - await boundStage("settings", () => - provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }), - ) + await provider.evictCurrentTask() + await provider.postStateToWebview() + await provider.postMessageToWebview({ type: "action", action: "chatButtonClicked" }) + await provider.postMessageToWebview({ type: "invoke", invoke: "newChat", text, images }) // API-driven tasks never stop to ask the user, so the hard stop is // effectively unlimited. The soft repetition tier is disabled too: @@ -330,11 +252,7 @@ export class API extends EventEmitter implements RooCodeAPI { toolRepetitionSoftLimit: 0, } - const task = await boundStage( - "taskCreation", - () => provider.createTask(text, images, undefined, options, configuration), - (late) => late && provider.removeClineFromStack(), - ) + const task = await provider.createTask(text, images, undefined, options, configuration) if (!task) { throw new Error("Failed to create task due to policy restrictions") From cd6244e7abb0316a4110e1ab778f751e783d941d Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 17 Oct 2026 23:18:01 +0000 Subject: [PATCH 03/11] feat(evals): add trusted fixture runner and execution traces Add scripts/agent-evals with a versioned fixture contract, registry, shared launcher, runner, and metadata-only execution trace. Rewire scripts/solheim-smoke as the first deterministic no-tools fixture on the shared contract. Wire agent-evals unit tests and type checks into Code QA and capture trace.json in the provider smoke artifacts. --- .github/workflows/code-qa.yml | 4 + .github/workflows/solheim-provider-smoke.yml | 1 + package.json | 2 + scripts/agent-evals/README.md | 83 +++ scripts/agent-evals/contract.test.ts | 99 ++++ scripts/agent-evals/contract.ts | 179 ++++++ scripts/agent-evals/isolation.ts | 36 ++ scripts/agent-evals/launcher.test.ts | 65 ++ scripts/agent-evals/launcher.ts | 189 ++++++ scripts/agent-evals/package.json | 4 + scripts/agent-evals/registry.test.ts | 23 + scripts/agent-evals/registry.ts | 22 + scripts/agent-evals/runner.test.ts | 587 +++++++++++++++++++ scripts/agent-evals/runner.ts | 379 ++++++++++++ scripts/agent-evals/task-start.ts | 96 +++ scripts/agent-evals/trace.test.ts | 221 +++++++ scripts/agent-evals/trace.ts | 402 +++++++++++++ scripts/agent-evals/tsconfig.json | 17 + scripts/solheim-smoke-workflow.test.mjs | 43 +- scripts/solheim-smoke/README.md | 27 +- scripts/solheim-smoke/config.ts | 38 +- scripts/solheim-smoke/driver.mts | 108 +--- scripts/solheim-smoke/fixture.ts | 89 +++ scripts/solheim-smoke/run.test.ts | 4 +- scripts/solheim-smoke/run.ts | 161 ++--- scripts/solheim-smoke/task-start.ts | 97 +-- 26 files changed, 2607 insertions(+), 369 deletions(-) create mode 100644 scripts/agent-evals/README.md create mode 100644 scripts/agent-evals/contract.test.ts create mode 100644 scripts/agent-evals/contract.ts create mode 100644 scripts/agent-evals/isolation.ts create mode 100644 scripts/agent-evals/launcher.test.ts create mode 100644 scripts/agent-evals/launcher.ts create mode 100644 scripts/agent-evals/package.json create mode 100644 scripts/agent-evals/registry.test.ts create mode 100644 scripts/agent-evals/registry.ts create mode 100644 scripts/agent-evals/runner.test.ts create mode 100644 scripts/agent-evals/runner.ts create mode 100644 scripts/agent-evals/task-start.ts create mode 100644 scripts/agent-evals/trace.test.ts create mode 100644 scripts/agent-evals/trace.ts create mode 100644 scripts/agent-evals/tsconfig.json create mode 100644 scripts/solheim-smoke/fixture.ts diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index e8cf511450..f03ee6b408 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -104,6 +104,10 @@ jobs: run: pnpm test:solheim-smoke:unit - name: Check Solheim provider smoke types run: pnpm solheim-smoke:check-types + - name: Test agent evals units + run: pnpm test:agent-evals:unit + - name: Check agent evals types + run: pnpm agent-evals:check-types - name: Model-check task lifecycle protocols run: pnpm lifecycle:model-check - name: Validate MCP OAuth integration diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index 7f61107186..f73ebd2c0b 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -55,6 +55,7 @@ jobs: name: solheim-provider-smoke-${{ github.sha }} path: | ${{ runner.temp }}/solheim-provider-smoke/verdict.json + ${{ runner.temp }}/solheim-provider-smoke/trace.json ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 if-no-files-found: error retention-days: 7 diff --git a/package.json b/package.json index 0fcde2ca98..6bb220713f 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs", "test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts", "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", + "test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts", + "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md new file mode 100644 index 0000000000..c1d71ade3d --- /dev/null +++ b/scripts/agent-evals/README.md @@ -0,0 +1,83 @@ +# Agent evals fixture substrate + +This directory holds the reusable, trusted fixture execution substrate. One +controller runs a registered fixture inside one isolated VS Code extension +host and writes a versioned, metadata-only structured trace. It carries no +knowledge of any provider, task solution, or grading rule. + +## Layout + +| File | Purpose | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`contract.ts`](./contract.ts) | Fixture contract and validation: identity, version, workspace files, prompt, tool policy, execution limits, settings projection, settings builder. | +| [`registry.ts`](./registry.ts) | Trusted in-memory registry. Fixtures enter only through validated registration from repo code. No file, URL, or dynamic loading exists. | +| [`runner.ts`](./runner.ts) | Execution controller: startup correlation, bounded event buffering, request accounting, bounded teardown. | +| [`trace.ts`](./trace.ts) | Trace schema version 1, request accountant, stable digests, fixed notes. | +| [`launcher.ts`](./launcher.ts) | The single VS Code launcher: download, isolated spawn, IPC client, atomic artifact writes, run identity. | +| [`isolation.ts`](./isolation.ts) | Child environment allowlist and launch arguments. | +| [`task-start.ts`](./task-start.ts) | Correlated StartNewTask waiter. | + +## Fixture contract + +A fixture is trusted in-repo data. The contract has no executable hooks, no +URLs, and no paths outside the declared workspace files. The only function is +`buildConfiguration(credential)`, which maps one credential to extension +settings. Validation rejects path traversal, drive letters, oversized files, +missing limits, and non-boolean tool allowances. The launcher repeats the +workspace containment check at write time. + +Execution limits are enforced, not decorative. The controller applies the +activation, task start, task run, usage drain, and teardown step limits, and +the trace records the measured durations. + +## Structured trace + +Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is +metadata only. Prompts, model output, reasoning, tool arguments, error +payloads, and storage paths are excluded. Provider settings appear only as +the explicit nonsecret projection in `settings`. + +Accounting rules: + +- Request rows upsert by message timestamp across created and updated + `api_req_started` messages. +- Usage snapshots come from extension events. The trace keeps the latest + snapshot only and never sums cumulative snapshots. +- Zero token or cost values can be defaults from the adapter or the task + layer. The trace labels event-sourced numbers and never claims a provider + measured a zero. +- `contextTokens` is an extension context estimate. +- `apiReqRetriedMarkers` counts retry marker messages. It is not an HTTP + retry count. +- Missing values carry a reason, such as `notReported` or + `noUsageEventsObserved`. They are never filled with zero. +- The settings digest is SHA-256 over an explicit nonsecret projection. The + run fails closed before launch if the projection contains the credential. +- Execution result, accounting completeness, and teardown status are three + separate sections. A teardown failure never changes an execution code. + +Events that arrive before the task start acknowledgment are buffered with a +fixed cap. After acceptance the controller replays only events that belong to +the accepted task and counts the rest as foreign or malformed. After an +extension-reported completion the controller drains late usage events for the +bounded drain window. + +## Checks + +```sh +pnpm test:agent-evals:unit +pnpm agent-evals:check-types +``` + +Both run offline. No check spawns VS Code or touches a provider. + +## Follow-up milestones + +The substrate supports these planned steps. None of them exist in this +change: + +1. Five frozen, real coding task fixtures with known-good workspaces. +2. Independent credential-free graders that judge workspace state only. +3. Paired exact trusted commit comparisons on the same fixtures. +4. Repeated runs with uncertainty reporting. +5. Advisory release integration aligned to the actual release workflows. diff --git a/scripts/agent-evals/contract.test.ts b/scripts/agent-evals/contract.test.ts new file mode 100644 index 0000000000..3f60100e5b --- /dev/null +++ b/scripts/agent-evals/contract.test.ts @@ -0,0 +1,99 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { FixtureValidationError, validateFixture, validateWorkspaceFilePath } from "./contract.ts" +import { buildSmokeFixture } from "../solheim-smoke/fixture.ts" +import { buildSmokeConfiguration, SMOKE_PROMPT } from "../solheim-smoke/config.ts" + +const expectFailure = (run: () => void, pattern: RegExp): void => { + assert.throws(run, (error: unknown) => error instanceof FixtureValidationError && pattern.test(error.message)) +} + +describe("agent fixture contract", () => { + it("accepts the registered smoke fixture", () => { + validateFixture(buildSmokeFixture()) + }) + + it("rejects malformed identity, prompt, and workspace declarations", () => { + const base = buildSmokeFixture() + expectFailure(() => validateFixture({ ...base, identity: { ...base.identity, name: "Bad Name" } }), /name/) + expectFailure(() => validateFixture({ ...base, identity: { ...base.identity, version: "1.0" } }), /version/) + expectFailure(() => validateFixture({ ...base, prompt: "" }), /prompt/) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "../escape.txt", content: "x" }] } }), + /segments/, + ) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "/etc/passwd", content: "x" }] } }), + /relative/, + ) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "a\\b.txt", content: "x" }] } }), + /forward slashes/, + ) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "C:run.txt", content: "x" }] } }), + /drive letter/, + ) + expectFailure( + () => + validateFixture({ ...base, workspace: { files: [{ path: "big.txt", content: "x".repeat(16_385) }] } }), + /too large/, + ) + }) + + it("rejects missing limits, non-boolean allowances, and hook-shaped builders", () => { + const base = buildSmokeFixture() + const limits = { ...base.limits } + delete (limits as Partial)["taskRunTimeoutMs"] + expectFailure(() => validateFixture({ ...base, limits: limits as typeof base.limits }), /taskRunTimeoutMs/) + expectFailure(() => validateFixture({ ...base, limits: { ...base.limits, taskRunTimeoutMs: 0 } }), /positive/) + expectFailure( + () => validateFixture({ ...base, limits: { ...base.limits, taskRunTimeoutMs: 601_000 } }), + /within bounds/, + ) + const allow = { ...base.toolPolicy.allow } + delete (allow as Partial)["execute"] + expectFailure( + () => validateFixture({ ...base, toolPolicy: { ...base.toolPolicy, allow: allow as typeof allow } }), + /execute/, + ) + expectFailure( + () => + validateFixture({ + ...base, + toolPolicy: { ...base.toolPolicy, disabledTools: [] }, + }), + /disable at least one tool/, + ) + const broken = buildSmokeFixture() as unknown as Record + broken["buildConfiguration"] = "not a function" + expectFailure(() => validateFixture(broken as unknown as typeof base), /builder/) + }) + + it("keeps the smoke projection in sync with the built configuration and free of credentials", () => { + const fixture = buildSmokeFixture() + const credential = "sk-live-credential" + assert.equal(fixture.prompt, SMOKE_PROMPT) + assert.equal(fixture.buildConfiguration, buildSmokeConfiguration) + const config = buildSmokeConfiguration(credential) + for (const [key, value] of Object.entries(fixture.settingsProjection)) { + const expected = + key === "openAiExtraBody" + ? JSON.parse(config.openAiExtraBody ?? "null") + : config[key as keyof typeof config] + assert.deepEqual(value, expected, `projection key ${key} drifts from the built configuration`) + } + assert.ok(!Object.keys(fixture.settingsProjection).includes("openAiApiKey")) + assert.ok(!JSON.stringify(fixture.settingsProjection).includes(credential)) + }) + + it("bounds workspace file paths at the contract layer", () => { + validateWorkspaceFilePath("docs/readme.md") + validateWorkspaceFilePath("src/nested/deep/file.ts") + expectFailure(() => validateWorkspaceFilePath(".."), /segments/) + expectFailure(() => validateWorkspaceFilePath("a/./b"), /segments/) + expectFailure(() => validateWorkspaceFilePath("a//b"), /segments/) + expectFailure(() => validateWorkspaceFilePath(""), /missing/) + expectFailure(() => validateWorkspaceFilePath(`a/${"x".repeat(300)}`), /out of bounds/) + }) +}) diff --git a/scripts/agent-evals/contract.ts b/scripts/agent-evals/contract.ts new file mode 100644 index 0000000000..b2e3a409d0 --- /dev/null +++ b/scripts/agent-evals/contract.ts @@ -0,0 +1,179 @@ +import type { RooCodeSettings } from "@roo-code/types" + +export interface AgentFixtureWorkspaceFile { + path: string + content: string +} + +export interface AgentFixtureToolAllow { + readOnly: boolean + readOnlyOutsideWorkspace: boolean + write: boolean + writeOutsideWorkspace: boolean + writeProtected: boolean + execute: boolean + mcp: boolean + modeSwitch: boolean + subtasks: boolean + followupQuestions: boolean +} + +export interface AgentFixtureLimits { + activationTimeoutMs: number + taskStartTimeoutMs: number + taskRunTimeoutMs: number + usageDrainMs: number + teardownStepMs: number +} + +// A fixture is trusted, in-repo data. The contract deliberately carries no +// executable hooks, no URLs to fetch, and no file paths outside the declared +// workspace files. The only function is the settings builder, which maps a +// credential to extension settings and never receives anything else. +export interface AgentFixture { + identity: { + name: string + version: string + description: string + } + prompt: string + workspace: { + files: readonly AgentFixtureWorkspaceFile[] + } + toolPolicy: { + mode: string + roleDefinition: string + groups: readonly string[] + disabledTools: readonly string[] + allow: AgentFixtureToolAllow + } + limits: AgentFixtureLimits + // Explicit nonsecret mirror of the built configuration. The trace digests + // this projection; it must never contain the credential. + settingsProjection: Record + buildConfiguration(credential: string): RooCodeSettings +} + +export class FixtureValidationError extends Error {} + +const NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/ +const VERSION_PATTERN = /^\d+\.\d+\.\d+$/ +const MAX_DESCRIPTION_CHARS = 300 +const MAX_PROMPT_CHARS = 8_000 +const MAX_MODE_CHARS = 64 +const MAX_ROLE_CHARS = 2_000 +const MAX_PATH_CHARS = 256 +const MAX_FILES = 64 +const MAX_FILE_CHARS = 16_384 +const MAX_TOTAL_FILE_CHARS = 262_144 +const MAX_PROJECTION_CHARS = 16_384 +const MAX_LIMIT_MS = 600_000 +const MAX_DRAIN_MS = 60_000 + +const ALLOW_KEYS = [ + "readOnly", + "readOnlyOutsideWorkspace", + "write", + "writeOutsideWorkspace", + "writeProtected", + "execute", + "mcp", + "modeSwitch", + "subtasks", + "followupQuestions", +] as const satisfies readonly (keyof AgentFixtureToolAllow)[] + +const LIMIT_KEYS = [ + "activationTimeoutMs", + "taskStartTimeoutMs", + "taskRunTimeoutMs", + "usageDrainMs", + "teardownStepMs", +] as const satisfies readonly (keyof AgentFixtureLimits)[] + +const fail = (message: string): never => { + throw new FixtureValidationError(message) +} + +const isPlainObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const boundedString = (value: unknown, max: number, label: string): string => { + if (typeof value !== "string" || value.length === 0 || value.length > max) + return fail(`${label} missing or out of bounds`) + return value +} + +// Trust-boundary check. The launcher repeats the containment check at write +// time; this check rejects malformed fixture data at registration. +export function validateWorkspaceFilePath(pathValue: string): void { + if (typeof pathValue !== "string" || pathValue.length === 0 || pathValue.length > MAX_PATH_CHARS) + fail("workspace file path missing or out of bounds") + if (pathValue.includes("\\")) fail("workspace file path must use forward slashes") + if (pathValue.includes("\0")) fail("workspace file path must not contain a NUL byte") + if (pathValue.startsWith("/")) fail("workspace file path must be relative") + if (/^[a-zA-Z]:/.test(pathValue)) fail("workspace file path must not carry a drive letter") + for (const segment of pathValue.split("/")) + if (segment === "" || segment === "." || segment === "..") + fail("workspace file path segments must be plain names") +} + +const serializeProjection = (projection: Record): string | undefined => { + try { + return JSON.stringify(projection) + } catch { + return undefined + } +} + +export function validateFixture(fixture: AgentFixture): void { + if (!isPlainObject(fixture)) fail("fixture must be an object") + if (!isPlainObject(fixture.identity)) fail("fixture identity missing") + boundedString(fixture.identity.name, 64, "fixture name") + if (!NAME_PATTERN.test(fixture.identity.name)) fail("fixture name must be lowercase kebab case") + if (typeof fixture.identity.version !== "string" || !VERSION_PATTERN.test(fixture.identity.version)) + fail("fixture version must be semver") + boundedString(fixture.identity.description, MAX_DESCRIPTION_CHARS, "fixture description") + boundedString(fixture.prompt, MAX_PROMPT_CHARS, "fixture prompt") + + if (!isPlainObject(fixture.workspace) || !Array.isArray(fixture.workspace.files)) fail("fixture workspace missing") + if (fixture.workspace.files.length > MAX_FILES) fail("too many workspace files") + let totalChars = 0 + for (const file of fixture.workspace.files) { + if (!isPlainObject(file)) fail("workspace file must be an object") + validateWorkspaceFilePath(file.path) + if (typeof file.content !== "string" || file.content.length > MAX_FILE_CHARS) + fail("workspace file content missing or too large") + totalChars += file.content.length + if (totalChars > MAX_TOTAL_FILE_CHARS) fail("workspace files exceed the total size budget") + } + + if (!isPlainObject(fixture.toolPolicy)) fail("fixture tool policy missing") + boundedString(fixture.toolPolicy.mode, MAX_MODE_CHARS, "tool policy mode") + boundedString(fixture.toolPolicy.roleDefinition, MAX_ROLE_CHARS, "tool policy role definition") + if (!Array.isArray(fixture.toolPolicy.groups)) fail("tool policy groups missing") + if (!fixture.toolPolicy.groups.every((group) => typeof group === "string")) + fail("tool policy groups must be strings") + if (!Array.isArray(fixture.toolPolicy.disabledTools) || fixture.toolPolicy.disabledTools.length === 0) + fail("tool policy must disable at least one tool") + if (!fixture.toolPolicy.disabledTools.every((tool) => typeof tool === "string")) + fail("disabled tools must be strings") + if (!isPlainObject(fixture.toolPolicy.allow)) fail("tool policy allowances missing") + for (const key of ALLOW_KEYS) + if (typeof fixture.toolPolicy.allow[key] !== "boolean") fail(`tool policy allowance ${key} must be boolean`) + + if (!isPlainObject(fixture.limits)) fail("fixture limits missing") + for (const key of LIMIT_KEYS) { + const limit = fixture.limits[key] + if (!Number.isInteger(limit) || (limit as number) <= 0 || (limit as number) > MAX_LIMIT_MS) + fail(`fixture limit ${key} must be a positive integer within bounds`) + } + if ((fixture.limits.usageDrainMs as number) > MAX_DRAIN_MS) fail("usage drain bound exceeds its cap") + + if (!isPlainObject(fixture.settingsProjection)) fail("settings projection missing") + const projectionJson = serializeProjection(fixture.settingsProjection) + if (projectionJson === undefined || projectionJson.length > MAX_PROJECTION_CHARS) + fail("settings projection out of bounds") + + if (typeof fixture.buildConfiguration !== "function") fail("fixture settings builder missing") +} diff --git a/scripts/agent-evals/isolation.ts b/scripts/agent-evals/isolation.ts new file mode 100644 index 0000000000..06ab950ef6 --- /dev/null +++ b/scripts/agent-evals/isolation.ts @@ -0,0 +1,36 @@ +import path from "node:path" + +// Inherit only host-launch necessities, never the credential-bearing parent environment. +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, + socketPath: string, +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of ["PATH", "DISPLAY", "XAUTHORITY", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT", "WINDIR"]) + if (env[name]) child[name] = env[name] + return { + ...child, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), + ROO_CODE_IPC_SOCKET_PATH: socketPath, + } +} + +export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { + return [ + workspace, + `--user-data-dir=${userData}`, + `--extensions-dir=${extensions}`, + `--extensionDevelopmentPath=${extension}`, + "--no-sandbox", + "--disable-gpu-sandbox", + "--password-store=basic", + "--disable-updates", + "--skip-welcome", + "--skip-release-notes", + "--disable-workspace-trust", + ] +} diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts new file mode 100644 index 0000000000..905ea6d1de --- /dev/null +++ b/scripts/agent-evals/launcher.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { fileURLToPath } from "node:url" +import { createHostRuntime } from "./launcher.ts" + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +const tempDir = async (prefix: string): Promise => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)) + return dir +} + +describe("host runtime", () => { + it("writes artifacts atomically and replaces them on rewrite", async () => { + const dir = await tempDir("agent-evals-artifacts-") + try { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: dir }) + await runtime.writeArtifact("trace.json", { v: 1 }) + assert.equal(await fs.readFile(path.join(dir, "trace.json"), "utf8"), JSON.stringify({ v: 1 }, null, 2)) + await runtime.writeArtifact("trace.json", { v: 2 }) + assert.equal(JSON.parse(await fs.readFile(path.join(dir, "trace.json"), "utf8")).v, 2) + const leftovers = (await fs.readdir(dir)).filter((name) => name.endsWith(".tmp")) + assert.deepEqual(leftovers, []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("contains artifact names inside the artifacts directory", async () => { + const dir = await tempDir("agent-evals-artifacts-") + try { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: dir }) + await runtime.writeArtifact("../escape.json", { ok: true }) + assert.ok((await fs.stat(path.join(dir, "escape.json"))).isFile()) + assert.equal( + await fs.stat(path.join(dir, "..", "escape.json")).then( + () => true, + () => false, + ), + false, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("reports run identity from the repository, or bounded unavailability", async () => { + const dir = await tempDir("agent-evals-identity-") + try { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: dir }) + const identity = await runtime.readIdentity?.() + assert.ok(identity) + assert.match(identity.nodeVersion, /^v\d+\./) + assert.equal(["linux", "darwin", "win32"].includes(identity.platform), true) + if (identity.commit.available) assert.match(identity.commit.value, /^[0-9a-f]{40}$/) + if (identity.extensionVersion.available) assert.match(identity.extensionVersion.value, /^\d+\.\d+\.\d+/) + if (identity.vscodeTarget.available) assert.match(identity.vscodeTarget.value, /^\d+\.\d+\.\d+/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts new file mode 100644 index 0000000000..1edd2917d2 --- /dev/null +++ b/scripts/agent-evals/launcher.ts @@ -0,0 +1,189 @@ +import { execFile, spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { promisify } from "node:util" +import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { IpcClient } from "@roo-code/ipc" +import { IpcMessageType } from "@roo-code/types" +import { buildChildEnvironment, buildLaunchArgs } from "./isolation.ts" +import type { AgentClient, AgentHost, AgentRuntime } from "./runner.ts" +import { identityUnavailable, measured, type MeasuredString, type TraceRunIdentity } from "./trace.ts" + +const execFileAsync = promisify(execFile) + +export interface HostRuntimeOptions { + root: string + artifactsDir: string + now?: () => number + pause?: (ms: number) => Promise +} + +const readCommit = async (root: string): Promise => { + try { + const fromCi = process.env.GITHUB_SHA + if (fromCi && /^[0-9a-f]{40}$/i.test(fromCi)) return { available: true, value: fromCi } + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) + const sha = stdout.trim() + return /^[0-9a-f]{40}$/i.test(sha) + ? { available: true, value: sha } + : { available: false, reason: "commitUnreadable" } + } catch { + return { available: false, reason: "gitUnavailable" } + } +} + +const readJsonField = async ( + file: string, + read: (parsed: Record) => unknown, +): Promise => { + try { + const parsed = JSON.parse(await fs.readFile(file, "utf8")) as Record + const value = read(parsed) + return measured(typeof value === "string" ? value : undefined, "fieldMissing") + } catch { + return { available: false, reason: "fileUnreadable" } + } +} + +// The only place where VS Code is downloaded, spawned, isolated and torn +// down. Both the smoke driver and future adapters get the same launcher, so +// process and credential isolation cannot drift between entry points. +export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { + const { root, artifactsDir } = options + let temp: string | undefined + + return { + async launch(workspaceFiles) { + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "agent-evals-")) + const workspace = path.join(temp, "workspace") + const isolatedHome = path.join(temp, "home") + await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder))) + const workspaceRoot = await fs.realpath(workspace) + for (const file of workspaceFiles) { + const target = path.resolve(workspaceRoot, file.path) + if (target !== workspaceRoot && !target.startsWith(workspaceRoot + path.sep)) + throw new Error("workspace file path escapes the workspace") + await fs.mkdir(path.dirname(target), { recursive: true }) + await fs.writeFile(target, file.content) + if (!(await fs.realpath(target)).startsWith(workspaceRoot + path.sep)) + throw new Error("workspace file escapes the workspace") + } + const socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs( + workspace, + path.join(temp, "userdata"), + path.join(temp, "extensions"), + extension, + ) + const child = spawn( + process.env.DISPLAY ? executable : "xvfb-run", + process.env.DISPLAY ? args : ["-a", executable, ...args], + { + detached: true, + env: buildChildEnvironment(process.env, isolatedHome, socket), + stdio: "ignore", + }, + ) + let exited = false + let onExit = () => {} + const closed = () => { + exited = true + onExit() + } + child.once("error", () => { + // Spawn failure has no live PID; a failed kill must not claim exit. + if (!child.pid) closed() + else onExit() + }) + child.once("close", closed) + const host: AgentHost = { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + if (exited) listener() + }, + signal(signal) { + if (!child.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } + }, + } + return host + }, + connect(): AgentClient { + const socket = path.join(temp ?? "", "ipc.sock") + const client = new IpcClient(socket, () => {}) + return { + get isReady() { + return client.isReady + }, + onTaskEvent(listener) { + client.on(IpcMessageType.TaskEvent, listener) + }, + onDisconnect(listener) { + client.on(IpcMessageType.Disconnect, listener) + }, + sendCommand(command) { + client.sendCommand(command) + }, + disconnect() { + client.disconnect() + }, + } + }, + async cleanup() { + if (temp) await fs.rm(temp, { recursive: true, force: true }) + }, + async writeArtifact(fileName, value) { + // Atomic, whitelisted metadata only; no provider text or host logs. + const safeName = path.basename(fileName) + const pending = path.join(artifactsDir, `${safeName}.${process.pid}.tmp`) + await fs.writeFile(pending, JSON.stringify(value, null, 2)) + await fs.rename(pending, path.join(artifactsDir, safeName)) + }, + onInterrupt(listener) { + process.once("SIGINT", listener) + process.once("SIGTERM", listener) + return () => { + process.removeListener("SIGINT", listener) + process.removeListener("SIGTERM", listener) + } + }, + now: options.now ?? Date.now, + pause: options.pause ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + async readIdentity(): Promise { + const fallback = identityUnavailable() + return { + commit: await readCommit(root), + extensionVersion: await readJsonField( + path.join(root, "src", "package.json"), + (parsed) => parsed["version"], + ), + vscodeTarget: await readJsonField(path.join(root, "apps", "vscode-e2e", "package.json"), (parsed) => { + const devDependencies = parsed["devDependencies"] + return typeof devDependencies === "object" && devDependencies !== null + ? (devDependencies as Record)["@types/vscode"] + : undefined + }), + nodeVersion: fallback.nodeVersion, + platform: fallback.platform, + arch: fallback.arch, + } + }, + } +} diff --git a/scripts/agent-evals/package.json b/scripts/agent-evals/package.json new file mode 100644 index 0000000000..c4fb2812b9 --- /dev/null +++ b/scripts/agent-evals/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/scripts/agent-evals/registry.test.ts b/scripts/agent-evals/registry.test.ts new file mode 100644 index 0000000000..a8ddf213e8 --- /dev/null +++ b/scripts/agent-evals/registry.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { getFixture, listFixtures, registerFixture } from "./registry.ts" +import { buildSmokeFixture, ensureSmokeFixtureRegistered, SMOKE_FIXTURE_NAME } from "../solheim-smoke/fixture.ts" + +describe("trusted fixture registry", () => { + it("registers, lists, and returns the smoke fixture by exact name", () => { + ensureSmokeFixtureRegistered() + assert.ok(listFixtures().includes(SMOKE_FIXTURE_NAME)) + assert.equal(getFixture(SMOKE_FIXTURE_NAME).identity.name, SMOKE_FIXTURE_NAME) + }) + + it("rejects duplicate registration and unknown names", () => { + assert.throws(() => registerFixture(buildSmokeFixture()), /already registered/) + assert.throws(() => getFixture("does-not-exist"), /unknown fixture/) + }) + + it("validates fixtures at the registry boundary", () => { + const broken = buildSmokeFixture() + const invalid = { ...broken, identity: { ...broken.identity, name: "UPPER" } } + assert.throws(() => registerFixture(invalid), /name/) + }) +}) diff --git a/scripts/agent-evals/registry.ts b/scripts/agent-evals/registry.ts new file mode 100644 index 0000000000..c1806cc737 --- /dev/null +++ b/scripts/agent-evals/registry.ts @@ -0,0 +1,22 @@ +import { validateFixture, type AgentFixture } from "./contract.ts" + +// Trusted in-memory registry. Fixtures enter only through validated +// registration from repo code. There is no file loading, no URL loading, and +// no dynamic import, so a registry name can never select untrusted content. +const fixtures = new Map() + +export function registerFixture(fixture: AgentFixture): void { + validateFixture(fixture) + if (fixtures.has(fixture.identity.name)) throw new Error(`fixture already registered: ${fixture.identity.name}`) + fixtures.set(fixture.identity.name, fixture) +} + +export function getFixture(name: string): AgentFixture { + const fixture = fixtures.get(name) + if (!fixture) throw new Error(`unknown fixture: ${name}`) + return fixture +} + +export function listFixtures(): string[] { + return [...fixtures.keys()].sort() +} diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts new file mode 100644 index 0000000000..0dc2938b47 --- /dev/null +++ b/scripts/agent-evals/runner.test.ts @@ -0,0 +1,587 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" +import { registerFixture } from "./registry.ts" +import { runAgentFixture, type AgentRuntime, type AgentRunResult } from "./runner.ts" +import { digestProjection, type AgentTrace } from "./trace.ts" +import type { AgentFixture } from "./contract.ts" + +const CREDENTIAL = "sk-sentinel-credential-123" + +const allow = { + readOnly: false, + readOnlyOutsideWorkspace: false, + write: false, + writeOutsideWorkspace: false, + writeProtected: false, + execute: false, + mcp: false, + modeSwitch: false, + subtasks: false, + followupQuestions: false, +} + +function registerTestFixture(name: string, overrides: Partial = {}): AgentFixture { + const fixture: AgentFixture = { + identity: { name, version: "1.0.0", description: "Synthetic controller test fixture." }, + prompt: `synthetic prompt for ${name}`, + workspace: { files: [] }, + toolPolicy: { + mode: "test", + roleDefinition: "Run one synthetic task.", + groups: [], + disabledTools: ["new_task"], + allow, + }, + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 5_000, + teardownStepMs: 5_000, + }, + settingsProjection: { provider: "test", model: `model-${name}` }, + buildConfiguration: () => ({ apiProvider: "openai", openAiApiKey: CREDENTIAL }), + ...overrides, + } + registerFixture(fixture) + return fixture +} + +type Scenario = "success" | "setup" | "activation" | "start-failure" | "disconnect" | "no-events" | "foreign" + +function harness( + scenario: Scenario, + options: { stuck?: boolean; closeThrows?: boolean; timeout?: () => void; onPause?: (count: number) => void } = {}, +) { + let now = 0 + let pauseCount = 0 + let exited = false + let onExit = () => {} + let onDisconnect = () => {} + let onEvent: (event: unknown) => void = () => {} + let interrupt = () => {} + let cleaned = false + let disconnected = false + let unsubscribed = false + let launched = false + let workspaceFiles: readonly { path: string; content: string }[] = [] + const commands: TaskCommand[] = [] + const signals: NodeJS.Signals[] = [] + const artifacts: { name: string; value: unknown }[] = [] + // Events emitted before the run registers its listener; they flush when + // the task command is sent, matching real post-connect delivery. + const pendingEvents: unknown[] = [] + const emit = (event: unknown) => { + if (launched) onEvent(event) + else pendingEvents.push(event) + } + const close = () => { + exited = true + onExit() + } + const runtime: AgentRuntime = { + async launch(files) { + launched = true + workspaceFiles = files + if (scenario === "setup") throw new Error("private setup detail") + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + }, + signal(signal) { + signals.push(signal) + if (!options.stuck) close() + }, + } + }, + connect() { + return { + isReady: scenario !== "activation", + onTaskEvent(listener) { + onEvent = listener + }, + onDisconnect(listener) { + onDisconnect = listener + }, + disconnect() { + disconnected = true + }, + sendCommand(command) { + commands.push(command) + if (command.commandName === TaskCommandName.CloseTask) { + if (options.closeThrows) throw new Error("private close error") + if (!options.stuck) close() + return + } + if (command.commandName !== TaskCommandName.StartNewTask) return + for (const event of pendingEvents.splice(0)) onEvent(event) + if (options.timeout) { + options.timeout() + return + } + onEvent({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + scenario === "start-failure" + ? { + requestId: command.data.requestId, + success: false, + errorCode: "task_start_failed", + errorMessage: "Task start failed", + } + : { requestId: command.data.requestId, success: true, taskId: "root" }, + ], + }) + const taskId = scenario === "foreign" ? "other" : "root" + if (scenario === "foreign") { + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 1, totalTokensOut: 1, totalCost: 0, contextTokens: 1 }, + {}, + ], + }) + } + if (scenario === "success") { + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "created", + message: { + ts: 1, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai" }), + }, + }, + ], + }) + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "updated", + message: { + ts: 1, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai", tokensIn: 7, tokensOut: 3 }), + }, + }, + ], + }) + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + ], + }) + } + if (scenario === "disconnect") onDisconnect() + }, + } + }, + async cleanup() { + cleaned = true + }, + async writeArtifact(name, value) { + artifacts.push({ name, value }) + }, + onInterrupt(listener) { + interrupt = listener + return () => { + unsubscribed = true + } + }, + now: () => now, + async pause(ms) { + now += ms + pauseCount++ + options.onPause?.(pauseCount) + }, + } + return { + runtime, + commands, + signals, + artifacts, + emit, + interrupt: () => interrupt(), + get cleaned() { + return cleaned + }, + get disconnected() { + return disconnected + }, + get unsubscribed() { + return unsubscribed + }, + get launched() { + return launched + }, + get workspaceFiles() { + return workspaceFiles + }, + } +} + +const writtenTrace = (f: ReturnType): AgentTrace => { + const entry = f.artifacts.find((artifact) => artifact.name === "trace.json") + assert.ok(entry) + return entry.value as AgentTrace +} + +describe("agent fixture controller", () => { + it("runs a fixture, writes a metadata-only trace, and keeps credentials and prompts out", async () => { + registerTestFixture("controller-success") + const f = harness("success") + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-success", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.activated, true) + assert.equal(result.execution.taskAccepted, true) + assert.equal(result.teardown.status, "ok") + assert.equal(result.teardown.hostExited, true) + assert.deepEqual( + f.artifacts.map((artifact) => artifact.name), + ["trace.json"], + ) + const trace = writtenTrace(f) + assert.equal(trace.traceSchemaVersion, 1) + assert.equal(trace.kind, "agent-evals-trace") + assert.equal(trace.settings.available, true) + assert.equal(trace.settings.digest, digestProjection({ provider: "test", model: "model-controller-success" })) + assert.equal(trace.accounting.requestCount, 1) + assert.deepEqual(trace.accounting.requests[0]?.tokensIn, { available: true, value: 7 }) + assert.equal(trace.accounting.requests[0]?.messageCount, 2) + assert.equal(trace.accounting.usage.snapshotsObserved, 1) + assert.equal(trace.accounting.completeness.status, "complete") + assert.deepEqual(trace.accounting.completeness.reasons, []) + const serialized = JSON.stringify(trace) + assert.ok(!serialized.includes(CREDENTIAL)) + assert.ok(!serialized.includes("synthetic prompt")) + assert.ok(!serialized.includes("root")) + assert.ok(f.cleaned && f.disconnected && f.unsubscribed) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + }) + + it("buffers events before the start ack, replays only accepted task events, and counts the rest", async () => { + registerTestFixture("controller-buffer", { + limits: { + activationTimeoutMs: 100, + taskStartTimeoutMs: 100, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("foreign") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-buffer", credential: CREDENTIAL }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + const trace = writtenTrace(f) + assert.equal(trace.eventFlow.observed, 2) + assert.equal(trace.eventFlow.foreignTaskEvents, 1) + assert.equal(trace.eventFlow.replayedAfterAck, 1) + assert.ok(trace.accounting.completeness.reasons.includes("foreignTaskEventsDropped")) + assert.ok(trace.accounting.completeness.reasons.includes("noUsageEventsObserved")) + + const mixed = harness("success") + mixed.emit({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId: "nope", success: true, taskId: "x" }], + }) + const accepted: TaskEvent[] = [] + const result2 = await runAgentFixture(mixed.runtime, { + fixtureName: "controller-buffer", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => accepted.push(event), + shouldStop: () => accepted.length >= 3, + }, + }) + assert.equal(result2.execution.code, "STOPPED") + assert.equal(writtenTrace(mixed).eventFlow.replayedAfterAck, 3) + assert.equal(accepted.length, 3) + }) + + it("bounds the run timeout, the activation wait, and the usage drain", async () => { + registerTestFixture("controller-fast", { + limits: { + activationTimeoutMs: 200, + taskStartTimeoutMs: 200, + taskRunTimeoutMs: 200, + usageDrainMs: 500, + teardownStepMs: 100, + }, + }) + const timedOut = harness("no-events") + const timeoutResult = await runAgentFixture(timedOut.runtime, { + fixtureName: "controller-fast", + credential: CREDENTIAL, + }) + assert.equal(timeoutResult.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(timeoutResult.execution.stopReason, "runTimeout") + assert.equal(writtenTrace(timedOut).execution.timings.taskRunMs, 200) + assert.equal(writtenTrace(timedOut).accounting.usage.available, false) + + const neverReady = harness("activation") + const activationResult = await runAgentFixture(neverReady.runtime, { + fixtureName: "controller-fast", + credential: CREDENTIAL, + }) + assert.equal(activationResult.execution.code, "ACTIVATION_FAILED") + assert.equal(writtenTrace(neverReady).execution.timings.activationMs, 200) + assert.equal(neverReady.commands.length, 0) + assert.ok(neverReady.cleaned) + + registerTestFixture("controller-drain", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 500, + teardownStepMs: 5_000, + }, + }) + let drained = false + const withCompletion = harness("success", { + onPause: (count) => { + if (count >= 2 && !drained) { + drained = true + withCompletion.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 40, totalTokensOut: 20, totalCost: 0, contextTokens: 30 }, + { attempt_completion: { attempts: 1, failures: 0 } }, + ], + }) + } + }, + }) + withCompletion.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + const drainResult = await runAgentFixture(withCompletion.runtime, { + fixtureName: "controller-drain", + credential: CREDENTIAL, + }) + assert.equal(drainResult.execution.code, "COMPLETED") + assert.equal(drainResult.execution.stopReason, "taskCompleted") + const drainTrace = writtenTrace(withCompletion) + assert.equal(drainTrace.execution.timings.usageDrainMs, 500) + assert.equal(drainTrace.accounting.usage.snapshotsObserved, 3) + assert.equal(drainTrace.accounting.usage.lastSnapshot?.source, "taskTokenUsageUpdated") + assert.equal(drainTrace.accounting.usage.lastSnapshot?.tokensIn, 40) + assert.deepEqual(drainTrace.accounting.toolSnapshot.snapshot, { + attempt_completion: { attempts: 1, failures: 0 }, + }) + }) + + it("records teardown failure separately from execution, and keeps live-host storage out of cleanup", async () => { + registerTestFixture("controller-stuck") + const f = harness("success", { stuck: true }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-stuck", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.teardown.status, "failed") + assert.equal(result.teardown.hostExited, false) + assert.deepEqual(result.teardown.escalatedSignals, ["SIGTERM", "SIGKILL"]) + assert.equal(result.teardown.cleanup, "skippedLiveHost") + assert.equal(f.cleaned, false) + assert.equal(f.disconnected, true) + const trace = writtenTrace(f) + assert.equal(trace.teardown.status, "failed") + assert.equal(trace.execution.timings.teardownMs, 15_000) + }) + + it("keeps teardown moving after a CloseTask exception", async () => { + registerTestFixture("controller-closerror") + const f = harness("success", { closeThrows: true }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-closerror", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.teardown.status, "failed") + assert.deepEqual(result.teardown.failures, ["closeTask"]) + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + }) + + it("fails closed on setup errors and still writes a trace", async () => { + registerTestFixture("controller-setup") + const f = harness("setup") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-setup", credential: CREDENTIAL }) + assert.equal(result.execution.code, "SETUP_FAILED") + assert.equal(result.execution.stopReason, "setupError") + const trace = writtenTrace(f) + assert.ok(trace.accounting.completeness.reasons.includes("taskNotAccepted")) + assert.ok(f.cleaned && f.unsubscribed) + }) + + it("fails closed without a credential before launching", async () => { + registerTestFixture("controller-nocred") + const f = harness("success") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-nocred", credential: "" }) + assert.equal(result.execution.code, "SETUP_FAILED") + assert.equal(result.execution.stopReason, "credentialMissing") + assert.equal(f.launched, false) + const trace = writtenTrace(f) + assert.ok(trace) + // The settings section must name the real cause, not claim a + // fixture rejection that never happened. + assert.deepEqual(trace.settings, { available: false, reason: "credentialMissing" }) + }) + + it("rejects a fixture whose projection carries the credential, and never launches", async () => { + registerTestFixture("controller-leak", { settingsProjection: { apiKey: CREDENTIAL } }) + const f = harness("success") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-leak", credential: CREDENTIAL }) + assert.equal(result.execution.code, "SETUP_FAILED") + assert.equal(result.execution.stopReason, "fixtureRejected") + assert.equal(f.launched, false) + const trace = writtenTrace(f) + assert.equal(trace.settings.available, false) + assert.equal(trace.settings.reason, "credentialInProjection") + assert.ok(!JSON.stringify(trace).includes(CREDENTIAL)) + }) + + it("counts malformed events and tool failures without storing payloads", async () => { + registerTestFixture("controller-noise") + const f = harness("success") + f.emit("garbage") + f.emit({ + eventName: RooCodeEventName.TaskToolFailed, + payload: ["root", "read_file", "private failure payload"], + }) + await runAgentFixture(f.runtime, { + fixtureName: "controller-noise", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + const trace = writtenTrace(f) + assert.equal(trace.eventFlow.malformed, 1) + assert.equal(trace.accounting.toolFailureEvents, 1) + assert.equal(trace.accounting.toolFailuresByTool["read_file"], 1) + assert.ok(!JSON.stringify(trace).includes("private failure payload")) + }) + + it("captures injected identity and reports unavailable identity with a reason", async () => { + registerTestFixture("controller-identity") + const withIdentity = harness("success") + withIdentity.runtime.readIdentity = async () => ({ + commit: { available: true, value: "a".repeat(40) }, + extensionVersion: { available: true, value: "3.0.0" }, + vscodeTarget: { available: true, value: "1.100.0" }, + nodeVersion: "v22.0.0", + platform: "linux", + arch: "x64", + }) + await runAgentFixture(withIdentity.runtime, { + fixtureName: "controller-identity", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.deepEqual(writtenTrace(withIdentity).run.identity.commit, { available: true, value: "a".repeat(40) }) + + const without = harness("success") + const plain = await runAgentFixture(without.runtime, { + fixtureName: "controller-identity", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.deepEqual(writtenTrace(without).run.identity.commit, { available: false, reason: "identityUnavailable" }) + assert.equal(plain.trace.run.identity.nodeVersion, process.version) + }) + + it("caveats accounting when the session dies inside the usage drain", async () => { + registerTestFixture("controller-drainloss", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 500, + teardownStepMs: 5_000, + }, + }) + let interruptCalled = false + const withCompletion = harness("success", { + onPause: (count) => { + // The drain is the first pause: the run loop exits on the + // pre-emitted completion without polling. + if (count === 1 && !interruptCalled) { + interruptCalled = true + withCompletion.interrupt() + } + }, + }) + withCompletion.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + const result = await runAgentFixture(withCompletion.runtime, { + fixtureName: "controller-drainloss", + credential: CREDENTIAL, + }) + // Completion was observed, so the run code stays COMPLETED and the + // run-phase session flag stays false. + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.sessionLost, false) + const trace = writtenTrace(withCompletion) + assert.equal(trace.execution.timings.usageDrainMs, 100) + assert.ok(trace.accounting.completeness.reasons.includes("sessionLostDuringUsageDrain")) + assert.ok(!trace.accounting.completeness.reasons.includes("sessionLost")) + assert.equal(trace.accounting.completeness.status, "partial") + }) + + it("passes fixture workspace files to the launcher and records session loss", async () => { + registerTestFixture("controller-workspace", { + workspace: { files: [{ path: "src/task.md", content: "fixture data" }] }, + }) + const f = harness("success") + await runAgentFixture(f.runtime, { + fixtureName: "controller-workspace", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.deepEqual([...f.workspaceFiles], [{ path: "src/task.md", content: "fixture data" }]) + + const lost = harness("disconnect") + const result = await runAgentFixture(lost.runtime, { + fixtureName: "controller-workspace", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "SESSION_LOST") + assert.equal(result.execution.stopReason, "sessionLost") + assert.equal(result.execution.sessionLost, true) + }) +}) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts new file mode 100644 index 0000000000..aa25623ee7 --- /dev/null +++ b/scripts/agent-evals/runner.ts @@ -0,0 +1,379 @@ +import { randomUUID } from "node:crypto" +import { RooCodeEventName, TaskCommandName, taskEventSchema, type TaskCommand, type TaskEvent } from "@roo-code/types" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" +import { getFixture } from "./registry.ts" +import { createTaskStartWaiter, type TaskStartWaiter } from "./task-start.ts" +import { + TRACE_KIND, + TRACE_NOTES, + TRACE_SCHEMA_VERSION, + createAccountant, + digestProjection, + eventTaskId, + identityUnavailable, + type Accountant, + type AgentEventFlow, + type AgentExecutionCode, + type AgentStopReason, + type AgentTeardown, + type AgentTrace, + type TraceRunIdentity, +} from "./trace.ts" + +export interface AgentHost { + readonly exited: boolean + onExit(listener: () => void): void + signal(signal: NodeJS.Signals): void +} + +export interface AgentClient { + readonly isReady: boolean + onTaskEvent(listener: (event: unknown) => void): void + onDisconnect(listener: () => void): void + sendCommand(command: TaskCommand): void + disconnect(): void +} + +// The adapter owns processes, sockets, files and credentials. The controller +// only sees these operations, so tests can drive it with synthetic IPC and a +// deterministic clock. +export interface AgentRuntime { + launch(workspaceFiles: readonly AgentFixtureWorkspaceFile[]): Promise + connect(): AgentClient + cleanup(): Promise + writeArtifact(fileName: string, value: unknown): Promise + onInterrupt(listener: () => void): () => void + now(): number + pause(ms: number): Promise + readIdentity?(): Promise +} + +export interface AgentRunObservers { + onAcceptedEvent?(event: TaskEvent, taskId: string): void + shouldStop?(): boolean +} + +export interface AgentRunOptions { + fixtureName: string + credential: string + observers?: AgentRunObservers +} + +export interface AgentRunResult { + execution: { + code: AgentExecutionCode + stopReason: AgentStopReason + activated: boolean + taskAccepted: boolean + sessionLost: boolean + } + teardown: AgentTeardown + trace: AgentTrace +} + +const EVENT_BUFFER_LIMIT = 256 + +type BufferedEvent = { kind: "parsed"; event: TaskEvent } + +const traceSettings = (fixtureProjection: Record, credential: string): AgentTrace["settings"] => { + try { + const projectionJson = JSON.stringify(fixtureProjection) + if (projectionJson === undefined) return { available: false, reason: "projectionNotSerializable" } + if (credential.length > 0 && projectionJson.includes(credential)) + return { available: false, reason: "credentialInProjection" } + return { available: true, projection: fixtureProjection, digest: digestProjection(fixtureProjection) } + } catch { + return { available: false, reason: "projectionNotSerializable" } + } +} + +export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOptions): Promise { + const fixture = getFixture(options.fixtureName) + const started = runtime.now() + let code: AgentExecutionCode = "SETUP_FAILED" + let stopReason: AgentStopReason = "setupError" + let activated = false + let accepted = false + let sessionLost = false + // Frozen at the end of the task run phase, before teardown can terminate + // the host. Teardown exits are reported in the teardown section only. + let runSessionLost = false + // Set when the session dies inside the post-completion usage drain. + let lostDuringDrain = false + let acceptedTaskId: string | undefined + let completedSeen = false + // The reason must name what actually stopped evaluation. "notEvaluated" + // means the projection was never inspected; it is replaced below. + let settings: AgentTrace["settings"] = { available: false, reason: "notEvaluated" } + let host: AgentHost | undefined + let client: AgentClient | undefined + let waiter: TaskStartWaiter | undefined + let teardown: AgentTeardown = { + status: "ok", + hostExited: true, + closeTaskSent: false, + escalatedSignals: [], + cleanup: "notNeeded", + failures: [], + } + const timings: AgentTrace["execution"]["timings"] = { totalMs: 0 } + const flow: AgentEventFlow = { + observed: 0, + malformed: 0, + foreignTaskEvents: 0, + bufferedBeforeAck: 0, + replayedAfterAck: 0, + eventBufferLimit: EVENT_BUFFER_LIMIT, + bufferOverflow: false, + droppedByOverflow: 0, + } + const buffer: BufferedEvent[] = [] + let buffering = false + const accountant: Accountant = createAccountant(() => runtime.now() - started) + + async function until(check: () => boolean, timeoutMs: number, stopOnLoss = true): Promise { + const deadline = runtime.now() + timeoutMs + while (!check() && (!stopOnLoss || !sessionLost) && runtime.now() < deadline) await runtime.pause(100) + } + + const markLost = () => { + sessionLost = true + waiter?.onExit() + } + + const unsubscribe = runtime.onInterrupt(() => { + markLost() + try { + host?.signal("SIGTERM") + } catch { + /* Teardown retries independently. */ + } + }) + + const dispatch = (event: TaskEvent): void => { + const taskId = eventTaskId(event) + if (acceptedTaskId === undefined || taskId !== acceptedTaskId) { + flow.foreignTaskEvents++ + return + } + accountant.observe(event) + if (event.eventName === RooCodeEventName.TaskCompleted) completedSeen = true + options.observers?.onAcceptedEvent?.(event, acceptedTaskId) + } + + const pushBuffer = (item: BufferedEvent): void => { + if (buffer.length >= EVENT_BUFFER_LIMIT) { + flow.bufferOverflow = true + flow.droppedByOverflow++ + return + } + buffer.push(item) + flow.bufferedBeforeAck++ + } + + const handleEvent = (raw: unknown): void => { + flow.observed++ + if ( + typeof raw === "object" && + raw !== null && + (raw as { eventName?: unknown }).eventName === RooCodeEventName.TaskStartResponse + ) { + waiter?.onTaskEvent(raw) + return + } + const parsed = taskEventSchema.safeParse(raw) + if (!parsed.success) { + flow.malformed++ + return + } + if (buffering) { + pushBuffer({ kind: "parsed", event: parsed.data }) + return + } + dispatch(parsed.data) + } + + const replay = (): void => { + buffering = false + for (const item of buffer.splice(0)) { + flow.replayedAfterAck++ + dispatch(item.event) + } + } + + try { + const credential = options.credential + if (!credential) { + stopReason = "credentialMissing" + settings = { available: false, reason: "credentialMissing" } + } else { + settings = traceSettings(fixture.settingsProjection, credential) + if (!settings.available) { + stopReason = "fixtureRejected" + } else { + host = await runtime.launch(fixture.workspace.files) + host.onExit(markLost) + client = runtime.connect() + client.onDisconnect(() => { + sessionLost = true + waiter?.onDisconnect() + }) + client.onTaskEvent(handleEvent) + buffering = true + const activationStart = runtime.now() + await until(() => client?.isReady === true, fixture.limits.activationTimeoutMs) + activated = client.isReady + timings.activationMs = runtime.now() - activationStart + if (!activated) { + code = "ACTIVATION_FAILED" + stopReason = "activationFailed" + } else { + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: fixture.limits.taskStartTimeoutMs }) + // A synchronous send failure must not leave a rejected timer promise. + void waiter.result.catch(() => {}) + const sentAt = runtime.now() + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { + requestId, + text: fixture.prompt, + newTab: false, + configuration: fixture.buildConfiguration(credential), + }, + }) + try { + acceptedTaskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + stopReason = "startNotAccepted" + } + if (accepted) { + timings.taskStartMs = runtime.now() - sentAt + replay() + const runStart = runtime.now() + await until( + () => sessionLost || completedSeen || options.observers?.shouldStop?.() === true, + fixture.limits.taskRunTimeoutMs, + ) + runSessionLost = sessionLost + timings.taskRunMs = runtime.now() - runStart + if (completedSeen && !runSessionLost) { + // Fixed bound; ends early only on session loss. + const drainStart = runtime.now() + await until(() => false, fixture.limits.usageDrainMs) + timings.usageDrainMs = runtime.now() - drainStart + // A loss inside the drain window can truncate late + // usage events. The run code stays COMPLETED, but the + // accounting must carry the caveat. + lostDuringDrain = sessionLost + } + const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true + if (runSessionLost) { + code = "SESSION_LOST" + stopReason = "sessionLost" + } else if (completedSeen) { + code = "COMPLETED" + stopReason = "taskCompleted" + } else if (stopped) { + code = "STOPPED" + stopReason = "adapterStop" + } else { + code = "TASK_RUN_TIMEOUT" + stopReason = "runTimeout" + } + } + } + } + } + } catch { + // Never expose thrown provider, filesystem or host messages. + } finally { + waiter?.onExit() + const teardownStart = runtime.now() + const attempt = async (step: string, operation: () => unknown) => { + try { + await operation() + } catch { + teardown.failures.push(step) + } + } + if (client?.isReady) { + teardown.closeTaskSent = true + await attempt("closeTask", () => client?.sendCommand({ commandName: TaskCommandName.CloseTask })) + } + if (host && !host.exited) { + await attempt("hostTermination", () => + until(() => host?.exited === true, fixture.limits.teardownStepMs, false), + ) + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + if (host.exited) break + await attempt("hostTermination", () => host?.signal(signal)) + teardown.escalatedSignals.push(signal) + await attempt("hostTermination", () => + until(() => host?.exited === true, fixture.limits.teardownStepMs, false), + ) + } + } + teardown.hostExited = host ? host.exited : true + await attempt("disconnect", () => client?.disconnect()) + if (!host || host.exited) { + teardown.cleanup = "done" + await attempt("cleanup", () => runtime.cleanup()) + } else { + teardown.cleanup = "skippedLiveHost" + } + await attempt("unsubscribe", () => unsubscribe()) + if (teardown.failures.length > 0 || (host !== undefined && !host.exited)) teardown.status = "failed" + timings.teardownMs = runtime.now() - teardownStart + } + + const finishedAt = runtime.now() + timings.totalMs = finishedAt - started + let identity: TraceRunIdentity + try { + identity = (await runtime.readIdentity?.()) ?? identityUnavailable() + } catch { + identity = identityUnavailable() + } + const accounting = accountant.snapshot({ + accepted, + sessionLost: runSessionLost, + sessionLostDuringDrain: lostDuringDrain, + bufferOverflow: flow.bufferOverflow, + malformedEvents: flow.malformed, + foreignTaskEvents: flow.foreignTaskEvents, + }) + const trace: AgentTrace = { + traceSchemaVersion: TRACE_SCHEMA_VERSION, + kind: TRACE_KIND, + run: { + runId: randomUUID(), + fixture: { name: fixture.identity.name, version: fixture.identity.version }, + startedAt: new Date(started).toISOString(), + finishedAt: new Date(finishedAt).toISOString(), + durationMs: timings.totalMs, + identity, + }, + settings, + execution: { + code, + stopReason, + activated, + taskAccepted: accepted, + sessionLost: runSessionLost, + timings, + }, + eventFlow: flow, + accounting, + teardown, + notes: TRACE_NOTES, + } + await runtime.writeArtifact("trace.json", trace) + return { + execution: { code, stopReason, activated, taskAccepted: accepted, sessionLost: runSessionLost }, + teardown, + trace, + } +} diff --git a/scripts/agent-evals/task-start.ts b/scripts/agent-evals/task-start.ts new file mode 100644 index 0000000000..2aafaaea57 --- /dev/null +++ b/scripts/agent-evals/task-start.ts @@ -0,0 +1,96 @@ +import { RooCodeEventName, taskStartResponseSchema } from "@roo-code/types" + +export type TaskStartWaiter = { + onTaskEvent: (event: unknown) => string | undefined + onDisconnect: () => void + onExit: () => void + result: Promise +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null + +// Correlates one StartNewTask request with its sanitized response, similar to +// JSON-RPC: the sender generates the requestId and the reply copies it back. +// The waiter settles exactly once. A matched success resolves with the +// accepted task ID. A matched failure, a timeout, an IPC disconnect, or a +// VS Code exit rejects. Unmatched or malformed responses never settle it. +export function createTaskStartWaiter({ + requestId, + timeoutMs, +}: { + requestId: string + timeoutMs: number +}): TaskStartWaiter { + let resolveResult!: (taskId: string) => void + let rejectResult!: (error: Error) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + + let settled = false + let timer: NodeJS.Timeout | undefined + + const settle = (finish: () => void): void => { + if (settled) { + return + } + settled = true + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + finish() + } + + const rejectWith = (message: string): void => { + settle(() => rejectResult(new Error(message))) + } + + timer = setTimeout(() => { + rejectWith(`task start response missing after ${timeoutMs}ms (requestId=${requestId})`) + }, timeoutMs) + + return { + onTaskEvent(event: unknown): string | undefined { + if (settled) return undefined + if (!isRecord(event) || event["eventName"] !== RooCodeEventName.TaskStartResponse) { + return undefined + } + + const payload = event["payload"] + if (!Array.isArray(payload) || payload.length === 0) { + return undefined + } + + const parsed = taskStartResponseSchema.safeParse(payload[0]) + if (!parsed.success || parsed.data.requestId !== requestId) { + return undefined + } + + const response = parsed.data + if (response.success) { + const acceptedTaskId = response.taskId + settle(() => resolveResult(acceptedTaskId)) + return acceptedTaskId + } + + const { errorCode, errorMessage, stage } = response + // The stage is a fixed sanitized token from the response schema; it + // names the startup phase that stopped responding. + const stageDetail = stage === undefined ? "" : ` (stage=${stage})` + rejectWith(`${errorCode}: ${errorMessage}${stageDetail}`) + return undefined + }, + + onDisconnect(): void { + rejectWith("IPC disconnected before the task start response arrived") + }, + + onExit(): void { + rejectWith("VS Code exited before the task start response arrived") + }, + + result, + } +} diff --git a/scripts/agent-evals/trace.test.ts b/scripts/agent-evals/trace.test.ts new file mode 100644 index 0000000000..463c789b4c --- /dev/null +++ b/scripts/agent-evals/trace.test.ts @@ -0,0 +1,221 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, type TaskEvent } from "@roo-code/types" +import { + TRACE_NOTES, + createAccountant, + digestProjection, + eventTaskId, + parseTaskEvent, + stableStringify, +} from "./trace.ts" + +let clock = 0 +const fresh = () => createAccountant(() => ++clock) + +const message = (ts: number, text: string | undefined): TaskEvent => ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId: "root", action: "created", message: { ts, type: "say", say: "api_req_started", text } }], +}) + +const usageEvent = (tokensIn: number, tokensOut: number, cacheWrites?: number): TaskEvent => ({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { + totalTokensIn: tokensIn, + totalTokensOut: tokensOut, + totalCost: 0, + contextTokens: 11, + ...(cacheWrites === undefined ? {} : { totalCacheWrites: cacheWrites }), + }, + {}, + ], +}) + +describe("trace request accounting", () => { + it("upserts one request row per message timestamp across created and updated events", () => { + const accountant = fresh() + accountant.observe(message(1, JSON.stringify({ apiProtocol: "openai" }))) + accountant.observe( + message( + 1, + JSON.stringify({ + apiProtocol: "openai", + tokensIn: 10, + tokensOut: 4, + cacheWrites: 2, + cacheReads: 3, + cost: 0.5, + }), + ), + ) + accountant.observe( + message( + 1, + JSON.stringify({ + apiProtocol: "openai", + tokensIn: 10, + tokensOut: 4, + cancelReason: "streaming_failed", + streamingFailedMessage: "private provider error", + }), + ), + ) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.requestCount, 1) + const row = accounting.requests[0] + assert.ok(row) + assert.equal(row.messageCount, 3) + assert.deepEqual(row.tokensIn, { available: true, value: 10 }) + assert.deepEqual(row.tokensOut, { available: true, value: 4 }) + assert.deepEqual(row.cacheWrites, { available: true, value: 2 }) + assert.deepEqual(row.cacheReads, { available: true, value: 3 }) + assert.deepEqual(row.cost, { available: true, value: 0.5 }) + assert.equal(row.cancelReason, "streaming_failed") + assert.equal(row.streamingFailed, true) + assert.ok(!JSON.stringify(accounting).includes("private provider error")) + }) + + it("marks requests without usage updates instead of filling zeros", () => { + const accountant = fresh() + accountant.observe(message(1, JSON.stringify({ tokensIn: 10, tokensOut: 2 }))) + accountant.observe(message(2, JSON.stringify({ apiProtocol: "openai" }))) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.requestsMissingUsage, 1) + assert.equal(accounting.completeness.status, "partial") + assert.ok(accounting.completeness.reasons.includes("finalRequestMissingUsage")) + const row = accounting.requests[1] + assert.ok(row) + assert.deepEqual(row.tokensIn, { available: false, reason: "notReported" }) + assert.deepEqual(row.tokensOut, { available: false, reason: "notReported" }) + }) + + it("keeps only the latest cumulative snapshot and never sums snapshots", () => { + const accountant = fresh() + accountant.observe(usageEvent(10, 5)) + accountant.observe(usageEvent(25, 9)) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.usage.snapshotsObserved, 2) + assert.equal(accounting.usage.lastSnapshot?.tokensIn, 25) + assert.equal(accounting.usage.lastSnapshot?.tokensOut, 9) + assert.equal(accounting.usage.available, true) + assert.ok(!("tokensInSum" in (accounting.usage as unknown as Record))) + }) + + it("reports missing cache fields and zero tokens as caveated, not measured", () => { + const accountant = fresh() + accountant.observe(usageEvent(0, 0)) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.usage.lastSnapshot?.tokensIn, 0) + assert.deepEqual(accounting.usage.lastSnapshot?.cacheWrites, { available: false, reason: "notReported" }) + assert.ok(TRACE_NOTES.some((note) => note.includes("Zero token or cost values"))) + }) + + it("counts tool failure events by name without storing error payloads", () => { + const accountant = fresh() + accountant.observe({ + eventName: RooCodeEventName.TaskToolFailed, + payload: ["root", "execute_command", "private failure detail that must never persist"], + }) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.toolFailureEvents, 1) + assert.deepEqual(accounting.toolFailuresByTool, { execute_command: 1 }) + assert.ok(!JSON.stringify(accounting).includes("private failure detail")) + }) + + it("counts retry marker messages separately from usage rows", () => { + const accountant = fresh() + accountant.observe({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 9, type: "say", say: "api_req_retried", text: "[retry marker]" }, + }, + ], + }) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.apiReqRetriedMarkers, 1) + assert.equal(accounting.requestCount, 0) + assert.ok(TRACE_NOTES.some((note) => note.includes("not an HTTP retry count"))) + }) + + it("reports no-usage completeness with a fixed reason", () => { + const accounting = fresh().snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.usage.available, false) + assert.equal(accounting.usage.unavailableReason, "noUsageEventsObserved") + assert.ok(accounting.completeness.reasons.includes("noUsageEventsObserved")) + }) +}) + +describe("trace helpers", () => { + it("documents the settings projection as the only configuration content", () => { + assert.ok(TRACE_NOTES.some((note) => note.includes("nonsecret settings projection"))) + }) + + it("digests projections stably regardless of key order", () => { + assert.equal(digestProjection({ a: 1, b: { c: 2 } }), digestProjection({ b: { c: 2 }, a: 1 })) + assert.notEqual(digestProjection({ a: 1 }), digestProjection({ a: 2 })) + assert.equal( + stableStringify({ b: 1, a: [2, { z: 3, y: 4 }] }), + stableStringify({ a: [2, { y: 4, z: 3 }], b: 1 }), + ) + }) + + it("parses events defensively and reads task identities", () => { + assert.equal(eventTaskId(parseTaskEvent(usageEvent(1, 1)) as TaskEvent), "root") + assert.equal( + eventTaskId(parseTaskEvent({ eventName: RooCodeEventName.TaskAborted, payload: ["root"] }) as TaskEvent), + "root", + ) + assert.equal( + eventTaskId(parseTaskEvent({ eventName: RooCodeEventName.ModelsResponse, payload: [{}] }) as TaskEvent), + undefined, + ) + assert.equal(parseTaskEvent({ eventName: "bogus" }), undefined) + }) +}) diff --git a/scripts/agent-evals/trace.ts b/scripts/agent-evals/trace.ts new file mode 100644 index 0000000000..fd5d6990c7 --- /dev/null +++ b/scripts/agent-evals/trace.ts @@ -0,0 +1,402 @@ +import { createHash } from "node:crypto" +import { RooCodeEventName, taskEventSchema, type TaskEvent } from "@roo-code/types" + +export const TRACE_SCHEMA_VERSION = 1 +export const TRACE_KIND = "agent-evals-trace" + +export type Measured = { available: true; value: T } | { available: false; reason: string } +export type MeasuredString = Measured +export type Accountant = ReturnType + +export const measured = (value: T | undefined | null, reason: string): Measured => + value === undefined || value === null ? { available: false, reason } : { available: true, value } + +export interface TraceRunIdentity { + commit: MeasuredString + extensionVersion: MeasuredString + vscodeTarget: MeasuredString + nodeVersion: string + platform: string + arch: string +} + +export const identityUnavailable = (reason = "identityUnavailable"): TraceRunIdentity => ({ + commit: { available: false, reason }, + extensionVersion: { available: false, reason }, + vscodeTarget: { available: false, reason }, + nodeVersion: process.version, + platform: process.platform, + arch: process.arch, +}) + +export type AgentExecutionCode = + | "COMPLETED" + | "STOPPED" + | "SETUP_FAILED" + | "ACTIVATION_FAILED" + | "TASK_START_FAILED" + | "SESSION_LOST" + | "TASK_RUN_TIMEOUT" + +export type AgentStopReason = + | "taskCompleted" + | "adapterStop" + | "runTimeout" + | "sessionLost" + | "startNotAccepted" + | "activationFailed" + | "credentialMissing" + | "fixtureRejected" + | "setupError" + +export interface AgentEventFlow { + observed: number + malformed: number + foreignTaskEvents: number + bufferedBeforeAck: number + replayedAfterAck: number + eventBufferLimit: number + bufferOverflow: boolean + droppedByOverflow: number +} + +export interface TraceRequestRow { + index: number + ts: number + apiProtocol: MeasuredString + tokensIn: Measured + tokensOut: Measured + cacheWrites: Measured + cacheReads: Measured + cost: Measured + cancelReason?: string + streamingFailed: boolean + firstObservedMs: number + lastUpdatedMs: number + messageCount: number +} + +export interface CumulativeUsageSnapshot { + source: "taskTokenUsageUpdated" | "taskCompleted" + observedAtMs: number + tokensIn: number + tokensOut: number + cacheWrites: Measured + cacheReads: Measured + cost: number + contextTokens: number +} + +export interface TraceUsage { + available: boolean + unavailableReason?: string + snapshotsObserved: number + lastSnapshot?: CumulativeUsageSnapshot +} + +export interface TraceToolSnapshot { + snapshot?: Record + observedAtMs?: number +} + +export interface TraceAccounting { + requests: TraceRequestRow[] + requestCount: number + requestsMissingUsage: number + apiReqRetriedMarkers: number + usage: TraceUsage + toolSnapshot: TraceToolSnapshot + toolFailureEvents: number + toolFailuresByTool: Record + completeness: { status: "complete" | "partial"; reasons: string[] } +} + +export interface AgentTeardown { + status: "ok" | "failed" + hostExited: boolean + closeTaskSent: boolean + escalatedSignals: string[] + cleanup: "done" | "skippedLiveHost" | "failed" | "notNeeded" + failures: string[] +} + +export interface AgentTrace { + traceSchemaVersion: typeof TRACE_SCHEMA_VERSION + kind: typeof TRACE_KIND + run: { + runId: string + fixture: { name: string; version: string } + startedAt: string + finishedAt: string + durationMs: number + identity: TraceRunIdentity + } + settings: { + available: boolean + reason?: string + projection?: Record + digest?: string + } + execution: { + code: AgentExecutionCode + stopReason: AgentStopReason + activated: boolean + taskAccepted: boolean + sessionLost: boolean + timings: { + activationMs?: number + taskStartMs?: number + taskRunMs?: number + usageDrainMs?: number + teardownMs?: number + totalMs: number + } + } + eventFlow: AgentEventFlow + accounting: TraceAccounting + teardown: AgentTeardown + notes: string[] +} + +// Fixed vocabulary. Reviewers can rely on these sentences across trace versions. +export const TRACE_NOTES = [ + "Usage numbers come from extension events. The trace never sums cumulative snapshots.", + "Zero token or cost values can be defaults from the adapter or task layer. They do not prove the provider measured zero.", + "contextTokens is an extension context estimate, not a provider measurement.", + "Request rows use per-request fields from api_req_started message updates, keyed by message timestamp.", + "apiReqRetriedMarkers counts retry marker messages. It is not an HTTP retry count.", + "The trace is metadata only. Prompts, model output, reasoning, tool arguments, error payloads, and storage paths are excluded. Provider settings appear only as the nonsecret settings projection.", + "Missing measurements carry a reason. They are never filled with zero.", +] + +const sortValue = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sortValue) + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sortValue(entry)]), + ) + } + return value +} + +export const stableStringify = (value: unknown): string => JSON.stringify(sortValue(value)) + +export const digestProjection = (projection: Record): string => + createHash("sha256").update(stableStringify(projection)).digest("hex") + +interface RequestRowState { + index: number + ts: number + apiProtocol?: string + tokensIn?: number + tokensOut?: number + cacheWrites?: number + cacheReads?: number + cost?: number + cancelReason?: string + streamingFailed: boolean + firstObservedMs: number + lastUpdatedMs: number + messageCount: number + hasUsage: boolean +} + +const MAX_REQUEST_TEXT_CHARS = 4_096 +const MAX_CANCEL_REASON_CHARS = 32 +const MAX_PROTOCOL_CHARS = 16 + +export interface AccountingInputs { + accepted: boolean + sessionLost: boolean + // True when the session died inside the post-completion usage drain. Late + // usage may be truncated, so the accounting carries its own caveat. + sessionLostDuringDrain?: boolean + bufferOverflow: boolean + malformedEvents: number + foreignTaskEvents: number +} + +export function createAccountant(nowMs: () => number): { + observe(event: TaskEvent): void + snapshot(inputs: AccountingInputs): TraceAccounting +} { + const rows = new Map() + let snapshotsObserved = 0 + let lastUsageSnapshot: CumulativeUsageSnapshot | undefined + let lastToolSnapshot: TraceToolSnapshot = {} + let apiReqRetriedMarkers = 0 + let toolFailureEvents = 0 + const toolFailuresByTool: Record = {} + + const usageSnapshot = ( + source: CumulativeUsageSnapshot["source"], + usage: { + totalTokensIn: number + totalTokensOut: number + totalCacheWrites?: number + totalCacheReads?: number + totalCost: number + contextTokens: number + }, + ): CumulativeUsageSnapshot => ({ + source, + observedAtMs: nowMs(), + tokensIn: usage.totalTokensIn, + tokensOut: usage.totalTokensOut, + cacheWrites: measured(usage.totalCacheWrites, "notReported"), + cacheReads: measured(usage.totalCacheReads, "notReported"), + cost: usage.totalCost, + contextTokens: usage.contextTokens, + }) + + const observe = (event: TaskEvent): void => { + if (event.eventName === RooCodeEventName.Message) { + const entry = event.payload[0] + const message = entry.message + if (message.type !== "say") return + if (message.say === "api_req_retried") { + apiReqRetriedMarkers++ + return + } + if (message.say !== "api_req_started") return + const ts = message.ts + const at = nowMs() + const row: RequestRowState = rows.get(ts) ?? { + index: rows.size, + ts, + streamingFailed: false, + firstObservedMs: at, + lastUpdatedMs: at, + messageCount: 0, + hasUsage: false, + } + row.messageCount++ + row.lastUpdatedMs = at + if (typeof message.text === "string" && message.text.length <= MAX_REQUEST_TEXT_CHARS) { + try { + const info = JSON.parse(message.text) as { + apiProtocol?: string + tokensIn?: number + tokensOut?: number + cacheWrites?: number + cacheReads?: number + cost?: number + cancelReason?: string + streamingFailedMessage?: string + } + if (typeof info.apiProtocol === "string" && info.apiProtocol.length <= MAX_PROTOCOL_CHARS) + row.apiProtocol = info.apiProtocol + if (typeof info.tokensIn === "number") { + row.tokensIn = info.tokensIn + row.hasUsage = true + } + if (typeof info.tokensOut === "number") { + row.tokensOut = info.tokensOut + row.hasUsage = true + } + if (typeof info.cacheWrites === "number") { + row.cacheWrites = info.cacheWrites + row.hasUsage = true + } + if (typeof info.cacheReads === "number") { + row.cacheReads = info.cacheReads + row.hasUsage = true + } + if (typeof info.cost === "number") row.cost = info.cost + if (typeof info.cancelReason === "string" && info.cancelReason.length <= MAX_CANCEL_REASON_CHARS) + row.cancelReason = info.cancelReason + // Presence only. The failure text itself is never stored. + if (typeof info.streamingFailedMessage === "string") row.streamingFailed = true + } catch { + // Unreadable request text stays an availability gap, not an error payload. + } + } + rows.set(ts, row) + return + } + if (event.eventName === RooCodeEventName.TaskTokenUsageUpdated) { + snapshotsObserved++ + lastUsageSnapshot = usageSnapshot("taskTokenUsageUpdated", event.payload[1]) + lastToolSnapshot = { snapshot: event.payload[2], observedAtMs: nowMs() } + return + } + if (event.eventName === RooCodeEventName.TaskCompleted) { + snapshotsObserved++ + lastUsageSnapshot = usageSnapshot("taskCompleted", event.payload[1]) + lastToolSnapshot = { snapshot: event.payload[2], observedAtMs: nowMs() } + return + } + if (event.eventName === RooCodeEventName.TaskToolFailed) { + const tool = event.payload[1] + toolFailureEvents++ + toolFailuresByTool[tool] = (toolFailuresByTool[tool] ?? 0) + 1 + } + } + + const snapshot = (inputs: AccountingInputs): TraceAccounting => { + const requests: TraceRequestRow[] = [...rows.values()] + .sort((a, b) => a.index - b.index) + .map((row) => ({ + index: row.index, + ts: row.ts, + apiProtocol: measured(row.apiProtocol, "notReported"), + tokensIn: measured(row.tokensIn, "notReported"), + tokensOut: measured(row.tokensOut, "notReported"), + cacheWrites: measured(row.cacheWrites, "notReported"), + cacheReads: measured(row.cacheReads, "notReported"), + cost: measured(row.cost, "notReported"), + cancelReason: row.cancelReason, + streamingFailed: row.streamingFailed, + firstObservedMs: row.firstObservedMs, + lastUpdatedMs: row.lastUpdatedMs, + messageCount: row.messageCount, + })) + const requestsMissingUsage = [...rows.values()].filter((row) => !row.hasUsage).length + const reasons: string[] = [] + if (!inputs.accepted) reasons.push("taskNotAccepted") + if (snapshotsObserved === 0) reasons.push("noUsageEventsObserved") + if (requestsMissingUsage > 0) reasons.push("finalRequestMissingUsage") + if (inputs.bufferOverflow) reasons.push("eventBufferOverflow") + if (inputs.malformedEvents > 0) reasons.push("malformedEventsObserved") + if (inputs.foreignTaskEvents > 0) reasons.push("foreignTaskEventsDropped") + if (inputs.sessionLost) reasons.push("sessionLost") + if (inputs.sessionLostDuringDrain) reasons.push("sessionLostDuringUsageDrain") + return { + requests, + requestCount: requests.length, + requestsMissingUsage, + apiReqRetriedMarkers, + usage: { + available: snapshotsObserved > 0, + unavailableReason: snapshotsObserved > 0 ? undefined : "noUsageEventsObserved", + snapshotsObserved, + lastSnapshot: lastUsageSnapshot, + }, + toolSnapshot: lastToolSnapshot, + toolFailureEvents, + toolFailuresByTool, + completeness: { status: reasons.length === 0 ? "complete" : "partial", reasons }, + } + } + + return { observe, snapshot } +} + +// Defense in depth for replay: schema-valid events only, and only their task +// identity is read. No payload content passes through this helper. +export const parseTaskEvent = (raw: unknown): TaskEvent | undefined => { + const parsed = taskEventSchema.safeParse(raw) + return parsed.success ? parsed.data : undefined +} + +export const eventTaskId = (event: TaskEvent): string | undefined => { + const first: unknown = (event.payload as readonly unknown[])[0] + if (typeof first === "string") return first + if (typeof first === "object" && first !== null && "taskId" in first) { + const taskId = (first as { taskId?: unknown }).taskId + if (typeof taskId === "string") return taskId + } + return undefined +} diff --git a/scripts/agent-evals/tsconfig.json b/scripts/agent-evals/tsconfig.json new file mode 100644 index 0000000000..55bddebe10 --- /dev/null +++ b/scripts/agent-evals/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "lib": ["ES2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./*.ts", "./*.mts"] +} diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 0bab69c438..3a2ed0689f 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -11,6 +11,12 @@ const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.ur const driver = fs.readFileSync(new URL("./solheim-smoke/driver.mts", import.meta.url), "utf8") const recorder = fs.readFileSync(new URL("./solheim-smoke/record.sh", import.meta.url), "utf8") const smokePkg = JSON.parse(fs.readFileSync(new URL("./solheim-smoke/package.json", import.meta.url), "utf8")) +const runner = fs.readFileSync(new URL("./agent-evals/runner.ts", import.meta.url), "utf8") +const launcher = fs.readFileSync(new URL("./agent-evals/launcher.ts", import.meta.url), "utf8") +const registry = fs.readFileSync(new URL("./agent-evals/registry.ts", import.meta.url), "utf8") +const fixture = fs.readFileSync(new URL("./solheim-smoke/fixture.ts", import.meta.url), "utf8") +const trace = fs.readFileSync(new URL("./agent-evals/trace.ts", import.meta.url), "utf8") +const evalsPkg = JSON.parse(fs.readFileSync(new URL("./agent-evals/package.json", import.meta.url), "utf8")) describe("Project A provider smoke workflow", () => { it("runs smoke TypeScript as ESM without requiring compiled workspace packages", () => { @@ -58,7 +64,7 @@ describe("Project A provider smoke workflow", () => { assert.match(workflow, /timeout-minutes: 8/) assert.match(workflow, /^ {8}timeout-minutes: 25$/m) }) - it("uploads only the verdict and video, never host logs or storage", () => { + it("uploads only the verdict, the structured trace, and video, never host logs or storage", () => { const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] assert.deepEqual( paths @@ -67,6 +73,7 @@ describe("Project A provider smoke workflow", () => { .map((line) => line.trim()), [ "${{ runner.temp }}/solheim-provider-smoke/verdict.json", + "${{ runner.temp }}/solheim-provider-smoke/trace.json", "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", ], ) @@ -87,14 +94,46 @@ describe("Project A provider smoke workflow", () => { assert.match(workflow, /retention-days: 7/) }) it("keeps Code QA and package commands focused on smoke contracts", () => { - for (const command of ["test:solheim-smoke:unit", "test:solheim-smoke-ci", "solheim-smoke:check-types"]) + for (const command of [ + "test:solheim-smoke:unit", + "test:solheim-smoke-ci", + "solheim-smoke:check-types", + "test:agent-evals:unit", + "agent-evals:check-types", + ]) assert.ok(codeQa.includes(command)) assert.ok(!codeQa.includes("model-qualification")) assert.ok(!Object.keys(pkg.scripts).some((name) => name.includes("model-qualification"))) assert.ok(!codeQa.includes("final-smoke")) assert.equal(pkg.scripts["solheim:smoke"], "node --import tsx scripts/solheim-smoke/driver.mts") + assert.equal(pkg.scripts["test:agent-evals:unit"], "node --import tsx --test scripts/agent-evals/*.test.ts") + assert.equal(pkg.scripts["agent-evals:check-types"], "tsc -p scripts/agent-evals/tsconfig.json") assert.ok(!fs.existsSync(new URL("../.github/workflows/final-vscode-review-smoke.yml", import.meta.url))) }) + it("keeps the shared controller free of smoke, provider, and solution knowledge", () => { + assert.equal(evalsPkg.type, "module") + assert.equal(evalsPkg.private, true) + assert.ok(!/solheim/i.test(runner)) + assert.ok(!/smoke/i.test(runner)) + assert.ok(!runner.includes("SOLHEIM_API_KEY")) + assert.ok(!/solheim/i.test(trace)) + assert.ok(trace.includes("traceSchemaVersion")) + }) + it("registers fixtures only through the validated in-memory registry", () => { + assert.ok(!registry.includes("import(")) + assert.ok(!registry.includes("fetch(")) + assert.ok(!registry.includes("readdir")) + assert.ok(fixture.includes("registerFixture")) + }) + it("keeps VS Code launch isolation in one shared launcher with an atomic artifact writer", () => { + assert.ok(driver.includes("createHostRuntime")) + assert.ok(!driver.includes("spawn(")) + assert.ok(!driver.includes("downloadAndUnzipVSCode")) + assert.ok(launcher.includes("downloadAndUnzipVSCode")) + assert.ok(launcher.includes("buildChildEnvironment")) + assert.ok(launcher.includes("await fs.rename(pending")) + assert.ok(launcher.includes("process.kill(-child.pid")) + }) }) // Runs record.sh with stub ffmpeg and ffprobe, so no display or codec is needed. diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index 219ce3341e..b7421fa078 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -4,6 +4,13 @@ Infrastructure validation only: a real VS Code extension activates, accepts a co receives output from Solheim's OpenAI-compatible provider, and produces a completion result. It does not read a PR, assess an answer's meaning, select checks, find defects, or post to GitHub. +The smoke is the first registered fixture of the shared substrate in +[`scripts/agent-evals`](../agent-evals/README.md). That substrate owns the fixture contract, the +trusted registry, the single VS Code launcher, startup correlation, bounded teardown, and the +metadata-only structured trace. [`run.ts`](./run.ts) is a thin adapter: it maps the shared +controller result onto the smoke evidence gates and verdict. The generic controller knows nothing +about Solheim, the prompt, or the expected completion. + The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to `refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the single credential-bearing step. There is one job, no PR input or approval gate, no review/posting job, @@ -31,13 +38,16 @@ Teardown escalates through close, terminate, and kill within independent five-se completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. -The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names) -and `smoke.mp4`, a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names), +`trace.json` (versioned metadata-only structured trace from the shared substrate; see +[`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`, +a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video -contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict, -it is not text-redacted. No settings screen is opened, and no host logs, storage, task configuration or -credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt interruption. -Ordinary success and failure runs upload both artifacts; cancellation may prevent finalization/upload. +contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict +and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task +configuration or credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt +interruption. Ordinary success and failure runs upload all three artifacts; cancellation may prevent +finalization/upload. Checks, without a provider key or network calls: @@ -58,3 +68,8 @@ established the underlying integration, not this refactor's live status. Review after the frozen Stage 1B pilot produced zero incremental verified defects. Its research harness and receipts are archived outside this smoke-only change; the chronology, results, caveats and preservation details are recorded in [tracking issue #1896](https://github.com/Zoo-Code-Org/Zoo-Code/issues/1896). + +Planned follow-ups on top of the shared substrate, in order: five frozen real coding tasks, independent +credential-free graders, paired exact trusted commit comparisons, repeated runs with uncertainty +reporting, then advisory release integration aligned to the actual release workflows. They are tracked +in [`scripts/agent-evals/README.md`](../agent-evals/README.md). diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts index 7b19187cdc..b394d12cf6 100644 --- a/scripts/solheim-smoke/config.ts +++ b/scripts/solheim-smoke/config.ts @@ -1,6 +1,7 @@ -import path from "node:path" import type { RooCodeSettings } from "@roo-code/types" +export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation.ts" + export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" export const SOLHEIM_MODEL = "qwen3.8-27b" export const SMOKE_TIMEOUT_MS = 120_000 @@ -47,38 +48,3 @@ export function buildSmokeConfiguration(apiKey: string): RooCodeSettings { alwaysAllowFollowupQuestions: false, } } - -// Inherit only host-launch necessities, never the credential-bearing parent environment. -export function buildChildEnvironment( - env: NodeJS.ProcessEnv, - isolatedHome: string, - socketPath: string, -): NodeJS.ProcessEnv { - const child: NodeJS.ProcessEnv = {} - for (const name of ["PATH", "DISPLAY", "XAUTHORITY", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT", "WINDIR"]) - if (env[name]) child[name] = env[name] - return { - ...child, - HOME: isolatedHome, - USERPROFILE: isolatedHome, - XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), - XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), - ROO_CODE_IPC_SOCKET_PATH: socketPath, - } -} - -export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { - return [ - workspace, - `--user-data-dir=${userData}`, - `--extensions-dir=${extensions}`, - `--extensionDevelopmentPath=${extension}`, - "--no-sandbox", - "--disable-gpu-sandbox", - "--password-store=basic", - "--disable-updates", - "--skip-welcome", - "--skip-release-notes", - "--disable-workspace-trust", - ] -} diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts index 383e2db206..df10cd5566 100644 --- a/scripts/solheim-smoke/driver.mts +++ b/scripts/solheim-smoke/driver.mts @@ -1,116 +1,14 @@ import fs from "node:fs/promises" -import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" -import { spawn } from "node:child_process" -import { downloadAndUnzipVSCode } from "@vscode/test-electron" -import { IpcClient } from "@roo-code/ipc" -import { IpcMessageType } from "@roo-code/types" -import { buildChildEnvironment, buildLaunchArgs } from "./config.ts" -import { runSmoke, smokeExitCode, type SmokeRuntime } from "./run.ts" +import { createHostRuntime } from "../agent-evals/launcher.ts" +import { runSmoke, smokeExitCode } from "./run.ts" const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const out = process.env.SOLHEIM_SMOKE_OUT_DIR if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") await fs.mkdir(out, { recursive: true }) -let temp: string | undefined -let socket = "" -const runtime: SmokeRuntime = { - async launch() { - const extension = path.join(root, "src") - await fs.access(path.join(extension, "package.json")) - temp = await fs.mkdtemp(path.join(os.tmpdir(), "solheim-smoke-")) - const workspace = path.join(temp, "workspace") - const isolatedHome = path.join(temp, "home") - await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder))) - socket = path.join(temp, "ipc.sock") - const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { - devDependencies: Record - } - const executable = await downloadAndUnzipVSCode({ - version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", - cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), - }) - const args = buildLaunchArgs(workspace, path.join(temp, "userdata"), path.join(temp, "extensions"), extension) - const child = spawn( - process.env.DISPLAY ? executable : "xvfb-run", - process.env.DISPLAY ? args : ["-a", executable, ...args], - { - detached: true, - env: buildChildEnvironment(process.env, isolatedHome, socket), - stdio: "ignore", - }, - ) - let exited = false - let onExit = () => {} - const closed = () => { - exited = true - onExit() - } - child.once("error", () => { - // Spawn failure has no live PID; a failed kill must not claim exit. - if (!child.pid) closed() - else onExit() - }) - child.once("close", closed) - return { - get exited() { - return exited - }, - onExit(listener) { - onExit = listener - if (exited) listener() - }, - signal(signal) { - if (!child.pid || exited) return - try { - process.kill(-child.pid, signal) - } catch { - child.kill(signal) - } - }, - } - }, - connect() { - const client = new IpcClient(socket, () => {}) - return { - get isReady() { - return client.isReady - }, - onTaskEvent(listener) { - client.on(IpcMessageType.TaskEvent, listener) - }, - onDisconnect(listener) { - client.on(IpcMessageType.Disconnect, listener) - }, - sendCommand(command) { - client.sendCommand(command) - }, - disconnect() { - client.disconnect() - }, - } - }, - async cleanup() { - if (temp) await fs.rm(temp, { recursive: true, force: true }) - }, - async writeVerdict(verdict) { - // Atomic, whitelisted metadata only; no provider text or host logs. - const pending = path.join(out, `verdict.json.${process.pid}.tmp`) - await fs.writeFile(pending, JSON.stringify(verdict, null, 2)) - await fs.rename(pending, path.join(out, "verdict.json")) - }, - onInterrupt(listener) { - process.once("SIGINT", listener) - process.once("SIGTERM", listener) - return () => { - process.removeListener("SIGINT", listener) - process.removeListener("SIGTERM", listener) - } - }, - now: Date.now, - pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), -} +const runtime = createHostRuntime({ root, artifactsDir: out }) try { const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") console.log(`Solheim provider smoke: ${verdict.code}`) diff --git a/scripts/solheim-smoke/fixture.ts b/scripts/solheim-smoke/fixture.ts new file mode 100644 index 0000000000..37916b54dd --- /dev/null +++ b/scripts/solheim-smoke/fixture.ts @@ -0,0 +1,89 @@ +import type { AgentFixture } from "../agent-evals/contract.ts" +import { registerFixture } from "../agent-evals/registry.ts" +import { SMOKE_TIMEOUT_MS, SMOKE_PROMPT, SOLHEIM_BASE_URL, SOLHEIM_MODEL, buildSmokeConfiguration } from "./config.ts" + +export const SMOKE_FIXTURE_NAME = "solheim-smoke" +const SMOKE_FIXTURE_VERSION = "1.0.0" +const ROLE_DEFINITION = "Complete one connectivity smoke task. You are not a reviewer." +const DISABLED_TOOLS = [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", +] as const + +// The first trusted fixture: the no-tools deterministic provider smoke on an +// empty workspace. The projection is the explicit nonsecret mirror of the +// built configuration; the trace digests it and never sees the credential. +export function buildSmokeFixture(): AgentFixture { + return { + identity: { + name: SMOKE_FIXTURE_NAME, + version: SMOKE_FIXTURE_VERSION, + description: "No-tools deterministic provider connectivity smoke on an empty workspace.", + }, + prompt: SMOKE_PROMPT, + workspace: { files: [] }, + toolPolicy: { + mode: "provider-smoke", + roleDefinition: ROLE_DEFINITION, + groups: [], + disabledTools: [...DISABLED_TOOLS], + allow: { + readOnly: false, + readOnlyOutsideWorkspace: false, + write: false, + writeOutsideWorkspace: false, + writeProtected: false, + execute: false, + mcp: false, + modeSwitch: false, + subtasks: false, + followupQuestions: false, + }, + }, + limits: { + activationTimeoutMs: SMOKE_TIMEOUT_MS, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: SMOKE_TIMEOUT_MS, + usageDrainMs: 5_000, + teardownStepMs: 5_000, + }, + settingsProjection: { + apiProvider: "openai", + openAiBaseUrl: SOLHEIM_BASE_URL, + openAiModelId: SOLHEIM_MODEL, + openAiR1FormatEnabled: true, + openAiExtraBody: { chat_template_kwargs: { enable_thinking: false } }, + modelMaxTokens: 512, + autoCondenseContext: false, + mode: "provider-smoke", + customModes: [ + { slug: "provider-smoke", name: "Provider smoke", roleDefinition: ROLE_DEFINITION, groups: [] }, + ], + disabledTools: [...DISABLED_TOOLS], + autoApprovalEnabled: true, + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + }, + buildConfiguration: buildSmokeConfiguration, + } +} + +let registered = false + +export function ensureSmokeFixtureRegistered(): void { + if (registered) return + registered = true + registerFixture(buildSmokeFixture()) +} diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts index 1c64ec53e4..9be74b3d75 100644 --- a/scripts/solheim-smoke/run.test.ts +++ b/scripts/solheim-smoke/run.test.ts @@ -126,8 +126,8 @@ function fixture( async cleanup() { cleaned = true }, - async writeVerdict(verdict) { - written.push(verdict) + async writeArtifact(fileName, value) { + if (fileName === "verdict.json") written.push(value as SmokeVerdict) }, onInterrupt(listener) { interrupt = listener diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts index 698216598b..8cbf1e9fb5 100644 --- a/scripts/solheim-smoke/run.ts +++ b/scripts/solheim-smoke/run.ts @@ -1,21 +1,11 @@ -import { randomUUID } from "node:crypto" -import { TaskCommandName, type TaskCommand } from "@roo-code/types" -import { buildSmokeConfiguration, SMOKE_PROMPT, SMOKE_TIMEOUT_MS, SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { runAgentFixture, type AgentRuntime } from "../agent-evals/runner.ts" +import { getFixture } from "../agent-evals/registry.ts" +import { SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" -import { createTaskStartWaiter } from "./task-start.ts" +import { SMOKE_FIXTURE_NAME, ensureSmokeFixtureRegistered } from "./fixture.ts" + +export type SmokeRuntime = AgentRuntime -export interface SmokeHost { - readonly exited: boolean - onExit(listener: () => void): void - signal(signal: NodeJS.Signals): void -} -export interface SmokeClient { - readonly isReady: boolean - onTaskEvent(listener: (event: unknown) => void): void - onDisconnect(listener: () => void): void - sendCommand(command: TaskCommand): void - disconnect(): void -} export type SmokeVerdict = { kind: "provider-smoke" model: string @@ -32,19 +22,12 @@ export type SmokeVerdict = { } durationMs: number } -export interface SmokeRuntime { - launch(): Promise - connect(): SmokeClient - cleanup(): Promise - writeVerdict(verdict: SmokeVerdict): Promise - onInterrupt(listener: () => void): () => void - now(): number - pause(ms: number): Promise -} -// Only the adapter can spawn VS Code, read credentials or write files. Tests -// exercise the same controller with synthetic IPC and a deterministic clock. +// Thin adapter: the shared controller runs the fixture and writes the trace; +// this function keeps the smoke-specific evidence gates and verdict shape. export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise { + ensureSmokeFixtureRegistered() + getFixture(SMOKE_FIXTURE_NAME) const started = runtime.now() const evidence: SmokeEvidence = { providerResponseSeen: false, @@ -52,98 +35,32 @@ export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise | undefined - async function until(check: () => boolean, timeoutMs: number, stopOnLoss = true): Promise { - const deadline = runtime.now() + timeoutMs - while (!check() && (!stopOnLoss || !sessionLost) && runtime.now() < deadline) await runtime.pause(100) - } - const markLost = () => { - sessionLost = true - waiter?.onExit() - } - const unsubscribe = runtime.onInterrupt(() => { - markLost() - try { - host?.signal("SIGTERM") - } catch { - /* Teardown retries independently. */ - } + const result = await runAgentFixture(runtime, { + fixtureName: SMOKE_FIXTURE_NAME, + credential: apiKey, + observers: { + onAcceptedEvent: (event, taskId) => collectEvidence(evidence, event, taskId), + shouldStop: () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + }, }) - try { - if (!apiKey) throw new Error("Provider credential missing") - host = await runtime.launch() - host.onExit(markLost) - client = runtime.connect() - client.onDisconnect(() => { - sessionLost = true - waiter?.onDisconnect() - }) - let taskId: string | undefined - client.onTaskEvent((event) => { - const id = waiter?.onTaskEvent(event) - if (id && taskId === undefined) taskId = id - collectEvidence(evidence, event, taskId) - }) - await until(() => client?.isReady === true, SMOKE_TIMEOUT_MS) - activated = client.isReady - if (!activated) code = "ACTIVATION_FAILED" - else { - const requestId = randomUUID() - waiter = createTaskStartWaiter({ requestId, timeoutMs: 35_000 }) - // A synchronous send failure must not leave a rejected timer promise. - void waiter.result.catch(() => {}) - client.sendCommand({ - commandName: TaskCommandName.StartNewTask, - data: { requestId, text: SMOKE_PROMPT, newTab: false, configuration: buildSmokeConfiguration(apiKey) }, - }) - try { - taskId = await waiter.result - accepted = true - } catch { - code = "TASK_START_FAILED" - } - if (accepted) { - await until( - () => - evidence.unexpectedTool || - evidence.aborted || - (evidence.providerResponseSeen && evidence.completionSeen), - SMOKE_TIMEOUT_MS, - ) - code = smokeCode(evidence, sessionLost) - } - } - } catch { - // Never expose thrown provider, filesystem or host messages. - } finally { - waiter?.onExit() - const attempt = async (operation: () => unknown) => { - try { - await operation() - } catch { - if (code === "OK") code = "TEARDOWN_FAILED" - } - } - if (client?.isReady) await attempt(() => client?.sendCommand({ commandName: TaskCommandName.CloseTask })) - if (host && !host.exited) { - await attempt(() => until(() => host?.exited === true, 5_000, false)) - for (const signal of ["SIGTERM", "SIGKILL"] as const) { - if (host.exited) break - await attempt(() => host?.signal(signal)) - await attempt(() => until(() => host?.exited === true, 5_000, false)) - } - if (!host.exited && code === "OK") code = "TEARDOWN_FAILED" - } - await attempt(() => client?.disconnect()) - if (!host || host.exited) await attempt(() => runtime.cleanup()) - await attempt(unsubscribe) + let code: SmokeCode + switch (result.execution.code) { + case "SETUP_FAILED": + code = "SETUP_FAILED" + break + case "ACTIVATION_FAILED": + code = "ACTIVATION_FAILED" + break + case "TASK_START_FAILED": + code = "TASK_START_FAILED" + break + default: + code = smokeCode(evidence, result.execution.sessionLost) } + if (result.teardown.status !== "ok" && code === "OK") code = "TEARDOWN_FAILED" const verdict: SmokeVerdict = { kind: "provider-smoke", model: SOLHEIM_MODEL, @@ -151,16 +68,16 @@ export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise string | undefined - onDisconnect: () => void - onExit: () => void - result: Promise -} - -const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null - -// Correlates one StartNewTask request with its sanitized response, similar to -// JSON-RPC: the sender generates the requestId and the reply copies it back. -// The waiter settles exactly once. A matched success resolves with the -// accepted task ID. A matched failure, a timeout, an IPC disconnect, or a -// VS Code exit rejects. Unmatched or malformed responses never settle it. -export function createTaskStartWaiter({ - requestId, - timeoutMs, -}: { - requestId: string - timeoutMs: number -}): TaskStartWaiter { - let resolveResult!: (taskId: string) => void - let rejectResult!: (error: Error) => void - const result = new Promise((resolve, reject) => { - resolveResult = resolve - rejectResult = reject - }) - - let settled = false - let timer: NodeJS.Timeout | undefined - - const settle = (finish: () => void): void => { - if (settled) { - return - } - settled = true - if (timer !== undefined) { - clearTimeout(timer) - timer = undefined - } - finish() - } - - const rejectWith = (message: string): void => { - settle(() => rejectResult(new Error(message))) - } - - timer = setTimeout(() => { - rejectWith(`task start response missing after ${timeoutMs}ms (requestId=${requestId})`) - }, timeoutMs) - - return { - onTaskEvent(event: unknown): string | undefined { - if (settled) return undefined - if (!isRecord(event) || event["eventName"] !== RooCodeEventName.TaskStartResponse) { - return undefined - } - - const payload = event["payload"] - if (!Array.isArray(payload) || payload.length === 0) { - return undefined - } - - const parsed = taskStartResponseSchema.safeParse(payload[0]) - if (!parsed.success || parsed.data.requestId !== requestId) { - return undefined - } - - const response = parsed.data - if (response.success) { - const acceptedTaskId = response.taskId - settle(() => resolveResult(acceptedTaskId)) - return acceptedTaskId - } - - const { errorCode, errorMessage, stage } = response - // The stage is a fixed sanitized token from the response schema; it - // names the startup phase that stopped responding. - const stageDetail = stage === undefined ? "" : ` (stage=${stage})` - rejectWith(`${errorCode}: ${errorMessage}${stageDetail}`) - return undefined - }, - - onDisconnect(): void { - rejectWith("IPC disconnected before the task start response arrived") - }, - - onExit(): void { - rejectWith("VS Code exited before the task start response arrived") - }, - - result, - } -} +export { createTaskStartWaiter, type TaskStartWaiter } from "../agent-evals/task-start.ts" From d0af4a218bd0958fa7b998b061b8e72079de9d50 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 00:47:15 +0000 Subject: [PATCH 04/11] docs(evals): separate completion output from accepted completion in traces The generic runner listened only for the taskCompleted event while the smoke completionSeen gate observes the final nonempty completion_result say message. The extension emits taskCompleted only after the result is accepted and the turn is persisted, so a headless smoke can stop before that event. Record the completion message as execution.completionMessageSeen, report an observer stop after it as adapterStopAfterCompletionMessage, and keep COMPLETED reserved for the accepted taskCompleted event. A manual loss stays SESSION_LOST, output alone stays TASK_RUN_TIMEOUT, and the bounded drain stays taskCompleted-only. Add a clearly labeled synthetic reproduction of the reported two-pre-ack-event topology. --- scripts/agent-evals/README.md | 21 +++ scripts/agent-evals/runner.test.ts | 287 ++++++++++++++++++++++++++++- scripts/agent-evals/runner.ts | 42 ++++- scripts/agent-evals/trace.test.ts | 6 + scripts/agent-evals/trace.ts | 7 + scripts/solheim-smoke/README.md | 16 ++ 6 files changed, 373 insertions(+), 6 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index c1d71ade3d..bd24b6cfba 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -62,6 +62,27 @@ the accepted task and counts the rest as foreign or malformed. After an extension-reported completion the controller drains late usage events for the bounded drain window. +## Terminal code semantics + +The trace carries two different completion signals. Reviewers must not merge +them: + +- `COMPLETED` requires the extension `taskCompleted` event. The extension emits + it only after the completion result is accepted, or a delegated subtask hands + back, and the turn is persisted. A stop request, teardown, or a later session + loss never changes this code. +- `execution.completionMessageSeen` records the final `completion_result` + message for the accepted task. That message is model output, not acceptance. + It never produces `COMPLETED`. +- `STOPPED` with `adapterStopAfterCompletionMessage` names an observer stop + after that message. `STOPPED` with `adapterStop` names an observer stop + without it. `TASK_RUN_TIMEOUT` is the bounded run window expiry. A seen + message still appears in `completionMessageSeen`. +- `SESSION_LOST` covers disconnects and interrupts. A manual stop after + completion output stays `SESSION_LOST`. +- The usage drain runs only after a `taskCompleted` event. It stays inside the + bounded drain window. + ## Checks ```sh diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index 0dc2938b47..90966dc677 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -52,7 +52,13 @@ type Scenario = "success" | "setup" | "activation" | "start-failure" | "disconne function harness( scenario: Scenario, - options: { stuck?: boolean; closeThrows?: boolean; timeout?: () => void; onPause?: (count: number) => void } = {}, + options: { + stuck?: boolean + closeThrows?: boolean + timeout?: () => void + onPause?: (count: number) => void + disconnectOnPause?: number + } = {}, ) { let now = 0 let pauseCount = 0 @@ -207,7 +213,9 @@ function harness( async pause(ms) { now += ms pauseCount++ + // The callback runs first so a test can emit before the loss lands. options.onPause?.(pauseCount) + if (options.disconnectOnPause === pauseCount) onDisconnect() }, } return { @@ -252,6 +260,7 @@ describe("agent fixture controller", () => { }) assert.equal(result.execution.code, "STOPPED") assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.completionMessageSeen, false) assert.equal(result.execution.activated, true) assert.equal(result.execution.taskAccepted, true) assert.equal(result.teardown.status, "ok") @@ -584,4 +593,280 @@ describe("agent fixture controller", () => { assert.equal(result.execution.stopReason, "sessionLost") assert.equal(result.execution.sessionLost, true) }) + + it("reproduces the synthetic PR 1909 topology: an observer stop after the completion message stays distinct", async () => { + // SYNTHETIC FIXTURE. The event topology mirrors one user-reported + // local live smoke run: two events before the start ack, one accounted + // request, two cumulative usage snapshots, one final completion_result + // message, then the observer stops. This is not a recorded provider + // artifact. Token counts are the reported aggregates. Timings are not + // reproduced. + registerTestFixture("pr1909-smoke-topology-synthetic", { + identity: { + name: "pr1909-smoke-topology-synthetic", + version: "0.1.0", + description: + "SYNTHETIC reproduction of one reported live smoke topology. Not a recorded provider artifact.", + }, + }) + const f = harness("no-events", { + onPause: (count) => { + if (count !== 1) return + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { + ts: 2, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai" }), + }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "updated", + message: { + ts: 2, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai", tokensIn: 2719, tokensOut: 29 }), + }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 2719, totalTokensOut: 12, totalCost: 0, contextTokens: 2719 }, + {}, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 2719, totalTokensOut: 29, totalCost: 0, contextTokens: 2748 }, + { attempt_completion: { attempts: 1, failures: 0 } }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 3, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + }, + }) + f.emit({ eventName: RooCodeEventName.TaskStarted, payload: ["root"] }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "text", text: "synthetic early event" }, + }, + ], + }) + let completionOutput = false + const result = await runAgentFixture(f.runtime, { + fixtureName: "pr1909-smoke-topology-synthetic", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => { + if ( + event.eventName === RooCodeEventName.Message && + event.payload[0].message.say === "completion_result" + ) + completionOutput = true + }, + shouldStop: () => completionOutput, + }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStopAfterCompletionMessage") + assert.equal(result.execution.completionMessageSeen, true) + assert.equal(result.execution.sessionLost, false) + assert.equal(result.execution.taskAccepted, true) + assert.equal(result.teardown.status, "ok") + assert.equal(result.teardown.cleanup, "done") + const trace = writtenTrace(f) + // Seven task events plus the counted TaskStartResponse envelope. + assert.equal(trace.eventFlow.observed, 8) + assert.equal(trace.eventFlow.bufferedBeforeAck, 2) + assert.equal(trace.eventFlow.replayedAfterAck, 2) + assert.equal(trace.eventFlow.malformed, 0) + assert.equal(trace.eventFlow.foreignTaskEvents, 0) + assert.equal(trace.accounting.requestCount, 1) + assert.deepEqual(trace.accounting.requests[0]?.tokensIn, { available: true, value: 2719 }) + assert.deepEqual(trace.accounting.requests[0]?.tokensOut, { available: true, value: 29 }) + assert.equal(trace.accounting.requests[0]?.messageCount, 2) + assert.equal(trace.accounting.usage.snapshotsObserved, 2) + assert.equal(trace.accounting.usage.lastSnapshot?.source, "taskTokenUsageUpdated") + assert.equal(trace.accounting.usage.lastSnapshot?.tokensOut, 29) + assert.deepEqual(trace.accounting.toolSnapshot.snapshot, { + attempt_completion: { attempts: 1, failures: 0 }, + }) + assert.equal(trace.accounting.completeness.status, "complete") + assert.deepEqual(trace.accounting.completeness.reasons, []) + // The drain is bound to the accepted-completion event. Without it, no + // drain window runs. + assert.equal(trace.execution.timings.usageDrainMs, undefined) + const serialized = JSON.stringify(trace) + assert.ok(serialized.includes("pr1909-smoke-topology-synthetic")) + assert.ok(!serialized.includes(CREDENTIAL)) + assert.ok(!serialized.includes("synthetic early event")) + assert.ok(!serialized.includes("synthetic final answer")) + assert.ok(!serialized.includes("root")) + }) + + it("keeps completion output non-completing on a run timeout", async () => { + registerTestFixture("controller-output-timeout", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("no-events", { + onPause: (count) => { + if (count !== 1) return + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + }, + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-output-timeout", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(result.execution.stopReason, "runTimeout") + assert.equal(result.execution.completionMessageSeen, true) + assert.equal(writtenTrace(f).accounting.usage.snapshotsObserved, 0) + }) + + it("keeps a manual interrupt a session loss even after completion output", async () => { + registerTestFixture("controller-manual-stop") + const f = harness("no-events", { + onPause: (count) => { + if (count === 1) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "synthetic final answer", + }, + }, + ], + }) + if (count === 2) f.interrupt() + }, + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-manual-stop", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "SESSION_LOST") + assert.equal(result.execution.stopReason, "sessionLost") + assert.equal(result.execution.completionMessageSeen, true) + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + const trace = writtenTrace(f) + assert.ok(trace.accounting.completeness.reasons.includes("sessionLost")) + }) + + it("keeps a confirmed completion when the session dies in the same window", async () => { + registerTestFixture("controller-completion-outranks-loss") + const f = harness("success", { + // The completion event lands, then the session dies, before the run + // loop can poll again. + onPause: (count) => { + if (count === 1) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + }, + disconnectOnPause: 1, + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-completion-outranks-loss", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.stopReason, "taskCompleted") + assert.equal(result.execution.sessionLost, true) + const trace = writtenTrace(f) + assert.ok(trace.accounting.completeness.reasons.includes("sessionLost")) + // A loss in the same window skips the drain and keeps the caveat. + assert.equal(trace.execution.timings.usageDrainMs, undefined) + }) + + it("keeps a confirmed completion when the observer requests a stop", async () => { + registerTestFixture("controller-completion-outranks-stop") + const f = harness("no-events") + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-completion-outranks-stop", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.stopReason, "taskCompleted") + assert.equal(result.execution.completionMessageSeen, true) + assert.equal(result.execution.sessionLost, false) + assert.equal(result.teardown.status, "ok") + }) }) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index aa25623ee7..1ec6335ce2 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -66,6 +66,7 @@ export interface AgentRunResult { activated: boolean taskAccepted: boolean sessionLost: boolean + completionMessageSeen: boolean } teardown: AgentTeardown trace: AgentTrace @@ -102,6 +103,10 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp let lostDuringDrain = false let acceptedTaskId: string | undefined let completedSeen = false + // True once the final completion_result say message for the accepted task + // arrived. This is model output, not the accepted-completion event. It + // never produces COMPLETED; it only refines how a stop is reported. + let completionMessageSeen = false // The reason must name what actually stopped evaluation. "notEvaluated" // means the projection was never inspected; it is replaced below. let settings: AgentTrace["settings"] = { available: false, reason: "notEvaluated" } @@ -157,6 +162,17 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp return } accountant.observe(event) + if (event.eventName === RooCodeEventName.Message) { + const message = event.payload[0].message + if ( + message.type === "say" && + message.say === "completion_result" && + message.partial !== true && + typeof message.text === "string" && + message.text.trim().length > 0 + ) + completionMessageSeen = true + } if (event.eventName === RooCodeEventName.TaskCompleted) completedSeen = true options.observers?.onAcceptedEvent?.(event, acceptedTaskId) } @@ -270,12 +286,20 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp lostDuringDrain = sessionLost } const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true - if (runSessionLost) { - code = "SESSION_LOST" - stopReason = "sessionLost" - } else if (completedSeen) { + if (completedSeen) { + // A confirmed accepted completion outranks a stop + // request or a loss observed in the same window. code = "COMPLETED" stopReason = "taskCompleted" + } else if (runSessionLost) { + code = "SESSION_LOST" + stopReason = "sessionLost" + } else if (stopped && completionMessageSeen) { + // The observer stopped after the model's final + // completion message. The accepted-completion event + // never fired, so this is not COMPLETED. + code = "STOPPED" + stopReason = "adapterStopAfterCompletionMessage" } else if (stopped) { code = "STOPPED" stopReason = "adapterStop" @@ -363,6 +387,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp activated, taskAccepted: accepted, sessionLost: runSessionLost, + completionMessageSeen, timings, }, eventFlow: flow, @@ -372,7 +397,14 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp } await runtime.writeArtifact("trace.json", trace) return { - execution: { code, stopReason, activated, taskAccepted: accepted, sessionLost: runSessionLost }, + execution: { + code, + stopReason, + activated, + taskAccepted: accepted, + sessionLost: runSessionLost, + completionMessageSeen, + }, teardown, trace, } diff --git a/scripts/agent-evals/trace.test.ts b/scripts/agent-evals/trace.test.ts index 463c789b4c..d669fced05 100644 --- a/scripts/agent-evals/trace.test.ts +++ b/scripts/agent-evals/trace.test.ts @@ -197,6 +197,12 @@ describe("trace helpers", () => { assert.ok(TRACE_NOTES.some((note) => note.includes("nonsecret settings projection"))) }) + it("documents the two completion signals and the observer stop reasons", () => { + assert.ok(TRACE_NOTES.some((note) => note.includes("taskCompleted event"))) + assert.ok(TRACE_NOTES.some((note) => note.includes("completion_result"))) + assert.ok(TRACE_NOTES.some((note) => note.includes("adapterStopAfterCompletionMessage"))) + }) + it("digests projections stably regardless of key order", () => { assert.equal(digestProjection({ a: 1, b: { c: 2 } }), digestProjection({ b: { c: 2 }, a: 1 })) assert.notEqual(digestProjection({ a: 1 }), digestProjection({ a: 2 })) diff --git a/scripts/agent-evals/trace.ts b/scripts/agent-evals/trace.ts index fd5d6990c7..2cdb2299ca 100644 --- a/scripts/agent-evals/trace.ts +++ b/scripts/agent-evals/trace.ts @@ -41,6 +41,7 @@ export type AgentExecutionCode = export type AgentStopReason = | "taskCompleted" | "adapterStop" + | "adapterStopAfterCompletionMessage" | "runTimeout" | "sessionLost" | "startNotAccepted" @@ -143,6 +144,9 @@ export interface AgentTrace { activated: boolean taskAccepted: boolean sessionLost: boolean + // True when the final completion_result message for the accepted task + // was observed. This is model output, not acceptance. + completionMessageSeen: boolean timings: { activationMs?: number taskStartMs?: number @@ -167,6 +171,9 @@ export const TRACE_NOTES = [ "apiReqRetriedMarkers counts retry marker messages. It is not an HTTP retry count.", "The trace is metadata only. Prompts, model output, reasoning, tool arguments, error payloads, and storage paths are excluded. Provider settings appear only as the nonsecret settings projection.", "Missing measurements carry a reason. They are never filled with zero.", + "execution.code COMPLETED requires the extension taskCompleted event. The extension emits it only after the completion result is accepted, or a delegated subtask hands back, and the turn is persisted.", + "The final completion_result message is model output, not acceptance. execution.completionMessageSeen records it and never produces COMPLETED.", + "An observer stop after that message is stopReason adapterStopAfterCompletionMessage with code STOPPED. An observer stop without it stays adapterStop.", ] const sortValue = (value: unknown): unknown => { diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index b7421fa078..fc80d643fb 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -38,6 +38,14 @@ Teardown escalates through close, terminate, and kill within independent five-se completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. The driver never approves dialogs or tool execution; it closes the task during teardown. +The `completionSeen` gate is the final non-empty `completion_result` message for the accepted task. That +message is model output. It is not the extension `taskCompleted` event, which fires only after a user +accepts the result. The driver never approves dialogs, so `taskCompleted` does not fire in a smoke run. +A passing smoke therefore ends with `execution.code: "STOPPED"` and +`stopReason: "adapterStopAfterCompletionMessage"` in `trace.json`. This is the expected terminal state. +It does not contradict the `OK` verdict. See +[`scripts/agent-evals/README.md`](../agent-evals/README.md) for the terminal code semantics. + The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names), `trace.json` (versioned metadata-only structured trace from the shared substrate; see [`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`, @@ -69,6 +77,14 @@ after the frozen Stage 1B pilot produced zero incremental verified defects. Its receipts are archived outside this smoke-only change; the chronology, results, caveats and preservation details are recorded in [tracking issue #1896](https://github.com/Zoo-Code-Org/Zoo-Code/issues/1896). +Validation status, kept as two separate records. One user-reported local live run passed the smoke +verdict with `completionSeen: true`: total about 6390 ms, activation about 3010 ms, task about 3110 ms, +2719 input and 29 output tokens, two early buffered and replayed events, two cumulative usage snapshots, +and a clean teardown with no malformed, foreign, tool failure, or retry events. That trace reported +`STOPPED` with `adapterStopAfterCompletionMessage`, which matches the signal distinction above. This +report is not independently verified. The trusted-main workflow run is still pending. Only a +`.github/workflows/solheim-provider-smoke.yml` run on trusted main counts as verified CI evidence. + Planned follow-ups on top of the shared substrate, in order: five frozen real coding tasks, independent credential-free graders, paired exact trusted commit comparisons, repeated runs with uncertainty reporting, then advisory release integration aligned to the actual release workflows. They are tracked From 51dfc8f5f2ab01575755e9922bfccb951a11b9c0 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 01:20:34 +0000 Subject: [PATCH 05/11] fix(smoke): upload only validated smoke recordings The recorder wrote ffmpeg straight to the final smoke.mp4. A failed run could leave a non-empty invalid MP4 at the uploaded path, and the workflow could not report a missing video through one multi-path if-no-files-found upload. - write ffmpeg to a unique temporary file inside SOLHEIM_SMOKE_OUT_DIR, with the mp4 muxer forced for the suffix-less name - publish through validation only: non-empty plus ffprobe, then an atomic same-directory rename onto smoke.mp4 - remove a stale final before recording starts, and remove the temporary file on every failure, interrupt, and validation failure - treat a recorder that stopped on its own with a non-zero status as a recording failure even when the fragments still probe; a recorder alive at shutdown is judged by content validation, because ffmpeg exits non-zero after its graceful INT or TERM stop - write the outcome to recording-status.txt, keep the provider exit code unchanged, and stop the provider during signal teardown - upload metadata and the recording as separate artifacts, with an explicit recording integrity check, so a missing video fails its own steps - extend the stub-based record.sh behavior tests to cover publication, invalid and stale output, recorder self-exit, and signal teardown --- .github/workflows/solheim-provider-smoke.yml | 28 +- scripts/solheim-smoke-workflow.test.mjs | 284 +++++++++++++++++-- scripts/solheim-smoke/README.md | 20 +- scripts/solheim-smoke/record.sh | 81 +++++- 4 files changed, 371 insertions(+), 42 deletions(-) diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index f73ebd2c0b..452ed5f86b 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -48,7 +48,20 @@ jobs: SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh - - name: Upload provider smoke verdict and recording + # The recorder publishes smoke.mp4 only after a nonempty check and + # ffprobe validation, so any other status is a recording failure. + - name: Verify recording integrity + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + env: + SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke + run: | + status="$(cat "$SOLHEIM_SMOKE_OUT_DIR/recording-status.txt" 2>/dev/null || echo missing)" + echo "Recording status: $status" + test "$status" = ok + + # Verdict and trace evidence must survive even when the recording + # is missing, so metadata and video are uploaded separately. + - name: Upload provider smoke verdict and trace if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -56,6 +69,17 @@ jobs: path: | ${{ runner.temp }}/solheim-provider-smoke/verdict.json ${{ runner.temp }}/solheim-provider-smoke/trace.json - ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 + ${{ runner.temp }}/solheim-provider-smoke/recording-status.txt + if-no-files-found: error + retention-days: 7 + + # Exact validated final path only; the recorder leaves no temporary + # file behind, so a failed upload here reports the missing recording. + - name: Upload provider smoke recording + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: solheim-provider-smoke-recording-${{ github.sha }} + path: ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 if-no-files-found: error retention-days: 7 diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 3a2ed0689f..84daa1125e 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict" import fs from "node:fs" import { URL } from "node:url" import { describe, it } from "node:test" -import { spawnSync } from "node:child_process" +import { spawn, spawnSync } from "node:child_process" import os from "node:os" import path from "node:path" const workflow = fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8") @@ -64,24 +64,53 @@ describe("Project A provider smoke workflow", () => { assert.match(workflow, /timeout-minutes: 8/) assert.match(workflow, /^ {8}timeout-minutes: 25$/m) }) - it("uploads only the verdict, the structured trace, and video, never host logs or storage", () => { - const paths = workflow.split("path: |\n")[1]?.split(" if-no-files-found:")[0] + it("uploads metadata and the recording as separate validated artifacts", () => { + const metadataStep = + workflow.split("- name: Upload provider smoke verdict and trace")[1]?.split("- name:")[0] ?? "" + const recordingStep = workflow.split("- name: Upload provider smoke recording")[1]?.split("- name:")[0] ?? "" + assert.ok(metadataStep.length > 0) + assert.ok(recordingStep.length > 0) assert.deepEqual( - paths - ?.trim() + metadataStep + .split("path: |")[1] + ?.split("if-no-files-found:")[0] + .trim() .split("\n") .map((line) => line.trim()), [ "${{ runner.temp }}/solheim-provider-smoke/verdict.json", "${{ runner.temp }}/solheim-provider-smoke/trace.json", - "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", + "${{ runner.temp }}/solheim-provider-smoke/recording-status.txt", ], ) - assert.match(workflow, /!cancelled\(\)/) - assert.match(workflow, /if-no-files-found: error/) + // The recording artifact carries only the exact validated final path. + assert.equal( + recordingStep.split("path: ")[1]?.split("\n")[0].trim(), + "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", + ) + for (const step of [metadataStep, recordingStep]) { + assert.match(step, /!cancelled\(\)/) + assert.match(step, /if-no-files-found: error/) + assert.match(step, /retention-days: 7/) + assert.match(step, /actions\/upload-artifact@[0-9a-f]{40}/) + assert.ok(!step.includes("*")) + } + // A missing recording cannot hide behind a multi-path if-no-files-found. + assert.match(workflow, /name: solheim-provider-smoke-recording-\$\{\{ github\.sha \}\}/) + // Temporary recorder output is never an upload path. + assert.ok(!workflow.includes(".tmp")) assert.ok(!driver.includes("console.error(")) assert.ok(!driver.includes("stderrLog")) }) + it("reports a missing or failed recording explicitly before the uploads", () => { + const checkStep = workflow.split("- name: Verify recording integrity")[1]?.split("- name:")[0] ?? "" + assert.match(checkStep, /recording-status\.txt/) + assert.match(checkStep, /test "\$status" = ok/) + assert.match(checkStep, /!cancelled\(\)/) + const checkIndex = workflow.indexOf("- name: Verify recording integrity") + const uploadIndex = workflow.indexOf("- name: Upload provider smoke verdict and trace") + assert.ok(checkIndex >= 0 && checkIndex < uploadIndex) + }) it("records a fresh display with credential-free ffmpeg and bounded teardown", () => { assert.match(workflow, /install -y xvfb ffmpeg/) assert.match(workflow, /xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp"/) @@ -92,6 +121,16 @@ describe("Project A provider smoke workflow", () => { assert.match(recorder, /attempt < 50/) assert.ok(!recorder.includes("SOLHEIM_API_KEY")) assert.match(workflow, /retention-days: 7/) + // The recorder writes a unique temporary file inside the output + // directory, forces the mp4 muxer because the name has no mp4 suffix, + // and publishes only validated output with an atomic rename. + assert.match(recorder, /temp_recording="\$SOLHEIM_SMOKE_OUT_DIR\/\.smoke\.mp4\.\$\$\.tmp"/) + assert.match(recorder, /-f mp4 "\$temp_recording"/) + assert.match(recorder, /ffprobe -v error "\$temp_recording"/) + assert.match(recorder, /mv -f "\$temp_recording" "\$final_recording"/) + assert.match(recorder, /rm -f "\$final_recording"/) + // ffmpeg must never write the final path directly. + assert.ok(!/ffmpeg[^\n]*\$final_recording/.test(recorder)) }) it("keeps Code QA and package commands focused on smoke contracts", () => { for (const command of [ @@ -137,43 +176,230 @@ describe("Project A provider smoke workflow", () => { }) // Runs record.sh with stub ffmpeg and ffprobe, so no display or codec is needed. -function runRecorder(args, { ffmpegWrites = true, ffprobeOk = true } = {}) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) +const defaultFfmpeg = (writes) => + `${writes ? 'for a; do :; done; echo data > "$a"' : ":"}\ntrap 'exit 0' TERM\nwhile true; do sleep 0.05; done` + +function stubRecorderBin(dir, { ffmpeg, ffmpegWrites = true, ffprobeOk = true } = {}) { const bin = path.join(dir, "bin") - fs.mkdirSync(bin) + fs.mkdirSync(bin, { recursive: true }) const stub = (name, body) => fs.writeFileSync(path.join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 }) - // The stub ffmpeg writes the last argument, stops on INT, and exits. - stub( - "ffmpeg", - `${ffmpegWrites ? 'for a; do :; done; echo data > "$a"' : ":"}\ntrap 'exit 0' INT\nwhile true; do sleep 0.05; done`, - ) + // The stub ffmpeg writes the last argument (the recorder's output path), + // stops on TERM, and exits. + stub("ffmpeg", ffmpeg ?? defaultFfmpeg(ffmpegWrites)) stub("ffprobe", ffprobeOk ? "exit 0" : "exit 1") - const result = spawnSync("bash", [new URL("./solheim-smoke/record.sh", import.meta.url).pathname, ...args], { - env: { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":99", SOLHEIM_SMOKE_OUT_DIR: path.join(dir, "out") }, + return bin +} + +function recorderPaths(dir) { + const outDir = path.join(dir, "out") + return { + outDir, + statusFile: path.join(outDir, "recording-status.txt"), + finalRecording: path.join(outDir, "smoke.mp4"), + } +} + +function recorderEnv(bin, outDir, providerMarker) { + return { + PATH: `${bin}:${process.env.PATH}`, + DISPLAY: ":99", + SOLHEIM_SMOKE_OUT_DIR: outDir, + ...(providerMarker ? { PROVIDER_MARKER: providerMarker } : {}), + } +} + +const recorderScript = new URL("./solheim-smoke/record.sh", import.meta.url).pathname + +function runRecorder(args, { ffmpeg, ffmpegWrites = true, ffprobeOk = true, seedFinal, providerMarker } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir, { ffmpeg, ffmpegWrites, ffprobeOk }) + const paths = recorderPaths(dir) + fs.mkdirSync(paths.outDir, { recursive: true }) + if (seedFinal !== undefined) fs.writeFileSync(paths.finalRecording, seedFinal) + const result = spawnSync("bash", [recorderScript, ...args], { + env: recorderEnv(bin, paths.outDir, providerMarker), encoding: "utf8", timeout: 20_000, }) - fs.rmSync(dir, { recursive: true, force: true }) - return result + return { result, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) } +} + +function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, providerMarker } = {}) { + return new Promise((resolve) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir, { ffmpeg, ffprobeOk }) + const paths = recorderPaths(dir) + const child = spawn("bash", [recorderScript, ...args], { + env: recorderEnv(bin, paths.outDir, providerMarker), + }) + let stdout = "" + child.stdout.on("data", (chunk) => { + stdout += chunk + }) + const watchdog = setTimeout(() => child.kill("SIGKILL"), 15_000) + const timer = setTimeout(() => child.kill("SIGTERM"), delayMs) + child.on("close", (code) => { + clearTimeout(watchdog) + clearTimeout(timer) + resolve({ code, stdout, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) }) + }) + }) } +const tempFilesLeft = (run) => fs.readdirSync(run.outDir).filter((name) => name.includes(".tmp")) +const recordingStatus = (run) => fs.readFileSync(run.statusFile, "utf8").trim() + describe("record.sh exit and validation behavior", () => { it("passes when the command succeeds and the recording is valid", () => { - assert.equal(runRecorder(["true"]).status, 0) + const run = runRecorder(["true"]) + try { + assert.equal(run.result.status, 0) + assert.equal(recordingStatus(run), "ok") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + } }) - it("keeps the exit code of a failing command", () => { - assert.equal(runRecorder(["bash", "-c", "exit 7"]).status, 7) + it("keeps the exit code of a failing command and still publishes a valid recording", () => { + const run = runRecorder(["bash", "-c", "exit 7"]) + try { + assert.equal(run.result.status, 7) + assert.equal(recordingStatus(run), "ok") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + } finally { + run.cleanup() + } }) it("fails when the recording is missing", () => { - const result = runRecorder(["true"], { ffmpegWrites: false }) - assert.equal(result.status, 1) - assert.match(result.stdout, /recording failed/) + const run = runRecorder(["true"], { ffmpegWrites: false }) + try { + assert.equal(run.result.status, 1) + assert.match(run.result.stdout, /recording failed/) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.notEqual(recordingStatus(run), "ok") + } finally { + run.cleanup() + } }) - it("fails when ffprobe rejects the recording", () => { - assert.equal(runRecorder(["true"], { ffprobeOk: false }).status, 1) + it("fails and cleans the temporary file when ffprobe rejects the recording", () => { + const run = runRecorder(["true"], { ffprobeOk: false }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.deepEqual(tempFilesLeft(run), []) + assert.notEqual(recordingStatus(run), "ok") + } finally { + run.cleanup() + } }) it("keeps a failing command code when the recording is also invalid", () => { - assert.equal(runRecorder(["bash", "-c", "exit 7"], { ffprobeOk: false }).status, 7) + const run = runRecorder(["bash", "-c", "exit 7"], { ffprobeOk: false }) + try { + assert.equal(run.result.status, 7) + assert.ok(!fs.existsSync(run.finalRecording)) + } finally { + run.cleanup() + } + }) + it("removes a stale final recording when the new recording fails", () => { + const run = runRecorder(["true"], { ffmpegWrites: false, seedFinal: "STALE" }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(run.finalRecording)) + } finally { + run.cleanup() + } + }) + it("replaces a stale final recording with the validated new recording", () => { + const run = runRecorder(["true"], { seedFinal: "STALE" }) + try { + assert.equal(run.result.status, 0) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + } finally { + run.cleanup() + } + }) + it("aborts before the command when the recorder dies at startup", () => { + const marker = path.join(os.tmpdir(), `record-sh-marker-${process.pid}-${Math.random()}`) + const run = runRecorder(["bash", "-c", 'echo ran > "$PROVIDER_MARKER"'], { + ffmpeg: 'for a; do :; done; echo partial > "$a"\nexit 1', + providerMarker: marker, + }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(marker)) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.match(recordingStatus(run), /stopped on its own/) + } finally { + run.cleanup() + fs.rmSync(marker, { force: true }) + } + }) + it("rejects recorder output that stopped on its own with a failure status", () => { + const run = runRecorder(["bash", "-c", "sleep 2"], { + // The content is readable and would pass ffprobe, so the rejection + // must come from the recorder's own non-zero exit status. + ffmpeg: 'for a; do :; done\nsleep 0.7\necho data > "$a"\nexit 1', + }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.deepEqual(tempFilesLeft(run), []) + assert.match(recordingStatus(run), /stopped on its own/) + } finally { + run.cleanup() + } + }) + it("publishes when the recorder stops on its own with success (size or time limit)", () => { + const run = runRecorder(["bash", "-c", "sleep 2"], { + ffmpeg: 'for a; do :; done\nsleep 0.7\necho data > "$a"\nexit 0', + }) + try { + assert.equal(run.result.status, 0) + assert.equal(recordingStatus(run), "ok") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + } finally { + run.cleanup() + } + }) +}) + +describe("record.sh signal teardown behavior", () => { + it("stops the recorder and the command on SIGTERM and keeps the signal exit code", async () => { + const marker = path.join(os.tmpdir(), `record-sh-marker-${process.pid}-${Math.random()}`) + const run = await runRecorderSignalled( + // The inner wait makes the stub command respond to TERM promptly, the + // way a signal-safe driver does. The trap exits so the "normal" write + // cannot run after a handled signal. + [ + "bash", + "-c", + `trap 'printf killed > "$PROVIDER_MARKER"; exit 143' TERM; sleep 5 & wait; printf normal > "$PROVIDER_MARKER"`, + ], + { delayMs: 800, providerMarker: marker }, + ) + try { + assert.equal(run.code, 143) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.equal(recordingStatus(run), "ok") + assert.equal(fs.readFileSync(marker, "utf8"), "killed") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + fs.rmSync(marker, { force: true }) + } + }) + it("leaves no final recording when the interrupted output fails validation", async () => { + const run = await runRecorderSignalled(["bash", "-c", "sleep 5"], { delayMs: 800, ffprobeOk: false }) + try { + assert.equal(run.code, 143) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.deepEqual(tempFilesLeft(run), []) + assert.notEqual(recordingStatus(run), "ok") + } finally { + run.cleanup() + } }) }) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index fc80d643fb..ed63972416 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -54,8 +54,24 @@ eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artif contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task configuration or credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt -interruption. Ordinary success and failure runs upload all three artifacts; cancellation may prevent -finalization/upload. +interruption. + +The recorder writes ffmpeg output to a unique temporary file inside the smoke output directory, with the +MP4 muxer forced because the temporary name has no mp4 suffix. After bounded shutdown it publishes the +recording only when the file is non-empty and `ffprobe` accepts it, through an atomic same-directory +rename onto `smoke.mp4`. Every failed check removes the temporary file and leaves no final recording, and +a stale `smoke.mp4` from an earlier attempt in a reused directory is removed before recording starts. +ffmpeg exits non-zero after its graceful INT or TERM shutdown, so a recorder that was still alive at +shutdown is judged by content validation alone. A recorder that stopped on its own before shutdown must +have exited zero (the `-t` time or `-fs` size limits); any other self-chosen status fails the recording +even when the fragments would still probe. The wrapper records the outcome in `recording-status.txt` +(`ok`, or the failure reason), reports the reason on stdout, and keeps the provider exit code unchanged. + +Metadata (`verdict.json`, `trace.json`, `recording-status.txt`) and the video are uploaded as two separate +artifacts, each with `if-no-files-found: error`. A missing recording therefore fails its own upload step +and the explicit integrity check step, instead of hiding behind a multi-path upload. Verdict and trace +evidence stay available when the provider failed and the video is missing. Cancellation may prevent +finalization and upload. Checks, without a provider key or network calls: diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh index f69e17b17f..b3c057d839 100644 --- a/scripts/solheim-smoke/record.sh +++ b/scripts/solheim-smoke/record.sh @@ -4,33 +4,87 @@ set -euo pipefail : "${DISPLAY:?Run this wrapper under xvfb-run}" : "${SOLHEIM_SMOKE_OUT_DIR:?Set a scratch output directory}" mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" -recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" +final_recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" +status_file="$SOLHEIM_SMOKE_OUT_DIR/recording-status.txt" + +# A stale final from an earlier attempt must never survive into this attempt's +# upload. Remove only this exact path. +rm -f "$final_recording" +echo pending > "$status_file" # The recorder needs display access, not provider/GitHub credentials. Fragmented # MP4 keeps completed fragments readable if the action is forcibly interrupted. +# The output muxer is forced with -f mp4 because the temporary file name has no +# mp4 suffix for ffmpeg to infer the format from. +temp_recording="$SOLHEIM_SMOKE_OUT_DIR/.smoke.mp4.$$.tmp" env -i "PATH=$PATH" "DISPLAY=$DISPLAY" "XAUTHORITY=${XAUTHORITY:-}" \ ffmpeg -nostdin -hide_banner -loglevel error -y \ -f x11grab -video_size 1280x720 -framerate 12 -i "$DISPLAY" \ -an -c:v libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p -g 24 \ -t 480 -fs 125829120 -movflags +frag_keyframe+empty_moov \ - "$recording" >/dev/null 2>&1 & + -f mp4 "$temp_recording" >/dev/null 2>&1 & recorder_pid=$! +provider_pid="" + +# Accepted normal shutdown semantics: ffmpeg exits non-zero after its graceful +# INT or TERM shutdown, so a recorder that was still alive when shutdown started +# is judged by content validation, not by its interrupted exit status. A recorder +# that stopped on its own before shutdown must have exited zero (the -t or -fs +# limits); any other self-chosen non-zero status fails the recording even when +# ffprobe can still read the fragments. +report_recording_failure() { + echo "Solheim smoke recording failed: $1" + echo "$1" > "$status_file" + [[ "$result" -ne 0 ]] || result=1 +} + +stop_provider() { + [[ -n "$provider_pid" ]] || return 0 + kill -TERM "$provider_pid" 2>/dev/null || return 0 + for ((attempt = 0; attempt < 30; attempt++)); do + kill -0 "$provider_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$provider_pid" 2>/dev/null || true + wait "$provider_pid" 2>/dev/null || true +} + finish() { result=$? trap - EXIT - kill -INT "$recorder_pid" 2>/dev/null || true + # Teardown must run to completion once it starts. + trap '' INT TERM + local recorder_status=0 recorder_was_alive=0 + if kill -0 "$recorder_pid" 2>/dev/null; then + recorder_was_alive=1 + fi + # TERM is a graceful stop for ffmpeg: it finalizes the fragments before exit. + kill -TERM "$recorder_pid" 2>/dev/null || true # A hung recorder must not hold the action open indefinitely. for ((attempt = 0; attempt < 50; attempt++)); do kill -0 "$recorder_pid" 2>/dev/null || break sleep 0.1 done kill -KILL "$recorder_pid" 2>/dev/null || true - wait "$recorder_pid" 2>/dev/null || true - if [[ ! -s "$recording" ]] || ! env -i "PATH=$PATH" \ - ffprobe -v error "$recording" >/dev/null 2>&1; then - echo "Solheim smoke recording failed" - [[ "$result" -ne 0 ]] || result=1 + wait "$recorder_pid" 2>/dev/null || recorder_status=$? + # An interrupted provider must not outlive the wrapper. + stop_provider + if [[ "$recorder_was_alive" -eq 0 ]] && [[ "$recorder_status" -ne 0 ]]; then + rm -f "$temp_recording" + report_recording_failure "recorder stopped on its own with exit status $recorder_status" + elif [[ ! -s "$temp_recording" ]]; then + rm -f "$temp_recording" + report_recording_failure "recorder output is missing or empty" + elif ! env -i "PATH=$PATH" ffprobe -v error "$temp_recording" >/dev/null 2>&1; then + rm -f "$temp_recording" + report_recording_failure "recorder output failed ffprobe validation" + elif mv -f "$temp_recording" "$final_recording"; then + # Same-directory rename: atomic, so no partially written final exists. + echo ok > "$status_file" + else + rm -f "$temp_recording" + report_recording_failure "validated recording could not be published" fi exit "$result" } @@ -41,4 +95,13 @@ sleep 0.5 kill -0 "$recorder_pid" 2>/dev/null || exit 1 # Arguments are used for credential-free recorder tests; CI supplies none. -if [[ "$#" -gt 0 ]]; then "$@"; else pnpm solheim:smoke; fi +# Backgrounding the provider keeps the signal traps responsive during the run. +if [[ "$#" -gt 0 ]]; then + "$@" & +else + pnpm solheim:smoke & +fi +provider_pid=$! +provider_status=0 +wait "$provider_pid" || provider_status=$? +exit "$provider_status" From 7d26aac484730b4b04e92e1203869dfd3076e390 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 01:26:58 +0000 Subject: [PATCH 06/11] test(evals): reject invalid completion_result variants before adapter stop Add the negative case for the completion filter. A partial stream, a whitespace-only final text, and an ask-side completion_result stay observed by the fixture observer but never mark completionMessageSeen. The stop keeps code STOPPED with stopReason adapterStop, and the written trace records the same values. --- scripts/agent-evals/runner.test.ts | 58 ++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index 90966dc677..25dd7e5bca 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -733,6 +733,64 @@ describe("agent fixture controller", () => { assert.ok(!serialized.includes("root")) }) + it("keeps invalid completion_result variants observed but not marked as the completion message", async () => { + registerTestFixture("controller-invalid-completion") + const f = harness("no-events") + // One message per rejection condition of the completion filter: a + // partial stream, whitespace-only final text, and the ask-side + // completion_result prompt. + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "partial output", partial: true }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 2, type: "say", say: "completion_result", text: " \n\t " }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 3, type: "ask", ask: "completion_result", text: "awaiting feedback" }, + }, + ], + }) + const accepted: TaskEvent[] = [] + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-invalid-completion", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => accepted.push(event), + shouldStop: () => true, + }, + }) + const trace = writtenTrace(f) + // Three task events plus the counted TaskStartResponse envelope. + assert.equal(accepted.length, 3) + assert.equal(trace.eventFlow.observed, 4) + assert.equal(trace.eventFlow.replayedAfterAck, 3) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.completionMessageSeen, false) + assert.equal(trace.execution.stopReason, "adapterStop") + assert.equal(trace.execution.completionMessageSeen, false) + assert.ok(JSON.stringify(trace).includes('"completionMessageSeen":false')) + }) + it("keeps completion output non-completing on a run timeout", async () => { registerTestFixture("controller-output-timeout", { limits: { From 80504815391fcc91e6fcf0a0c5e19a686b1b9098 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 02:15:57 +0000 Subject: [PATCH 07/11] fix(smoke): stop the driver PID directly and publish only atomic finals The recorder previously started pnpm solheim:smoke and managed the package-manager PID with a 3 second stop window, shorter than the driver's 3x5s teardown windows. It now starts node --import tsx driver.mts directly, so TERM and the documented stop budget (SOLHEIM_SMOKE_STOP_BUDGET_S, default 30s) reach the driver PID that owns teardown and the atomic verdict and trace writes. KILL stays the last resort for a hung driver and can leave host processes behind. The redundant recording-status.txt marker is removed instead of made transactional. The published smoke.mp4 is the commit marker: the recorder removes a stale final at start, validates the temp recording (nonempty plus ffprobe), then renames it into place atomically. The workflow checks the final is nonempty and keeps metadata and video uploads separate, so a missing video stays explicit and failure metadata survives. --- .github/workflows/solheim-provider-smoke.yml | 12 +- scripts/solheim-smoke-workflow.test.mjs | 151 ++++++++++++++++--- scripts/solheim-smoke/README.md | 24 ++- scripts/solheim-smoke/record.sh | 23 ++- 4 files changed, 171 insertions(+), 39 deletions(-) diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index 452ed5f86b..bf21463897 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -48,16 +48,17 @@ jobs: SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh - # The recorder publishes smoke.mp4 only after a nonempty check and - # ffprobe validation, so any other status is a recording failure. + # The recorder publishes smoke.mp4 only through an atomic rename + # after a nonempty check and ffprobe validation, so the published + # final is the commit marker: present and nonempty means complete. + # There is no separate persisted status marker. - name: Verify recording integrity if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} env: SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke run: | - status="$(cat "$SOLHEIM_SMOKE_OUT_DIR/recording-status.txt" 2>/dev/null || echo missing)" - echo "Recording status: $status" - test "$status" = ok + test -s "$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" + echo "Recording published and nonempty: $SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" # Verdict and trace evidence must survive even when the recording # is missing, so metadata and video are uploaded separately. @@ -69,7 +70,6 @@ jobs: path: | ${{ runner.temp }}/solheim-provider-smoke/verdict.json ${{ runner.temp }}/solheim-provider-smoke/trace.json - ${{ runner.temp }}/solheim-provider-smoke/recording-status.txt if-no-files-found: error retention-days: 7 diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 84daa1125e..b2614626d4 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -80,7 +80,6 @@ describe("Project A provider smoke workflow", () => { [ "${{ runner.temp }}/solheim-provider-smoke/verdict.json", "${{ runner.temp }}/solheim-provider-smoke/trace.json", - "${{ runner.temp }}/solheim-provider-smoke/recording-status.txt", ], ) // The recording artifact carries only the exact validated final path. @@ -104,8 +103,9 @@ describe("Project A provider smoke workflow", () => { }) it("reports a missing or failed recording explicitly before the uploads", () => { const checkStep = workflow.split("- name: Verify recording integrity")[1]?.split("- name:")[0] ?? "" - assert.match(checkStep, /recording-status\.txt/) - assert.match(checkStep, /test "\$status" = ok/) + // The published final is the commit marker; no separate status file exists. + assert.match(checkStep, /test -s "\$SOLHEIM_SMOKE_OUT_DIR\/smoke\.mp4"/) + assert.ok(!workflow.includes("recording-status")) assert.match(checkStep, /!cancelled\(\)/) const checkIndex = workflow.indexOf("- name: Verify recording integrity") const uploadIndex = workflow.indexOf("- name: Upload provider smoke verdict and trace") @@ -132,6 +132,18 @@ describe("Project A provider smoke workflow", () => { // ffmpeg must never write the final path directly. assert.ok(!/ffmpeg[^\n]*\$final_recording/.test(recorder)) }) + it("signals the Node driver directly with a documented stop budget", () => { + // A package-manager wrapper between the recorder and the driver would + // receive the TERM and KILL meant for the driver that performs teardown + // and the atomic verdict and trace writes. + assert.ok(!recorder.includes("pnpm")) + assert.match(recorder, /node --import tsx "\$script_dir\/driver\.mts" &/) + assert.match(recorder, /SOLHEIM_SMOKE_STOP_BUDGET_S:-30/) + assert.match(recorder, /kill -TERM "\$provider_pid"/) + assert.match(recorder, /kill -KILL "\$provider_pid"/) + assert.match(recorder, /wait "\$provider_pid"/) + assert.ok(!recorder.includes("status_file")) + }) it("keeps Code QA and package commands focused on smoke contracts", () => { for (const command of [ "test:solheim-smoke:unit", @@ -195,43 +207,51 @@ function recorderPaths(dir) { const outDir = path.join(dir, "out") return { outDir, - statusFile: path.join(outDir, "recording-status.txt"), finalRecording: path.join(outDir, "smoke.mp4"), } } -function recorderEnv(bin, outDir, providerMarker) { +function recorderEnv(bin, outDir, providerMarker, extraEnv) { return { PATH: `${bin}:${process.env.PATH}`, DISPLAY: ":99", SOLHEIM_SMOKE_OUT_DIR: outDir, ...(providerMarker ? { PROVIDER_MARKER: providerMarker } : {}), + ...extraEnv, } } +const stopBudgetEnv = (stopBudget) => (stopBudget ? { SOLHEIM_SMOKE_STOP_BUDGET_S: String(stopBudget) } : {}) + const recorderScript = new URL("./solheim-smoke/record.sh", import.meta.url).pathname -function runRecorder(args, { ffmpeg, ffmpegWrites = true, ffprobeOk = true, seedFinal, providerMarker } = {}) { +function runRecorder( + args, + { ffmpeg, ffmpegWrites = true, ffprobeOk = true, seedFinal, providerMarker, stopBudget } = {}, +) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) const bin = stubRecorderBin(dir, { ffmpeg, ffmpegWrites, ffprobeOk }) const paths = recorderPaths(dir) fs.mkdirSync(paths.outDir, { recursive: true }) if (seedFinal !== undefined) fs.writeFileSync(paths.finalRecording, seedFinal) const result = spawnSync("bash", [recorderScript, ...args], { - env: recorderEnv(bin, paths.outDir, providerMarker), + env: recorderEnv(bin, paths.outDir, providerMarker, stopBudgetEnv(stopBudget)), encoding: "utf8", timeout: 20_000, }) return { result, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) } } -function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, providerMarker } = {}) { +function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, providerMarker, stopBudget } = {}) { return new Promise((resolve) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) const bin = stubRecorderBin(dir, { ffmpeg, ffprobeOk }) const paths = recorderPaths(dir) + // The output directory is created here, before the recorder runs, so + // every process in the test observes the same directory object. + fs.mkdirSync(paths.outDir, { recursive: true }) const child = spawn("bash", [recorderScript, ...args], { - env: recorderEnv(bin, paths.outDir, providerMarker), + env: recorderEnv(bin, paths.outDir, providerMarker, stopBudgetEnv(stopBudget)), }) let stdout = "" child.stdout.on("data", (chunk) => { @@ -248,14 +268,21 @@ function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, provide } const tempFilesLeft = (run) => fs.readdirSync(run.outDir).filter((name) => name.includes(".tmp")) -const recordingStatus = (run) => fs.readFileSync(run.statusFile, "utf8").trim() + +const pidAlive = (pid) => { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} describe("record.sh exit and validation behavior", () => { it("passes when the command succeeds and the recording is valid", () => { const run = runRecorder(["true"]) try { assert.equal(run.result.status, 0) - assert.equal(recordingStatus(run), "ok") assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") assert.deepEqual(tempFilesLeft(run), []) } finally { @@ -266,7 +293,6 @@ describe("record.sh exit and validation behavior", () => { const run = runRecorder(["bash", "-c", "exit 7"]) try { assert.equal(run.result.status, 7) - assert.equal(recordingStatus(run), "ok") assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") } finally { run.cleanup() @@ -278,7 +304,6 @@ describe("record.sh exit and validation behavior", () => { assert.equal(run.result.status, 1) assert.match(run.result.stdout, /recording failed/) assert.ok(!fs.existsSync(run.finalRecording)) - assert.notEqual(recordingStatus(run), "ok") } finally { run.cleanup() } @@ -289,7 +314,6 @@ describe("record.sh exit and validation behavior", () => { assert.equal(run.result.status, 1) assert.ok(!fs.existsSync(run.finalRecording)) assert.deepEqual(tempFilesLeft(run), []) - assert.notEqual(recordingStatus(run), "ok") } finally { run.cleanup() } @@ -331,7 +355,7 @@ describe("record.sh exit and validation behavior", () => { assert.equal(run.result.status, 1) assert.ok(!fs.existsSync(marker)) assert.ok(!fs.existsSync(run.finalRecording)) - assert.match(recordingStatus(run), /stopped on its own/) + assert.match(run.result.stdout, /stopped on its own/) } finally { run.cleanup() fs.rmSync(marker, { force: true }) @@ -347,7 +371,7 @@ describe("record.sh exit and validation behavior", () => { assert.equal(run.result.status, 1) assert.ok(!fs.existsSync(run.finalRecording)) assert.deepEqual(tempFilesLeft(run), []) - assert.match(recordingStatus(run), /stopped on its own/) + assert.match(run.result.stdout, /stopped on its own/) } finally { run.cleanup() } @@ -358,7 +382,6 @@ describe("record.sh exit and validation behavior", () => { }) try { assert.equal(run.result.status, 0) - assert.equal(recordingStatus(run), "ok") assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") } finally { run.cleanup() @@ -383,7 +406,6 @@ describe("record.sh signal teardown behavior", () => { try { assert.equal(run.code, 143) assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") - assert.equal(recordingStatus(run), "ok") assert.equal(fs.readFileSync(marker, "utf8"), "killed") assert.deepEqual(tempFilesLeft(run), []) } finally { @@ -397,7 +419,98 @@ describe("record.sh signal teardown behavior", () => { assert.equal(run.code, 143) assert.ok(!fs.existsSync(run.finalRecording)) assert.deepEqual(tempFilesLeft(run), []) - assert.notEqual(recordingStatus(run), "ok") + } finally { + run.cleanup() + } + }) +}) + +describe("record.sh provider stop behavior", () => { + it("lets a driver-style stub finish teardown and atomic artifact writes on SIGTERM", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir) + const paths = recorderPaths(dir) + // The artifact lives beside the stub, not inside the recorder output + // directory, so the assertion only observes the driver-style write. + const artifact = path.join(dir, "verdict.json") + const stub = path.join(dir, "driver-stub.mjs") + fs.writeFileSync( + stub, + [ + `import fs from "node:fs"`, + `const artifact = process.argv[2]`, + `process.on("SIGTERM", () => {`, + ` // Simulate driver teardown before the atomic artifact write.`, + ` setTimeout(() => {`, + ` fs.writeFileSync(artifact + ".tmp", "complete")`, + ` fs.renameSync(artifact + ".tmp", artifact)`, + ` process.exit(143)`, + ` }, 1500)`, + `})`, + `setInterval(() => {}, 1000)`, + ].join("\n"), + ) + const run = await runRecorderSignalled(["node", stub, artifact], { delayMs: 800, stopBudget: 10 }) + try { + assert.equal(run.code, 143) + // The wrapper waited for the driver teardown instead of killing the + // driver mid-write, so the atomic artifact write completed. + assert.equal(fs.readFileSync(artifact, "utf8"), "complete") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + it("escalates to KILL and reaps a hung driver within the stop budget", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir) + const paths = recorderPaths(dir) + const hungPidFile = path.join(dir, "hung-pid") + const termSeenFile = path.join(dir, "term-seen") + const hungSource = [ + `const fs = require("node:fs")`, + `process.on("SIGTERM", () => fs.writeFileSync(${JSON.stringify(termSeenFile)}, "seen"))`, + `fs.writeFileSync(${JSON.stringify(hungPidFile)}, String(process.pid))`, + `setInterval(() => {}, 1000)`, + ].join("\n") + const startedAt = Date.now() + // The TERM lands well after the child booted, so the ignored TERM is a + // handled signal, not a boot-time default kill. + const run = await runRecorderSignalled(["node", "-e", hungSource], { delayMs: 900, stopBudget: 2 }) + try { + assert.equal(run.code, 143) + // TERM reached the real driver child, the wrapper waited the + // documented budget, then escalated. Teardown stays bounded. + assert.equal(fs.readFileSync(termSeenFile, "utf8"), "seen") + assert.ok(Date.now() - startedAt < 10_000) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + const hungPid = Number(fs.readFileSync(hungPidFile, "utf8").trim()) + assert.ok(Number.isInteger(hungPid) && hungPid > 0) + let gone = false + for (let attempt = 0; attempt < 50 && !gone; attempt++) { + gone = !pidAlive(hungPid) + if (!gone) await new Promise((resolve) => setTimeout(resolve, 100)) + } + assert.ok(gone, "hung driver survived the KILL escalation") + } finally { + run.cleanup() + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + it("runs the real Node driver with no arguments and keeps metadata on failure", () => { + // The credential-free driver fails fast with a SETUP_FAILED verdict, so + // this exercises the exact CI spawn path end to end. + const run = runRecorder([]) + try { + assert.equal(run.result.status, 1) + const verdict = JSON.parse(fs.readFileSync(path.join(run.outDir, "verdict.json"), "utf8")) + assert.equal(verdict.code, "SETUP_FAILED") + const trace = JSON.parse(fs.readFileSync(path.join(run.outDir, "trace.json"), "utf8")) + assert.equal(trace.execution.code, "SETUP_FAILED") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.deepEqual(tempFilesLeft(run), []) } finally { run.cleanup() } diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index ed63972416..af36513eeb 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -64,14 +64,22 @@ a stale `smoke.mp4` from an earlier attempt in a reused directory is removed bef ffmpeg exits non-zero after its graceful INT or TERM shutdown, so a recorder that was still alive at shutdown is judged by content validation alone. A recorder that stopped on its own before shutdown must have exited zero (the `-t` time or `-fs` size limits); any other self-chosen status fails the recording -even when the fragments would still probe. The wrapper records the outcome in `recording-status.txt` -(`ok`, or the failure reason), reports the reason on stdout, and keeps the provider exit code unchanged. - -Metadata (`verdict.json`, `trace.json`, `recording-status.txt`) and the video are uploaded as two separate -artifacts, each with `if-no-files-found: error`. A missing recording therefore fails its own upload step -and the explicit integrity check step, instead of hiding behind a multi-path upload. Verdict and trace -evidence stay available when the provider failed and the video is missing. Cancellation may prevent -finalization and upload. +even when the fragments would still probe. The wrapper reports failure reasons on stdout and keeps the +provider exit code unchanged. There is no separate persisted status marker: a published, non-empty +`smoke.mp4` is the only success marker, and the workflow checks exactly that. + +The wrapper starts the Node driver directly, not through a package-manager wrapper, so its stop routine +signals and monitors the driver PID that owns teardown and the atomic verdict and trace writes. It sends +`TERM`, waits a budget, then escalates to `KILL` for that PID only. The budget defaults to 30 seconds and +covers the driver teardown worst case: three 5 second host termination windows plus cleanup and writes. +`SOLHEIM_SMOKE_STOP_BUDGET_S` overrides the budget for tests. A `KILL`ed driver can leave host processes +behind; no wrapper cleanup can repair that boundary. + +Metadata (`verdict.json`, `trace.json`) and the video are uploaded as two separate artifacts, each with +`if-no-files-found: error`. A missing recording therefore fails its own upload step and the explicit +integrity check step, instead of hiding behind a multi-path upload. Verdict and trace evidence stay +available when the provider failed and the video is missing. Cancellation may prevent finalization and +upload. Checks, without a provider key or network calls: diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh index b3c057d839..e15f3c38e8 100644 --- a/scripts/solheim-smoke/record.sh +++ b/scripts/solheim-smoke/record.sh @@ -5,12 +5,11 @@ set -euo pipefail : "${SOLHEIM_SMOKE_OUT_DIR:?Set a scratch output directory}" mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" final_recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" -status_file="$SOLHEIM_SMOKE_OUT_DIR/recording-status.txt" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # A stale final from an earlier attempt must never survive into this attempt's # upload. Remove only this exact path. rm -f "$final_recording" -echo pending > "$status_file" # The recorder needs display access, not provider/GitHub credentials. Fragmented # MP4 keeps completed fragments readable if the action is forcibly interrupted. @@ -35,14 +34,22 @@ provider_pid="" # ffprobe can still read the fragments. report_recording_failure() { echo "Solheim smoke recording failed: $1" - echo "$1" > "$status_file" [[ "$result" -ne 0 ]] || result=1 } +# The provider child is the Node driver itself, so TERM reaches the one process +# that owns teardown and the atomic verdict and trace writes. The 30 second +# budget covers the driver teardown worst case: three 5 second host termination +# windows plus cleanup and artifact writes. KILL is the last resort for a hung +# driver; a killed driver can leave host processes behind, and no wrapper +# cleanup can repair that boundary. +provider_stop_budget="${SOLHEIM_SMOKE_STOP_BUDGET_S:-30}" +[[ "$provider_stop_budget" =~ ^[0-9]+$ ]] || provider_stop_budget=30 + stop_provider() { [[ -n "$provider_pid" ]] || return 0 kill -TERM "$provider_pid" 2>/dev/null || return 0 - for ((attempt = 0; attempt < 30; attempt++)); do + for ((attempt = 0; attempt < provider_stop_budget * 10; attempt++)); do kill -0 "$provider_pid" 2>/dev/null || break sleep 0.1 done @@ -81,7 +88,8 @@ finish() { report_recording_failure "recorder output failed ffprobe validation" elif mv -f "$temp_recording" "$final_recording"; then # Same-directory rename: atomic, so no partially written final exists. - echo ok > "$status_file" + # The published, nonempty final is the only success marker. + : else rm -f "$temp_recording" report_recording_failure "validated recording could not be published" @@ -99,7 +107,10 @@ kill -0 "$recorder_pid" 2>/dev/null || exit 1 if [[ "$#" -gt 0 ]]; then "$@" & else - pnpm solheim:smoke & + # Invoke the Node driver directly: a package-manager wrapper would sit + # between this wrapper and the driver, and stop_provider must signal and + # monitor the driver PID that performs teardown and artifact writes. + node --import tsx "$script_dir/driver.mts" & fi provider_pid=$! provider_status=0 From 4c73905a17d61c6c43dfc374107d980630a9a14c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 03:04:44 +0000 Subject: [PATCH 08/11] test(smoke): pass hung-driver paths as argv, not generated source The KILL-escalation test now writes a static driver stub and passes both scratch paths as command arguments, read via process.argv. Quote and backslash characters in the paths guard the argv handoff. No runtime or workflow changes. --- scripts/solheim-smoke-workflow.test.mjs | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index b2614626d4..27e10f727c 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -467,18 +467,27 @@ describe("record.sh provider stop behavior", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) const bin = stubRecorderBin(dir) const paths = recorderPaths(dir) - const hungPidFile = path.join(dir, "hung-pid") - const termSeenFile = path.join(dir, "term-seen") - const hungSource = [ - `const fs = require("node:fs")`, - `process.on("SIGTERM", () => fs.writeFileSync(${JSON.stringify(termSeenFile)}, "seen"))`, - `fs.writeFileSync(${JSON.stringify(hungPidFile)}, String(process.pid))`, - `setInterval(() => {}, 1000)`, - ].join("\n") + // The quote and backslash prove the paths reach the driver as argv, + // never interpolated into the driver source. + const hungPidFile = path.join(dir, "hung\\pid") + const termSeenFile = path.join(dir, 'term"seen') + const hungStub = path.join(dir, "hung-driver.mjs") + fs.writeFileSync( + hungStub, + [ + `import fs from "node:fs"`, + `process.on("SIGTERM", () => fs.writeFileSync(process.argv[2], "seen"))`, + `fs.writeFileSync(process.argv[3], String(process.pid))`, + `setInterval(() => {}, 1000)`, + ].join("\n"), + ) const startedAt = Date.now() // The TERM lands well after the child booted, so the ignored TERM is a // handled signal, not a boot-time default kill. - const run = await runRecorderSignalled(["node", "-e", hungSource], { delayMs: 900, stopBudget: 2 }) + const run = await runRecorderSignalled(["node", hungStub, termSeenFile, hungPidFile], { + delayMs: 900, + stopBudget: 2, + }) try { assert.equal(run.code, 143) // TERM reached the real driver child, the wrapper waited the From 8ef7b37054e806609b7824557c3fb0193db67617 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 19 Oct 2026 22:21:51 +0000 Subject: [PATCH 09/11] fix(ci): document smoke approval and test the main-only secret guard --- .github/workflows/solheim-provider-smoke.yml | 5 ++++- scripts/solheim-smoke-workflow.test.mjs | 22 ++++++++++++++++++++ scripts/solheim-smoke/README.md | 7 +++++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml index bf21463897..7e0298818c 100644 --- a/.github/workflows/solheim-provider-smoke.yml +++ b/.github/workflows/solheim-provider-smoke.yml @@ -18,7 +18,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 25 # Retain the existing protected environment and secret names for compatibility. - # Repository policy must restrict deployments to main and require approval. + # The environment requires reviewer approval, matching the deploy + # environment. The job guard above restricts runs to main, so the + # secret is released only for an approved main run. See + # scripts/solheim-smoke/README.md. environment: final-vscode-review-smoke steps: - name: Checkout exact trusted main revision diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 27e10f727c..4aa25e8659 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -32,6 +32,28 @@ describe("Project A provider smoke workflow", () => { assert.match(workflow, /persist-credentials: false/) assert.match(workflow, /environment: final-vscode-review-smoke/) }) + it("gates the provider secret behind an environment-bound, main-only smoke job", () => { + const jobs = workflow.split("jobs:\n")[1] ?? "" + const smokeJob = jobs.split("\n steps:")[0] ?? "" + assert.ok(smokeJob.length > 0) + // The job runs only on main, so no other ref reaches the secret step. + assert.match(smokeJob, /^ {8}if: github\.ref == 'refs\/heads\/main'$/m) + // The secret-holding environment gates the whole job, so GitHub + // applies its protection rules before the first step runs. + assert.match(smokeJob, /^ {8}environment: final-vscode-review-smoke$/m) + // The environment reference must precede the credential-bearing step. + assert.ok(smokeJob.indexOf("environment: final-vscode-review-smoke") < jobs.indexOf("secrets.")) + // The workflow must consume the secret scoped to that environment. + assert.match(workflow, /secrets\.FINAL_SMOKE_OPENAI_API_KEY/) + }) + it("documents the required environment protection policy for the secret", () => { + const readme = fs.readFileSync(new URL("./solheim-smoke/README.md", import.meta.url), "utf8") + // The environment requires the same reviewer approval as the deploy + // environment, and the workflow keeps the main-only deployment guard. + assert.match(readme, /same reviewer approval as the `marketplace-production` deploy environment/) + assert.match(readme, /must approve each deployment before GitHub releases the secret/) + assert.match(readme, /runs only on `refs\/heads\/main`/) + }) it("has one credential-bearing step, no GitHub write permissions, and no posting", () => { assert.equal((workflow.match(/secrets\./g) ?? []).length, 1) const before = workflow.slice(0, workflow.indexOf("- name: Run Solheim provider smoke")) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index af36513eeb..ce7395b444 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -18,8 +18,11 @@ and `contents: read` is the only GitHub permission. This deliberately validates on main, not arbitrary PR artifacts. It is not a required PR review check. The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOKE_OPENAI_API_KEY` -retain their existing names to avoid an unnecessary repository-settings migration. Restrict its deployment -policy to main and require approval. No environment or secret settings have been changed by this cleanup. +retain their existing names to avoid an unnecessary repository-settings migration. The environment +requires the same reviewer approval as the `marketplace-production` deploy environment: a member of +the `Zoo-Code-Org/admins` team must approve each deployment before GitHub releases the secret. The +main-only deployment restriction stays in the workflow: the smoke job runs only on `refs/heads/main`. +Workflow files cannot declare reviewer rules; repository settings own them. The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary From 9dc5c7c1c03212b5ce0e990256401649f70b4591 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 20 Oct 2026 00:31:41 +0000 Subject: [PATCH 10/11] test(ci): replace smoke workflow source assertions with parsed YAML checks --- package.json | 3 +- pnpm-lock.yaml | 3 + scripts/solheim-smoke-workflow.test.mjs | 253 ++++++------------------ scripts/solheim-smoke/README.md | 5 +- 4 files changed, 71 insertions(+), 193 deletions(-) diff --git a/package.json b/package.json index 6bb220713f..874ff020a2 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,8 @@ "rimraf": "6.0.1", "tsx": "4.23.15", "turbo": "2.11.8", - "typescript": "5.9.3" + "typescript": "5.9.3", + "yaml": "2.9.0" }, "lint-staged": { "src/**/*.ts": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 58d45a98b2..0d89651c61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,6 +89,9 @@ importers: typescript: specifier: 5.9.3 version: 5.9.3 + yaml: + specifier: 2.9.0 + version: 2.9.0 apps/cli: dependencies: diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index 4aa25e8659..ba9e04424a 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -5,207 +5,80 @@ import { describe, it } from "node:test" import { spawn, spawnSync } from "node:child_process" import os from "node:os" import path from "node:path" -const workflow = fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8") -const codeQa = fs.readFileSync(new URL("../.github/workflows/code-qa.yml", import.meta.url), "utf8") -const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8")) -const driver = fs.readFileSync(new URL("./solheim-smoke/driver.mts", import.meta.url), "utf8") -const recorder = fs.readFileSync(new URL("./solheim-smoke/record.sh", import.meta.url), "utf8") -const smokePkg = JSON.parse(fs.readFileSync(new URL("./solheim-smoke/package.json", import.meta.url), "utf8")) -const runner = fs.readFileSync(new URL("./agent-evals/runner.ts", import.meta.url), "utf8") -const launcher = fs.readFileSync(new URL("./agent-evals/launcher.ts", import.meta.url), "utf8") -const registry = fs.readFileSync(new URL("./agent-evals/registry.ts", import.meta.url), "utf8") -const fixture = fs.readFileSync(new URL("./solheim-smoke/fixture.ts", import.meta.url), "utf8") -const trace = fs.readFileSync(new URL("./agent-evals/trace.ts", import.meta.url), "utf8") -const evalsPkg = JSON.parse(fs.readFileSync(new URL("./agent-evals/package.json", import.meta.url), "utf8")) +import { parse } from "yaml" -describe("Project A provider smoke workflow", () => { - it("runs smoke TypeScript as ESM without requiring compiled workspace packages", () => { - assert.equal(smokePkg.type, "module") - assert.equal(smokePkg.private, true) - }) - it("runs only by manual dispatch against the exact trusted main revision", () => { - assert.match(workflow, /workflow_dispatch:/) - assert.match(workflow, /if: github.ref == 'refs\/heads\/main'/) - assert.match(workflow, /ref: \$\{\{ github.sha \}\}/) - assert.ok(!workflow.includes("pull_request_review:")) - assert.ok(!workflow.includes("pull_request_target:")) - assert.match(workflow, /persist-credentials: false/) - assert.match(workflow, /environment: final-vscode-review-smoke/) - }) - it("gates the provider secret behind an environment-bound, main-only smoke job", () => { - const jobs = workflow.split("jobs:\n")[1] ?? "" - const smokeJob = jobs.split("\n steps:")[0] ?? "" - assert.ok(smokeJob.length > 0) - // The job runs only on main, so no other ref reaches the secret step. - assert.match(smokeJob, /^ {8}if: github\.ref == 'refs\/heads\/main'$/m) - // The secret-holding environment gates the whole job, so GitHub - // applies its protection rules before the first step runs. - assert.match(smokeJob, /^ {8}environment: final-vscode-review-smoke$/m) - // The environment reference must precede the credential-bearing step. - assert.ok(smokeJob.indexOf("environment: final-vscode-review-smoke") < jobs.indexOf("secrets.")) - // The workflow must consume the secret scoped to that environment. - assert.match(workflow, /secrets\.FINAL_SMOKE_OPENAI_API_KEY/) - }) - it("documents the required environment protection policy for the secret", () => { - const readme = fs.readFileSync(new URL("./solheim-smoke/README.md", import.meta.url), "utf8") - // The environment requires the same reviewer approval as the deploy - // environment, and the workflow keeps the main-only deployment guard. - assert.match(readme, /same reviewer approval as the `marketplace-production` deploy environment/) - assert.match(readme, /must approve each deployment before GitHub releases the secret/) - assert.match(readme, /runs only on `refs\/heads\/main`/) +// Parse configuration so comments, indentation, and key order are irrelevant. +// GitHub expressions remain strings: these checks protect the configured policy, +// while the recorder tests below exercise process behavior. +const workflow = parse( + fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8"), +) +const smoke = workflow.jobs.smoke +const steps = smoke.steps +const actionSteps = (action) => steps.filter((step) => step.uses?.startsWith(action + "@")) + +describe("Solheim provider smoke workflow policy", () => { + it("allows only manual runs on main behind the protected environment", () => { + assert.deepEqual(Object.keys(workflow.on), ["workflow_dispatch"]) + assert.deepEqual(Object.keys(workflow.jobs), ["smoke"]) + assert.equal(smoke.if, "github.ref == 'refs/heads/main'") + assert.equal(smoke.environment, "final-vscode-review-smoke") + assert.deepEqual(workflow.permissions, { contents: "read" }) + assert.equal(smoke.permissions, undefined) }) - it("has one credential-bearing step, no GitHub write permissions, and no posting", () => { - assert.equal((workflow.match(/secrets\./g) ?? []).length, 1) - const before = workflow.slice(0, workflow.indexOf("- name: Run Solheim provider smoke")) - assert.ok(!before.includes("secrets.")) - assert.match(workflow, /permissions:\n\s+contents: read/) - assert.ok(!/\bwrite\b/.test(workflow)) - assert.ok(!workflow.includes("github-script")) - assert.ok(!workflow.includes("post-review")) + + it("checks out the triggering revision without persisting credentials", () => { + const checkouts = actionSteps("actions/checkout") + assert.equal(checkouts.length, 1) + assert.equal(checkouts[0].with.ref, "${{ github.sha }}") + assert.equal(checkouts[0].with["persist-credentials"], false) }) - it("has one smoke job and no review, routing, shadow, or deterministic lanes", () => { - const jobs = workflow.split("jobs:\n")[1] ?? "" - assert.deepEqual( - [...jobs.matchAll(/^ {4}([a-z-]+):$/gm)].map((match) => match[1]), - ["smoke"], + + it("exposes the provider credential only to the bounded provider step", () => { + const provider = steps.find((step) => step.id === "provider") + assert.ok(provider, "missing provider step") + assert.equal(provider.env.SOLHEIM_API_KEY, "${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }}") + assert.equal(workflow.env, undefined) + assert.equal(smoke.env, undefined) + // Inspect parsed values, excluding comments and display names. + const credentialSteps = steps.filter((step) => + JSON.stringify({ env: step.env, with: step.with, run: step.run }).includes("secrets."), ) - for (const name of [ - "theme-review", - "probe-dispatch", - "rules-review", - "merge-report", - "bounded-code-review", - "model-qualification", - ]) - assert.ok(!workflow.includes(name)) - assert.ok(!driver.includes("model-qualification")) - assert.ok(!driver.includes("AcknowledgeCompletion")) - }) - it("bounds runtime and serializes the single provider instance", () => { - assert.match(workflow, /cancel-in-progress: false/) - assert.match(workflow, /timeout-minutes: 8/) - assert.match(workflow, /^ {8}timeout-minutes: 25$/m) + assert.deepEqual(credentialSteps, [provider]) + assert.ok(provider["timeout-minutes"] > 0 && provider["timeout-minutes"] <= 8) + assert.ok(smoke["timeout-minutes"] > provider["timeout-minutes"] && smoke["timeout-minutes"] <= 25) + assert.equal(workflow.concurrency.group, "solheim-provider-smoke") + assert.equal(workflow.concurrency["cancel-in-progress"], false) }) - it("uploads metadata and the recording as separate validated artifacts", () => { - const metadataStep = - workflow.split("- name: Upload provider smoke verdict and trace")[1]?.split("- name:")[0] ?? "" - const recordingStep = workflow.split("- name: Upload provider smoke recording")[1]?.split("- name:")[0] ?? "" - assert.ok(metadataStep.length > 0) - assert.ok(recordingStep.length > 0) - assert.deepEqual( - metadataStep - .split("path: |")[1] - ?.split("if-no-files-found:")[0] + + it("uploads metadata and the final recording separately, including after provider failure", () => { + const uploads = actionSteps("actions/upload-artifact") + assert.equal(uploads.length, 2) + const output = "${{ runner.temp }}/solheim-provider-smoke" + const paths = uploads.map((step) => + step.with.path .trim() .split("\n") .map((line) => line.trim()), - [ - "${{ runner.temp }}/solheim-provider-smoke/verdict.json", - "${{ runner.temp }}/solheim-provider-smoke/trace.json", - ], ) - // The recording artifact carries only the exact validated final path. - assert.equal( - recordingStep.split("path: ")[1]?.split("\n")[0].trim(), - "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4", - ) - for (const step of [metadataStep, recordingStep]) { - assert.match(step, /!cancelled\(\)/) - assert.match(step, /if-no-files-found: error/) - assert.match(step, /retention-days: 7/) - assert.match(step, /actions\/upload-artifact@[0-9a-f]{40}/) - assert.ok(!step.includes("*")) + assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]]) + for (const step of uploads) { + assert.equal( + step.if, + "${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}", + ) + assert.equal(step.with["if-no-files-found"], "error") + assert.equal(step.with["retention-days"], 7) + assert.ok(steps.indexOf(step) > steps.findIndex((candidate) => candidate.id === "provider")) } - // A missing recording cannot hide behind a multi-path if-no-files-found. - assert.match(workflow, /name: solheim-provider-smoke-recording-\$\{\{ github\.sha \}\}/) - // Temporary recorder output is never an upload path. - assert.ok(!workflow.includes(".tmp")) - assert.ok(!driver.includes("console.error(")) - assert.ok(!driver.includes("stderrLog")) - }) - it("reports a missing or failed recording explicitly before the uploads", () => { - const checkStep = workflow.split("- name: Verify recording integrity")[1]?.split("- name:")[0] ?? "" - // The published final is the commit marker; no separate status file exists. - assert.match(checkStep, /test -s "\$SOLHEIM_SMOKE_OUT_DIR\/smoke\.mp4"/) - assert.ok(!workflow.includes("recording-status")) - assert.match(checkStep, /!cancelled\(\)/) - const checkIndex = workflow.indexOf("- name: Verify recording integrity") - const uploadIndex = workflow.indexOf("- name: Upload provider smoke verdict and trace") - assert.ok(checkIndex >= 0 && checkIndex < uploadIndex) - }) - it("records a fresh display with credential-free ffmpeg and bounded teardown", () => { - assert.match(workflow, /install -y xvfb ffmpeg/) - assert.match(workflow, /xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp"/) - assert.match(recorder, /env -i "PATH=\$PATH" "DISPLAY=\$DISPLAY"/) - assert.match(recorder, /-t 480 -fs 125829120/) - assert.match(recorder, /frag_keyframe\+empty_moov/) - assert.match(recorder, /trap finish EXIT/) - assert.match(recorder, /attempt < 50/) - assert.ok(!recorder.includes("SOLHEIM_API_KEY")) - assert.match(workflow, /retention-days: 7/) - // The recorder writes a unique temporary file inside the output - // directory, forces the mp4 muxer because the name has no mp4 suffix, - // and publishes only validated output with an atomic rename. - assert.match(recorder, /temp_recording="\$SOLHEIM_SMOKE_OUT_DIR\/\.smoke\.mp4\.\$\$\.tmp"/) - assert.match(recorder, /-f mp4 "\$temp_recording"/) - assert.match(recorder, /ffprobe -v error "\$temp_recording"/) - assert.match(recorder, /mv -f "\$temp_recording" "\$final_recording"/) - assert.match(recorder, /rm -f "\$final_recording"/) - // ffmpeg must never write the final path directly. - assert.ok(!/ffmpeg[^\n]*\$final_recording/.test(recorder)) - }) - it("signals the Node driver directly with a documented stop budget", () => { - // A package-manager wrapper between the recorder and the driver would - // receive the TERM and KILL meant for the driver that performs teardown - // and the atomic verdict and trace writes. - assert.ok(!recorder.includes("pnpm")) - assert.match(recorder, /node --import tsx "\$script_dir\/driver\.mts" &/) - assert.match(recorder, /SOLHEIM_SMOKE_STOP_BUDGET_S:-30/) - assert.match(recorder, /kill -TERM "\$provider_pid"/) - assert.match(recorder, /kill -KILL "\$provider_pid"/) - assert.match(recorder, /wait "\$provider_pid"/) - assert.ok(!recorder.includes("status_file")) - }) - it("keeps Code QA and package commands focused on smoke contracts", () => { - for (const command of [ - "test:solheim-smoke:unit", - "test:solheim-smoke-ci", - "solheim-smoke:check-types", - "test:agent-evals:unit", - "agent-evals:check-types", - ]) - assert.ok(codeQa.includes(command)) - assert.ok(!codeQa.includes("model-qualification")) - assert.ok(!Object.keys(pkg.scripts).some((name) => name.includes("model-qualification"))) - assert.ok(!codeQa.includes("final-smoke")) - assert.equal(pkg.scripts["solheim:smoke"], "node --import tsx scripts/solheim-smoke/driver.mts") - assert.equal(pkg.scripts["test:agent-evals:unit"], "node --import tsx --test scripts/agent-evals/*.test.ts") - assert.equal(pkg.scripts["agent-evals:check-types"], "tsc -p scripts/agent-evals/tsconfig.json") - assert.ok(!fs.existsSync(new URL("../.github/workflows/final-vscode-review-smoke.yml", import.meta.url))) + assert.equal(new Set(uploads.map((step) => step.with.name)).size, 2) }) - it("keeps the shared controller free of smoke, provider, and solution knowledge", () => { - assert.equal(evalsPkg.type, "module") - assert.equal(evalsPkg.private, true) - assert.ok(!/solheim/i.test(runner)) - assert.ok(!/smoke/i.test(runner)) - assert.ok(!runner.includes("SOLHEIM_API_KEY")) - assert.ok(!/solheim/i.test(trace)) - assert.ok(trace.includes("traceSchemaVersion")) - }) - it("registers fixtures only through the validated in-memory registry", () => { - assert.ok(!registry.includes("import(")) - assert.ok(!registry.includes("fetch(")) - assert.ok(!registry.includes("readdir")) - assert.ok(fixture.includes("registerFixture")) - }) - it("keeps VS Code launch isolation in one shared launcher with an atomic artifact writer", () => { - assert.ok(driver.includes("createHostRuntime")) - assert.ok(!driver.includes("spawn(")) - assert.ok(!driver.includes("downloadAndUnzipVSCode")) - assert.ok(launcher.includes("downloadAndUnzipVSCode")) - assert.ok(launcher.includes("buildChildEnvironment")) - assert.ok(launcher.includes("await fs.rename(pending")) - assert.ok(launcher.includes("process.kill(-child.pid")) + + it("pins remote actions to immutable commit revisions", () => { + for (const step of steps.filter((step) => step.uses && !step.uses.startsWith("./"))) { + const revision = step.uses.split("@")[1] + assert.match(revision ?? "", /^[0-9a-f]{40}$/, step.uses) + } }) }) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index ce7395b444..f215f11845 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -21,8 +21,9 @@ The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOK retain their existing names to avoid an unnecessary repository-settings migration. The environment requires the same reviewer approval as the `marketplace-production` deploy environment: a member of the `Zoo-Code-Org/admins` team must approve each deployment before GitHub releases the secret. The -main-only deployment restriction stays in the workflow: the smoke job runs only on `refs/heads/main`. -Workflow files cannot declare reviewer rules; repository settings own them. +environment also allows deployments from one selected branch, `main`, and no tag policies. The +workflow keeps its own guard: the smoke job runs only on `refs/heads/main`. Workflow files cannot +declare reviewer rules; repository settings own both protections. The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary From 1ccf5125466664d72a6a017c374d74936d18746c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 21 Oct 2026 03:43:01 +0000 Subject: [PATCH 11/11] fix(smoke): address review on recorder, runner, and fixture policy --- package.json | 2 +- pnpm-lock.yaml | 4 +-- scripts/agent-evals/contract.ts | 16 ++++++++++ scripts/agent-evals/runner.test.ts | 30 ++++++++++++++++++- scripts/agent-evals/runner.ts | 16 ++++++++-- scripts/solheim-smoke-workflow.test.mjs | 40 ++++++++++++++++++++++--- scripts/solheim-smoke/README.md | 2 +- scripts/solheim-smoke/record.sh | 12 ++++++-- 8 files changed, 107 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 874ff020a2..cff60a205b 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "tsx": "4.23.15", "turbo": "2.11.8", "typescript": "5.9.3", - "yaml": "2.9.0" + "yaml": "2.9.1" }, "lint-staged": { "src/**/*.ts": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d89651c61..b81b295307 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -90,8 +90,8 @@ importers: specifier: 5.9.3 version: 5.9.3 yaml: - specifier: 2.9.0 - version: 2.9.0 + specifier: 2.9.1 + version: 2.9.1 apps/cli: dependencies: diff --git a/scripts/agent-evals/contract.ts b/scripts/agent-evals/contract.ts index b2e3a409d0..2e6c0ba30b 100644 --- a/scripts/agent-evals/contract.ts +++ b/scripts/agent-evals/contract.ts @@ -126,6 +126,22 @@ const serializeProjection = (projection: Record): string | unde } } +// The built configuration is what the extension runs. It must carry exactly the +// declared tool policy, so a drifted builder cannot widen tool access. +export function configurationMatchesToolPolicy( + config: Record, + policy: AgentFixture["toolPolicy"], +): boolean { + const sameList = (a: unknown, b: readonly string[]) => + Array.isArray(a) && a.length === b.length && b.every((item) => a.includes(item)) + if (config["mode"] !== policy.mode || !sameList(config["disabledTools"], policy.disabledTools)) return false + const modes = config["customModes"] + const custom = Array.isArray(modes) ? modes.find((m) => isPlainObject(m) && m["slug"] === policy.mode) : undefined + if (!isPlainObject(custom) || custom["roleDefinition"] !== policy.roleDefinition) return false + if (!sameList(custom["groups"], policy.groups)) return false + return ALLOW_KEYS.every((key) => config[`alwaysAllow${key[0]!.toUpperCase()}${key.slice(1)}`] === policy.allow[key]) +} + export function validateFixture(fixture: AgentFixture): void { if (!isPlainObject(fixture)) fail("fixture must be an object") if (!isPlainObject(fixture.identity)) fail("fixture identity missing") diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index 25dd7e5bca..70721f9075 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -41,7 +41,23 @@ function registerTestFixture(name: string, overrides: Partial = {} teardownStepMs: 5_000, }, settingsProjection: { provider: "test", model: `model-${name}` }, - buildConfiguration: () => ({ apiProvider: "openai", openAiApiKey: CREDENTIAL }), + buildConfiguration: () => ({ + apiProvider: "openai", + openAiApiKey: CREDENTIAL, + mode: "test", + customModes: [{ slug: "test", name: "Test", roleDefinition: "Run one synthetic task.", groups: [] }], + disabledTools: ["new_task"], + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + }), ...overrides, } registerFixture(fixture) @@ -466,6 +482,18 @@ describe("agent fixture controller", () => { assert.deepEqual(trace.settings, { available: false, reason: "credentialMissing" }) }) + it("rejects a configuration that widens the declared tool policy, and never launches", async () => { + const base = registerTestFixture("controller-policy-base") + registerTestFixture("controller-policy", { + buildConfiguration: (credential) => ({ ...base.buildConfiguration(credential), alwaysAllowExecute: true }), + }) + const f = harness("success") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-policy", credential: CREDENTIAL }) + assert.equal(result.execution.stopReason, "fixtureRejected") + assert.equal(f.launched, false) + assert.deepEqual(writtenTrace(f).settings, { available: false, reason: "configurationViolatesToolPolicy" }) + }) + it("rejects a fixture whose projection carries the credential, and never launches", async () => { registerTestFixture("controller-leak", { settingsProjection: { apiKey: CREDENTIAL } }) const f = harness("success") diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index 1ec6335ce2..5c12961db3 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto" import { RooCodeEventName, TaskCommandName, taskEventSchema, type TaskCommand, type TaskEvent } from "@roo-code/types" -import type { AgentFixtureWorkspaceFile } from "./contract.ts" +import { configurationMatchesToolPolicy, type AgentFixtureWorkspaceFile } from "./contract.ts" import { getFixture } from "./registry.ts" import { createTaskStartWaiter, type TaskStartWaiter } from "./task-start.ts" import { @@ -82,7 +82,11 @@ const traceSettings = (fixtureProjection: Record, credential: s if (projectionJson === undefined) return { available: false, reason: "projectionNotSerializable" } if (credential.length > 0 && projectionJson.includes(credential)) return { available: false, reason: "credentialInProjection" } - return { available: true, projection: fixtureProjection, digest: digestProjection(fixtureProjection) } + return { + available: true, + projection: JSON.parse(projectionJson) as Record, + digest: digestProjection(fixtureProjection), + } } catch { return { available: false, reason: "projectionNotSerializable" } } @@ -224,11 +228,16 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp settings = { available: false, reason: "credentialMissing" } } else { settings = traceSettings(fixture.settingsProjection, credential) + const configuration = fixture.buildConfiguration(credential) + if (settings.available && !configurationMatchesToolPolicy(configuration, fixture.toolPolicy)) + settings = { available: false, reason: "configurationViolatesToolPolicy" } if (!settings.available) { stopReason = "fixtureRejected" } else { host = await runtime.launch(fixture.workspace.files) host.onExit(markLost) + // An interrupt during launch must not connect or send a task. + if (sessionLost) throw new Error("startup interrupted") client = runtime.connect() client.onDisconnect(() => { sessionLost = true @@ -244,6 +253,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp code = "ACTIVATION_FAILED" stopReason = "activationFailed" } else { + if (sessionLost) throw new Error("startup interrupted") const requestId = randomUUID() waiter = createTaskStartWaiter({ requestId, timeoutMs: fixture.limits.taskStartTimeoutMs }) // A synchronous send failure must not leave a rejected timer promise. @@ -255,7 +265,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp requestId, text: fixture.prompt, newTab: false, - configuration: fixture.buildConfiguration(credential), + configuration, }, }) try { diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs index ba9e04424a..e14ec4a320 100644 --- a/scripts/solheim-smoke-workflow.test.mjs +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict" import fs from "node:fs" -import { URL } from "node:url" +import { URL, fileURLToPath } from "node:url" import { describe, it } from "node:test" import { spawn, spawnSync } from "node:child_process" import os from "node:os" @@ -118,19 +118,29 @@ function recorderEnv(bin, outDir, providerMarker, extraEnv) { const stopBudgetEnv = (stopBudget) => (stopBudget ? { SOLHEIM_SMOKE_STOP_BUDGET_S: String(stopBudget) } : {}) -const recorderScript = new URL("./solheim-smoke/record.sh", import.meta.url).pathname +const recorderScript = fileURLToPath(new URL("./solheim-smoke/record.sh", import.meta.url)) function runRecorder( args, - { ffmpeg, ffmpegWrites = true, ffprobeOk = true, seedFinal, providerMarker, stopBudget } = {}, + { + ffmpeg, + ffmpegWrites = true, + ffprobeOk = true, + seedFinal, + seedMetadata, + providerMarker, + stopBudget, + extraEnv, + } = {}, ) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) const bin = stubRecorderBin(dir, { ffmpeg, ffmpegWrites, ffprobeOk }) const paths = recorderPaths(dir) fs.mkdirSync(paths.outDir, { recursive: true }) if (seedFinal !== undefined) fs.writeFileSync(paths.finalRecording, seedFinal) + for (const name of seedMetadata ?? []) fs.writeFileSync(path.join(paths.outDir, name), "stale") const result = spawnSync("bash", [recorderScript, ...args], { - env: recorderEnv(bin, paths.outDir, providerMarker, stopBudgetEnv(stopBudget)), + env: recorderEnv(bin, paths.outDir, providerMarker, { ...stopBudgetEnv(stopBudget), ...extraEnv }), encoding: "utf8", timeout: 20_000, }) @@ -193,6 +203,28 @@ describe("record.sh exit and validation behavior", () => { run.cleanup() } }) + it("removes stale verdict and trace metadata at start", () => { + const run = runRecorder(["true"], { + seedMetadata: ["verdict.json", "trace.json", "verdict.json.1.tmp"], + }) + try { + for (const name of ["verdict.json", "trace.json", "verdict.json.1.tmp"]) + assert.equal(fs.existsSync(path.join(run.outDir, name)), false, name) + } finally { + run.cleanup() + } + }) + + it("refuses a supplied command while the provider credential is set", () => { + const run = runRecorder(["true"], { extraEnv: { SOLHEIM_API_KEY: "sk-test" } }) + try { + assert.equal(run.result.status, 2) + assert.match(run.result.stderr, /Refusing/) + } finally { + run.cleanup() + } + }) + it("fails when the recording is missing", () => { const run = runRecorder(["true"], { ffmpegWrites: false }) try { diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md index f215f11845..30b331ae90 100644 --- a/scripts/solheim-smoke/README.md +++ b/scripts/solheim-smoke/README.md @@ -13,7 +13,7 @@ about Solheim, the prompt, or the expected completion. The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to `refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the -single credential-bearing step. There is one job, no PR input or approval gate, no review/posting job, +single credential-bearing step. There is one job, no PR input, no review/posting job, and `contents: read` is the only GitHub permission. This deliberately validates the provider integration on main, not arbitrary PR artifacts. It is not a required PR review check. diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh index e15f3c38e8..436c57c0b8 100644 --- a/scripts/solheim-smoke/record.sh +++ b/scripts/solheim-smoke/record.sh @@ -7,9 +7,10 @@ mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" final_recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# A stale final from an earlier attempt must never survive into this attempt's -# upload. Remove only this exact path. -rm -f "$final_recording" +# Stale output from an earlier attempt must never survive into this attempt's +# upload. Remove only these exact paths and pending artifact writes. +rm -f "$final_recording" "$SOLHEIM_SMOKE_OUT_DIR/verdict.json" "$SOLHEIM_SMOKE_OUT_DIR/trace.json" \ + "$SOLHEIM_SMOKE_OUT_DIR"/verdict.json.*.tmp "$SOLHEIM_SMOKE_OUT_DIR"/trace.json.*.tmp # The recorder needs display access, not provider/GitHub credentials. Fragmented # MP4 keeps completed fragments readable if the action is forcibly interrupted. @@ -103,8 +104,13 @@ sleep 0.5 kill -0 "$recorder_pid" 2>/dev/null || exit 1 # Arguments are used for credential-free recorder tests; CI supplies none. +# An injected command would inherit the provider credential, so reject it. # Backgrounding the provider keeps the signal traps responsive during the run. if [[ "$#" -gt 0 ]]; then + if [[ -n "${SOLHEIM_API_KEY:-}" ]]; then + echo "Refusing to run a supplied command while SOLHEIM_API_KEY is set" >&2 + exit 2 + fi "$@" & else # Invoke the Node driver directly: a package-manager wrapper would sit