diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index f4508f270e..f03ee6b408 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -98,6 +98,16 @@ jobs: run: pnpm check-types - name: Validate Code QA workflow run: pnpm test:code-qa-ci + - name: Validate Solheim provider smoke workflow + run: pnpm test:solheim-smoke-ci + - name: Test Solheim provider smoke units + run: pnpm test:solheim-smoke:unit + - name: Check Solheim provider smoke types + run: pnpm solheim-smoke:check-types + - name: Test agent evals units + run: pnpm test:agent-evals:unit + - name: Check agent evals types + run: pnpm agent-evals:check-types - name: Model-check task lifecycle protocols run: pnpm lifecycle:model-check - name: Validate MCP OAuth integration diff --git a/.github/workflows/solheim-provider-smoke.yml b/.github/workflows/solheim-provider-smoke.yml new file mode 100644 index 0000000000..7e0298818c --- /dev/null +++ b/.github/workflows/solheim-provider-smoke.yml @@ -0,0 +1,88 @@ +name: Solheim provider smoke + +# Project A: infrastructure validation only. No PR diff, review judgment, router, +# deterministic review rule, or GitHub posting. Only trusted main-branch code runs. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: solheim-provider-smoke + cancel-in-progress: false + +jobs: + smoke: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + # Retain the existing protected environment and secret names for compatibility. + # The environment requires reviewer approval, matching the deploy + # environment. The job guard above restricts runs to main, so the + # secret is released only for an approved main run. See + # scripts/solheim-smoke/README.md. + environment: final-vscode-review-smoke + steps: + - name: Checkout exact trusted main revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup-node-pnpm + with: + install-args: "--frozen-lockfile" + + - name: Build trusted extension and webview + run: | + pnpm -w bundle + pnpm --filter @roo-code/vscode-webview build + + - name: Install display dependencies + run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64 + + - name: Run Solheim provider smoke + id: provider + timeout-minutes: 8 + env: + SOLHEIM_API_KEY: ${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }} + SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke + run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh + + # The recorder publishes smoke.mp4 only through an atomic rename + # after a nonempty check and ffprobe validation, so the published + # final is the commit marker: present and nonempty means complete. + # There is no separate persisted status marker. + - name: Verify recording integrity + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + env: + SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke + run: | + test -s "$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" + echo "Recording published and nonempty: $SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" + + # Verdict and trace evidence must survive even when the recording + # is missing, so metadata and video are uploaded separately. + - name: Upload provider smoke verdict and trace + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: solheim-provider-smoke-${{ github.sha }} + path: | + ${{ runner.temp }}/solheim-provider-smoke/verdict.json + ${{ runner.temp }}/solheim-provider-smoke/trace.json + if-no-files-found: error + retention-days: 7 + + # Exact validated final path only; the recorder leaves no temporary + # file behind, so a failed upload here reports the missing recording. + - name: Upload provider smoke recording + if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: solheim-provider-smoke-recording-${{ github.sha }} + path: ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4 + if-no-files-found: error + retention-days: 7 diff --git a/package.json b/package.json index 4ca0b672c1..cff60a205b 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,12 @@ "check-types": "turbo check-types --log-order grouped --output-logs new-only", "test": "turbo test --log-order grouped --output-logs new-only", "test:code-qa-ci": "node --test scripts/code-qa-workflow.test.mjs", + "solheim:smoke": "node --import tsx scripts/solheim-smoke/driver.mts", + "test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs", + "test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts", + "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", + "test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts", + "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", @@ -38,6 +44,9 @@ "devDependencies": { "@changesets/cli": "2.31.1", "@roo-code/config-typescript": "workspace:^", + "@roo-code/ipc": "workspace:^", + "@roo-code/types": "workspace:^", + "@vscode/test-electron": "2.5.2", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", "@types/node": "22.20.1", @@ -52,7 +61,8 @@ "rimraf": "6.0.1", "tsx": "4.23.15", "turbo": "2.11.8", - "typescript": "5.9.3" + "typescript": "5.9.3", + "yaml": "2.9.1" }, "lint-staged": { "src/**/*.ts": [ diff --git a/packages/ipc/package.json b/packages/ipc/package.json index 374c4240b6..4dd505bb18 100644 --- a/packages/ipc/package.json +++ b/packages/ipc/package.json @@ -5,6 +5,7 @@ "type": "module", "exports": "./src/index.ts", "scripts": { + "test": "node --import tsx --test src/__tests__/*.test.ts", "lint": "eslint src --ext=ts --max-warnings=0", "check-types": "tsc --noEmit", "clean": "rimraf .turbo" diff --git a/packages/ipc/src/__tests__/ipc-server.test.ts b/packages/ipc/src/__tests__/ipc-server.test.ts new file mode 100644 index 0000000000..3419ebf982 --- /dev/null +++ b/packages/ipc/src/__tests__/ipc-server.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { IpcMessageType, IpcOrigin, TaskCommandName, ipcMessageSchema } from "@roo-code/types" +import { IpcServer } from "../ipc-server.js" + +function fixture() { + const logs: unknown[][] = [] + const commands: unknown[] = [] + const server = new IpcServer("/unused-test.sock", (...args) => logs.push(args)) + server.on(IpcMessageType.TaskCommand, (clientId, command) => commands.push({ clientId, command })) + return { logs, commands, receive: (data: unknown) => server["onMessage"](data) } +} +const key = "dummy-api-key-not-for-public-logging" +const command = (requestId: string) => ({ + type: IpcMessageType.TaskCommand, + origin: IpcOrigin.Client, + clientId: "client-1", + data: { + commandName: TaskCommandName.StartNewTask, + data: { requestId, text: "private prompt", configuration: { apiProvider: "openai", openAiApiKey: key } }, + }, +}) + +describe("IPC rejection logging", () => { + it("rejects an invalid request ID without logging its key, prompt or schema issue values", () => { + const f = fixture() + f.receive(command(`invalid request containing ${key}`)) + assert.deepEqual(f.commands, []) + assert.equal(f.logs.length, 1) + assert.equal(f.logs[0]?.[0], "[server#onMessage] invalid payload") + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.ok(!JSON.stringify(f.logs).includes("private prompt")) + assert.ok(!JSON.stringify(f.logs).includes("invalid request containing")) + }) + it("does not log credential-shaped primitive input or dynamic issue paths", () => { + const f = fixture() + const valid = command("valid-id") + const malformedHeaders = { + ...valid, + data: { + ...valid.data, + data: { + ...valid.data.data, + configuration: { ...valid.data.data.configuration, openAiHeaders: { [key]: null } }, + }, + }, + } + const rejected = ipcMessageSchema.safeParse(malformedHeaders) + assert.ok(!rejected.success) + assert.ok(rejected.error.issues.some((issue) => issue.path.includes(key))) + for (const data of [key, null, { type: key }, malformedHeaders]) { + f.receive(data) + } + assert.equal(f.logs.length, 4) + assert.ok(!JSON.stringify(f.logs).includes(key)) + assert.deepEqual(f.commands, []) + }) + it("still dispatches a valid correlated command to its client without logging its settings", () => { + const f = fixture() + f.receive(command("valid-id")) + assert.equal(f.commands.length, 1) + assert.deepEqual(f.commands[0], { clientId: "client-1", command: command("valid-id").data }) + assert.deepEqual(f.logs, []) + }) +}) diff --git a/packages/ipc/src/ipc-server.ts b/packages/ipc/src/ipc-server.ts index 200e241014..26dd626a9e 100644 --- a/packages/ipc/src/ipc-server.ts +++ b/packages/ipc/src/ipc-server.ts @@ -76,16 +76,15 @@ export class IpcServer extends EventEmitter implements RooCodeI private onMessage(data: unknown) { if (typeof data !== "object") { - this.log(`[server#onMessage] invalid data -> ${JSON.stringify(data)}`) + this.log("[server#onMessage] invalid data") return } const result = ipcMessageSchema.safeParse(data) if (!result.success) { - this.log( - `[server#onMessage] invalid payload -> ${JSON.stringify(result.error.issues)} -> ${JSON.stringify(data)}`, - ) + // Both the input and Zod issue values/paths may contain credentials. + this.log("[server#onMessage] invalid payload", { issueCount: result.error.issues.length }) return } @@ -98,7 +97,7 @@ export class IpcServer extends EventEmitter implements RooCodeI this.emit(IpcMessageType.TaskCommand, payload.clientId, payload.data) break default: - this.log(`[server#onMessage] unhandled payload: ${JSON.stringify(payload)}`) + this.log("[server#onMessage] unhandled payload") break } } diff --git a/packages/types/src/__tests__/events.test.ts b/packages/types/src/__tests__/events.test.ts new file mode 100644 index 0000000000..3525e75eac --- /dev/null +++ b/packages/types/src/__tests__/events.test.ts @@ -0,0 +1,234 @@ +import { + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + TASK_START_STAGES, + taskEventSchema, + taskStartResponseSchema, +} from "../events.js" + +const requestId = "final-smoke-0123abcdef456789" + +describe("taskStartResponseSchema", () => { + it("accepts a success response with the accepted task ID", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true, taskId: "task-1" }) + expect(result.success).toBe(true) + + if (result.success) { + expect(result.data).toEqual({ requestId, success: true, taskId: "task-1" }) + } + }) + + it("accepts the fixed failure response", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.success).toBe(false) + expect(result.data.stage).toBeUndefined() + } + }) + + it("accepts a failure response carrying each allowed stage", () => { + for (const stage of TASK_START_STAGES) { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage, + }) + expect(result.success).toBe(true) + + if (result.success && !result.data.success) { + expect(result.data.stage).toBe(stage) + } + } + }) + + it("rejects a failure response carrying an unknown stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "secretStore", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a non-string stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: { name: "taskCreation" }, + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response carrying a stage", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + stage: "taskCreation", + }) + expect(result.success).toBe(false) + }) + + it("rejects a success response without a taskId", () => { + const result = taskStartResponseSchema.safeParse({ requestId, success: true }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying an unfixed error code", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: "provider_auth_failed", + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response carrying a raw error message", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: "openai rejected the key sk-secret-123 at https://api.example.com", + }) + expect(result.success).toBe(false) + }) + + it("rejects an empty requestId", () => { + const result = taskStartResponseSchema.safeParse({ requestId: "", success: true, taskId: "task-1" }) + expect(result.success).toBe(false) + }) + + it("rejects a success response with an extra top-level field", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: true, + taskId: "task-1", + providerError: "openai rejected the key sk-secret-123", + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure response with an extra field on the error object", () => { + const result = taskStartResponseSchema.safeParse({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }) + expect(result.success).toBe(false) + }) +}) + +describe("taskEventSchema TaskStartResponse", () => { + it("routes a success response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1" }], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("routes a failure response with a stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + }, + ], + }) + expect(result.success).toBe(true) + }) + + it("rejects a failure payload with an unknown stage through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "keyring", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with a stage plus an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + stage: "taskCreation", + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) + + it("rejects a success payload with an extra field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1", stack: "at run (api.ts:1:1)" }], + }) + expect(result.success).toBe(false) + }) + + it("rejects a failure payload with an extra error field through the task event envelope", () => { + const result = taskEventSchema.safeParse({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + rawError: "sk-secret-123", + }, + ], + }) + expect(result.success).toBe(false) + }) +}) diff --git a/packages/types/src/__tests__/ipc.test.ts b/packages/types/src/__tests__/ipc.test.ts index a843354a55..daf9b3cb1d 100644 --- a/packages/types/src/__tests__/ipc.test.ts +++ b/packages/types/src/__tests__/ipc.test.ts @@ -117,4 +117,59 @@ describe("IPC Types", () => { expect(result.success).toBe(false) }) }) + + describe("StartNewTask requestId", () => { + const legacyCommand = { + commandName: TaskCommandName.StartNewTask, + data: { + configuration: {}, + text: "review the diff", + }, + } + + it("should accept a legacy StartNewTask without requestId", () => { + const result = taskCommandSchema.safeParse(legacyCommand) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBeUndefined() + } + }) + + it("should accept and preserve a valid requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "final-smoke-0123abcdef456789" }, + }) + expect(result.success).toBe(true) + + if (result.success && result.data.commandName === TaskCommandName.StartNewTask) { + expect(result.data.data.requestId).toBe("final-smoke-0123abcdef456789") + } + }) + + it("should reject an empty requestId", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "" }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId longer than 128 characters", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "a".repeat(129) }, + }) + expect(result.success).toBe(false) + }) + + it("should reject a requestId with characters outside the allowed set", () => { + const result = taskCommandSchema.safeParse({ + ...legacyCommand, + data: { ...legacyCommand.data, requestId: "id with spaces" }, + }) + expect(result.success).toBe(false) + }) + }) }) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 20f7f7e71e..204256980d 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -51,8 +51,59 @@ export enum RooCodeEventName { CommandsResponse = "commandsResponse", ModesResponse = "modesResponse", ModelsResponse = "modelsResponse", + /** Correlated reply to a StartNewTask command that carried a requestId. */ + TaskStartResponse = "taskStartResponse", } +/** + * TaskStartResponse + */ + +// The requestId is generated by the command sender and copied into the reply, +// so its shape is bounded and restricted to characters that cannot smuggle +// log structure. +export const taskStartRequestIdSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9._:-]+$/) + +// The failure values are fixed constants. The response never carries raw +// thrown messages, prompts, images, provider settings, secrets, URLs, or +// stack traces. Both variants reject unknown keys, so an extra or +// secret-bearing field fails validation instead of being stripped. +export const TASK_START_FAILURE_ERROR_CODE = "task_start_failed" +export const TASK_START_FAILURE_ERROR_MESSAGE = "Task start failed" + +// Fixed stage vocabulary for a failed correlated start. The stage names the +// startup phase that stopped responding; it carries no error content. +export const TASK_START_STAGES = ["eviction", "settings", "taskCreation", "unknown"] as const + +export const taskStartStageSchema = z.enum(TASK_START_STAGES) + +export type TaskStartStage = (typeof TASK_START_STAGES)[number] + +export const taskStartResponseSchema = z.discriminatedUnion("success", [ + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(true), + taskId: z.string().min(1), + }) + .strict(), + z + .object({ + requestId: taskStartRequestIdSchema, + success: z.literal(false), + errorCode: z.literal(TASK_START_FAILURE_ERROR_CODE), + errorMessage: z.literal(TASK_START_FAILURE_ERROR_MESSAGE), + stage: taskStartStageSchema.optional(), + }) + .strict(), +]) + +export type TaskStartResponse = z.infer + /** * RooCodeEvents */ @@ -125,6 +176,7 @@ export const rooCodeEventsSchema = z.object({ ]), [RooCodeEventName.ModesResponse]: z.tuple([z.array(z.object({ slug: z.string(), name: z.string() }))]), [RooCodeEventName.ModelsResponse]: z.tuple([z.record(z.string(), modelInfoSchema)]), + [RooCodeEventName.TaskStartResponse]: z.tuple([taskStartResponseSchema]), }) export type RooCodeEvents = z.infer @@ -270,6 +322,11 @@ export const taskEventSchema = z.discriminatedUnion("eventName", [ payload: rooCodeEventsSchema.shape[RooCodeEventName.ModelsResponse], taskId: z.number().optional(), }), + z.object({ + eventName: z.literal(RooCodeEventName.TaskStartResponse), + payload: rooCodeEventsSchema.shape[RooCodeEventName.TaskStartResponse], + taskId: z.number().optional(), + }), ]) export type TaskEvent = z.infer diff --git a/packages/types/src/ipc.ts b/packages/types/src/ipc.ts index fea040af0b..e7df762069 100644 --- a/packages/types/src/ipc.ts +++ b/packages/types/src/ipc.ts @@ -1,6 +1,6 @@ import { z } from "zod" -import { type TaskEvent, taskEventSchema } from "./events.js" +import { type TaskEvent, taskEventSchema, taskStartRequestIdSchema } from "./events.js" import { rooCodeSettingsSchema } from "./global-settings.js" /** @@ -64,6 +64,8 @@ export const taskCommandSchema = z.discriminatedUnion("commandName", [ text: z.string(), images: z.array(z.string()).optional(), newTab: z.boolean().optional(), + // Senders without a requestId keep the fire-and-forget contract. + requestId: taskStartRequestIdSchema.optional(), }), }), z.object({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db4358d6e7..b81b295307 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,6 +35,12 @@ importers: '@roo-code/config-typescript': specifier: workspace:^ version: link:packages/config-typescript + '@roo-code/ipc': + specifier: workspace:^ + version: link:packages/ipc + '@roo-code/types': + specifier: workspace:^ + version: link:packages/types '@stryker-mutator/core': specifier: 10.0.0 version: 10.0.0(@types/node@22.20.1) @@ -44,6 +50,9 @@ importers: '@types/node': specifier: 22.20.1 version: 22.20.1 + '@vscode/test-electron': + specifier: 2.5.2 + version: 2.5.2 '@vscode/vsce': specifier: 3.9.2 version: 3.9.2 @@ -80,6 +89,9 @@ importers: typescript: specifier: 5.9.3 version: 5.9.3 + yaml: + specifier: 2.9.1 + version: 2.9.1 apps/cli: dependencies: diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md new file mode 100644 index 0000000000..bd24b6cfba --- /dev/null +++ b/scripts/agent-evals/README.md @@ -0,0 +1,104 @@ +# Agent evals fixture substrate + +This directory holds the reusable, trusted fixture execution substrate. One +controller runs a registered fixture inside one isolated VS Code extension +host and writes a versioned, metadata-only structured trace. It carries no +knowledge of any provider, task solution, or grading rule. + +## Layout + +| File | Purpose | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`contract.ts`](./contract.ts) | Fixture contract and validation: identity, version, workspace files, prompt, tool policy, execution limits, settings projection, settings builder. | +| [`registry.ts`](./registry.ts) | Trusted in-memory registry. Fixtures enter only through validated registration from repo code. No file, URL, or dynamic loading exists. | +| [`runner.ts`](./runner.ts) | Execution controller: startup correlation, bounded event buffering, request accounting, bounded teardown. | +| [`trace.ts`](./trace.ts) | Trace schema version 1, request accountant, stable digests, fixed notes. | +| [`launcher.ts`](./launcher.ts) | The single VS Code launcher: download, isolated spawn, IPC client, atomic artifact writes, run identity. | +| [`isolation.ts`](./isolation.ts) | Child environment allowlist and launch arguments. | +| [`task-start.ts`](./task-start.ts) | Correlated StartNewTask waiter. | + +## Fixture contract + +A fixture is trusted in-repo data. The contract has no executable hooks, no +URLs, and no paths outside the declared workspace files. The only function is +`buildConfiguration(credential)`, which maps one credential to extension +settings. Validation rejects path traversal, drive letters, oversized files, +missing limits, and non-boolean tool allowances. The launcher repeats the +workspace containment check at write time. + +Execution limits are enforced, not decorative. The controller applies the +activation, task start, task run, usage drain, and teardown step limits, and +the trace records the measured durations. + +## Structured trace + +Each run writes `trace.json` with `traceSchemaVersion: 1`. The trace is +metadata only. Prompts, model output, reasoning, tool arguments, error +payloads, and storage paths are excluded. Provider settings appear only as +the explicit nonsecret projection in `settings`. + +Accounting rules: + +- Request rows upsert by message timestamp across created and updated + `api_req_started` messages. +- Usage snapshots come from extension events. The trace keeps the latest + snapshot only and never sums cumulative snapshots. +- Zero token or cost values can be defaults from the adapter or the task + layer. The trace labels event-sourced numbers and never claims a provider + measured a zero. +- `contextTokens` is an extension context estimate. +- `apiReqRetriedMarkers` counts retry marker messages. It is not an HTTP + retry count. +- Missing values carry a reason, such as `notReported` or + `noUsageEventsObserved`. They are never filled with zero. +- The settings digest is SHA-256 over an explicit nonsecret projection. The + run fails closed before launch if the projection contains the credential. +- Execution result, accounting completeness, and teardown status are three + separate sections. A teardown failure never changes an execution code. + +Events that arrive before the task start acknowledgment are buffered with a +fixed cap. After acceptance the controller replays only events that belong to +the accepted task and counts the rest as foreign or malformed. After an +extension-reported completion the controller drains late usage events for the +bounded drain window. + +## Terminal code semantics + +The trace carries two different completion signals. Reviewers must not merge +them: + +- `COMPLETED` requires the extension `taskCompleted` event. The extension emits + it only after the completion result is accepted, or a delegated subtask hands + back, and the turn is persisted. A stop request, teardown, or a later session + loss never changes this code. +- `execution.completionMessageSeen` records the final `completion_result` + message for the accepted task. That message is model output, not acceptance. + It never produces `COMPLETED`. +- `STOPPED` with `adapterStopAfterCompletionMessage` names an observer stop + after that message. `STOPPED` with `adapterStop` names an observer stop + without it. `TASK_RUN_TIMEOUT` is the bounded run window expiry. A seen + message still appears in `completionMessageSeen`. +- `SESSION_LOST` covers disconnects and interrupts. A manual stop after + completion output stays `SESSION_LOST`. +- The usage drain runs only after a `taskCompleted` event. It stays inside the + bounded drain window. + +## Checks + +```sh +pnpm test:agent-evals:unit +pnpm agent-evals:check-types +``` + +Both run offline. No check spawns VS Code or touches a provider. + +## Follow-up milestones + +The substrate supports these planned steps. None of them exist in this +change: + +1. Five frozen, real coding task fixtures with known-good workspaces. +2. Independent credential-free graders that judge workspace state only. +3. Paired exact trusted commit comparisons on the same fixtures. +4. Repeated runs with uncertainty reporting. +5. Advisory release integration aligned to the actual release workflows. diff --git a/scripts/agent-evals/contract.test.ts b/scripts/agent-evals/contract.test.ts new file mode 100644 index 0000000000..3f60100e5b --- /dev/null +++ b/scripts/agent-evals/contract.test.ts @@ -0,0 +1,99 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { FixtureValidationError, validateFixture, validateWorkspaceFilePath } from "./contract.ts" +import { buildSmokeFixture } from "../solheim-smoke/fixture.ts" +import { buildSmokeConfiguration, SMOKE_PROMPT } from "../solheim-smoke/config.ts" + +const expectFailure = (run: () => void, pattern: RegExp): void => { + assert.throws(run, (error: unknown) => error instanceof FixtureValidationError && pattern.test(error.message)) +} + +describe("agent fixture contract", () => { + it("accepts the registered smoke fixture", () => { + validateFixture(buildSmokeFixture()) + }) + + it("rejects malformed identity, prompt, and workspace declarations", () => { + const base = buildSmokeFixture() + expectFailure(() => validateFixture({ ...base, identity: { ...base.identity, name: "Bad Name" } }), /name/) + expectFailure(() => validateFixture({ ...base, identity: { ...base.identity, version: "1.0" } }), /version/) + expectFailure(() => validateFixture({ ...base, prompt: "" }), /prompt/) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "../escape.txt", content: "x" }] } }), + /segments/, + ) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "/etc/passwd", content: "x" }] } }), + /relative/, + ) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "a\\b.txt", content: "x" }] } }), + /forward slashes/, + ) + expectFailure( + () => validateFixture({ ...base, workspace: { files: [{ path: "C:run.txt", content: "x" }] } }), + /drive letter/, + ) + expectFailure( + () => + validateFixture({ ...base, workspace: { files: [{ path: "big.txt", content: "x".repeat(16_385) }] } }), + /too large/, + ) + }) + + it("rejects missing limits, non-boolean allowances, and hook-shaped builders", () => { + const base = buildSmokeFixture() + const limits = { ...base.limits } + delete (limits as Partial)["taskRunTimeoutMs"] + expectFailure(() => validateFixture({ ...base, limits: limits as typeof base.limits }), /taskRunTimeoutMs/) + expectFailure(() => validateFixture({ ...base, limits: { ...base.limits, taskRunTimeoutMs: 0 } }), /positive/) + expectFailure( + () => validateFixture({ ...base, limits: { ...base.limits, taskRunTimeoutMs: 601_000 } }), + /within bounds/, + ) + const allow = { ...base.toolPolicy.allow } + delete (allow as Partial)["execute"] + expectFailure( + () => validateFixture({ ...base, toolPolicy: { ...base.toolPolicy, allow: allow as typeof allow } }), + /execute/, + ) + expectFailure( + () => + validateFixture({ + ...base, + toolPolicy: { ...base.toolPolicy, disabledTools: [] }, + }), + /disable at least one tool/, + ) + const broken = buildSmokeFixture() as unknown as Record + broken["buildConfiguration"] = "not a function" + expectFailure(() => validateFixture(broken as unknown as typeof base), /builder/) + }) + + it("keeps the smoke projection in sync with the built configuration and free of credentials", () => { + const fixture = buildSmokeFixture() + const credential = "sk-live-credential" + assert.equal(fixture.prompt, SMOKE_PROMPT) + assert.equal(fixture.buildConfiguration, buildSmokeConfiguration) + const config = buildSmokeConfiguration(credential) + for (const [key, value] of Object.entries(fixture.settingsProjection)) { + const expected = + key === "openAiExtraBody" + ? JSON.parse(config.openAiExtraBody ?? "null") + : config[key as keyof typeof config] + assert.deepEqual(value, expected, `projection key ${key} drifts from the built configuration`) + } + assert.ok(!Object.keys(fixture.settingsProjection).includes("openAiApiKey")) + assert.ok(!JSON.stringify(fixture.settingsProjection).includes(credential)) + }) + + it("bounds workspace file paths at the contract layer", () => { + validateWorkspaceFilePath("docs/readme.md") + validateWorkspaceFilePath("src/nested/deep/file.ts") + expectFailure(() => validateWorkspaceFilePath(".."), /segments/) + expectFailure(() => validateWorkspaceFilePath("a/./b"), /segments/) + expectFailure(() => validateWorkspaceFilePath("a//b"), /segments/) + expectFailure(() => validateWorkspaceFilePath(""), /missing/) + expectFailure(() => validateWorkspaceFilePath(`a/${"x".repeat(300)}`), /out of bounds/) + }) +}) diff --git a/scripts/agent-evals/contract.ts b/scripts/agent-evals/contract.ts new file mode 100644 index 0000000000..2e6c0ba30b --- /dev/null +++ b/scripts/agent-evals/contract.ts @@ -0,0 +1,195 @@ +import type { RooCodeSettings } from "@roo-code/types" + +export interface AgentFixtureWorkspaceFile { + path: string + content: string +} + +export interface AgentFixtureToolAllow { + readOnly: boolean + readOnlyOutsideWorkspace: boolean + write: boolean + writeOutsideWorkspace: boolean + writeProtected: boolean + execute: boolean + mcp: boolean + modeSwitch: boolean + subtasks: boolean + followupQuestions: boolean +} + +export interface AgentFixtureLimits { + activationTimeoutMs: number + taskStartTimeoutMs: number + taskRunTimeoutMs: number + usageDrainMs: number + teardownStepMs: number +} + +// A fixture is trusted, in-repo data. The contract deliberately carries no +// executable hooks, no URLs to fetch, and no file paths outside the declared +// workspace files. The only function is the settings builder, which maps a +// credential to extension settings and never receives anything else. +export interface AgentFixture { + identity: { + name: string + version: string + description: string + } + prompt: string + workspace: { + files: readonly AgentFixtureWorkspaceFile[] + } + toolPolicy: { + mode: string + roleDefinition: string + groups: readonly string[] + disabledTools: readonly string[] + allow: AgentFixtureToolAllow + } + limits: AgentFixtureLimits + // Explicit nonsecret mirror of the built configuration. The trace digests + // this projection; it must never contain the credential. + settingsProjection: Record + buildConfiguration(credential: string): RooCodeSettings +} + +export class FixtureValidationError extends Error {} + +const NAME_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/ +const VERSION_PATTERN = /^\d+\.\d+\.\d+$/ +const MAX_DESCRIPTION_CHARS = 300 +const MAX_PROMPT_CHARS = 8_000 +const MAX_MODE_CHARS = 64 +const MAX_ROLE_CHARS = 2_000 +const MAX_PATH_CHARS = 256 +const MAX_FILES = 64 +const MAX_FILE_CHARS = 16_384 +const MAX_TOTAL_FILE_CHARS = 262_144 +const MAX_PROJECTION_CHARS = 16_384 +const MAX_LIMIT_MS = 600_000 +const MAX_DRAIN_MS = 60_000 + +const ALLOW_KEYS = [ + "readOnly", + "readOnlyOutsideWorkspace", + "write", + "writeOutsideWorkspace", + "writeProtected", + "execute", + "mcp", + "modeSwitch", + "subtasks", + "followupQuestions", +] as const satisfies readonly (keyof AgentFixtureToolAllow)[] + +const LIMIT_KEYS = [ + "activationTimeoutMs", + "taskStartTimeoutMs", + "taskRunTimeoutMs", + "usageDrainMs", + "teardownStepMs", +] as const satisfies readonly (keyof AgentFixtureLimits)[] + +const fail = (message: string): never => { + throw new FixtureValidationError(message) +} + +const isPlainObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const boundedString = (value: unknown, max: number, label: string): string => { + if (typeof value !== "string" || value.length === 0 || value.length > max) + return fail(`${label} missing or out of bounds`) + return value +} + +// Trust-boundary check. The launcher repeats the containment check at write +// time; this check rejects malformed fixture data at registration. +export function validateWorkspaceFilePath(pathValue: string): void { + if (typeof pathValue !== "string" || pathValue.length === 0 || pathValue.length > MAX_PATH_CHARS) + fail("workspace file path missing or out of bounds") + if (pathValue.includes("\\")) fail("workspace file path must use forward slashes") + if (pathValue.includes("\0")) fail("workspace file path must not contain a NUL byte") + if (pathValue.startsWith("/")) fail("workspace file path must be relative") + if (/^[a-zA-Z]:/.test(pathValue)) fail("workspace file path must not carry a drive letter") + for (const segment of pathValue.split("/")) + if (segment === "" || segment === "." || segment === "..") + fail("workspace file path segments must be plain names") +} + +const serializeProjection = (projection: Record): string | undefined => { + try { + return JSON.stringify(projection) + } catch { + return undefined + } +} + +// The built configuration is what the extension runs. It must carry exactly the +// declared tool policy, so a drifted builder cannot widen tool access. +export function configurationMatchesToolPolicy( + config: Record, + policy: AgentFixture["toolPolicy"], +): boolean { + const sameList = (a: unknown, b: readonly string[]) => + Array.isArray(a) && a.length === b.length && b.every((item) => a.includes(item)) + if (config["mode"] !== policy.mode || !sameList(config["disabledTools"], policy.disabledTools)) return false + const modes = config["customModes"] + const custom = Array.isArray(modes) ? modes.find((m) => isPlainObject(m) && m["slug"] === policy.mode) : undefined + if (!isPlainObject(custom) || custom["roleDefinition"] !== policy.roleDefinition) return false + if (!sameList(custom["groups"], policy.groups)) return false + return ALLOW_KEYS.every((key) => config[`alwaysAllow${key[0]!.toUpperCase()}${key.slice(1)}`] === policy.allow[key]) +} + +export function validateFixture(fixture: AgentFixture): void { + if (!isPlainObject(fixture)) fail("fixture must be an object") + if (!isPlainObject(fixture.identity)) fail("fixture identity missing") + boundedString(fixture.identity.name, 64, "fixture name") + if (!NAME_PATTERN.test(fixture.identity.name)) fail("fixture name must be lowercase kebab case") + if (typeof fixture.identity.version !== "string" || !VERSION_PATTERN.test(fixture.identity.version)) + fail("fixture version must be semver") + boundedString(fixture.identity.description, MAX_DESCRIPTION_CHARS, "fixture description") + boundedString(fixture.prompt, MAX_PROMPT_CHARS, "fixture prompt") + + if (!isPlainObject(fixture.workspace) || !Array.isArray(fixture.workspace.files)) fail("fixture workspace missing") + if (fixture.workspace.files.length > MAX_FILES) fail("too many workspace files") + let totalChars = 0 + for (const file of fixture.workspace.files) { + if (!isPlainObject(file)) fail("workspace file must be an object") + validateWorkspaceFilePath(file.path) + if (typeof file.content !== "string" || file.content.length > MAX_FILE_CHARS) + fail("workspace file content missing or too large") + totalChars += file.content.length + if (totalChars > MAX_TOTAL_FILE_CHARS) fail("workspace files exceed the total size budget") + } + + if (!isPlainObject(fixture.toolPolicy)) fail("fixture tool policy missing") + boundedString(fixture.toolPolicy.mode, MAX_MODE_CHARS, "tool policy mode") + boundedString(fixture.toolPolicy.roleDefinition, MAX_ROLE_CHARS, "tool policy role definition") + if (!Array.isArray(fixture.toolPolicy.groups)) fail("tool policy groups missing") + if (!fixture.toolPolicy.groups.every((group) => typeof group === "string")) + fail("tool policy groups must be strings") + if (!Array.isArray(fixture.toolPolicy.disabledTools) || fixture.toolPolicy.disabledTools.length === 0) + fail("tool policy must disable at least one tool") + if (!fixture.toolPolicy.disabledTools.every((tool) => typeof tool === "string")) + fail("disabled tools must be strings") + if (!isPlainObject(fixture.toolPolicy.allow)) fail("tool policy allowances missing") + for (const key of ALLOW_KEYS) + if (typeof fixture.toolPolicy.allow[key] !== "boolean") fail(`tool policy allowance ${key} must be boolean`) + + if (!isPlainObject(fixture.limits)) fail("fixture limits missing") + for (const key of LIMIT_KEYS) { + const limit = fixture.limits[key] + if (!Number.isInteger(limit) || (limit as number) <= 0 || (limit as number) > MAX_LIMIT_MS) + fail(`fixture limit ${key} must be a positive integer within bounds`) + } + if ((fixture.limits.usageDrainMs as number) > MAX_DRAIN_MS) fail("usage drain bound exceeds its cap") + + if (!isPlainObject(fixture.settingsProjection)) fail("settings projection missing") + const projectionJson = serializeProjection(fixture.settingsProjection) + if (projectionJson === undefined || projectionJson.length > MAX_PROJECTION_CHARS) + fail("settings projection out of bounds") + + if (typeof fixture.buildConfiguration !== "function") fail("fixture settings builder missing") +} diff --git a/scripts/agent-evals/isolation.ts b/scripts/agent-evals/isolation.ts new file mode 100644 index 0000000000..06ab950ef6 --- /dev/null +++ b/scripts/agent-evals/isolation.ts @@ -0,0 +1,36 @@ +import path from "node:path" + +// Inherit only host-launch necessities, never the credential-bearing parent environment. +export function buildChildEnvironment( + env: NodeJS.ProcessEnv, + isolatedHome: string, + socketPath: string, +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of ["PATH", "DISPLAY", "XAUTHORITY", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT", "WINDIR"]) + if (env[name]) child[name] = env[name] + return { + ...child, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + XDG_CONFIG_HOME: path.join(isolatedHome, ".config"), + XDG_CACHE_HOME: path.join(isolatedHome, ".cache"), + ROO_CODE_IPC_SOCKET_PATH: socketPath, + } +} + +export function buildLaunchArgs(workspace: string, userData: string, extensions: string, extension: string): string[] { + return [ + workspace, + `--user-data-dir=${userData}`, + `--extensions-dir=${extensions}`, + `--extensionDevelopmentPath=${extension}`, + "--no-sandbox", + "--disable-gpu-sandbox", + "--password-store=basic", + "--disable-updates", + "--skip-welcome", + "--skip-release-notes", + "--disable-workspace-trust", + ] +} diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts new file mode 100644 index 0000000000..905ea6d1de --- /dev/null +++ b/scripts/agent-evals/launcher.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { fileURLToPath } from "node:url" +import { createHostRuntime } from "./launcher.ts" + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +const tempDir = async (prefix: string): Promise => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)) + return dir +} + +describe("host runtime", () => { + it("writes artifacts atomically and replaces them on rewrite", async () => { + const dir = await tempDir("agent-evals-artifacts-") + try { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: dir }) + await runtime.writeArtifact("trace.json", { v: 1 }) + assert.equal(await fs.readFile(path.join(dir, "trace.json"), "utf8"), JSON.stringify({ v: 1 }, null, 2)) + await runtime.writeArtifact("trace.json", { v: 2 }) + assert.equal(JSON.parse(await fs.readFile(path.join(dir, "trace.json"), "utf8")).v, 2) + const leftovers = (await fs.readdir(dir)).filter((name) => name.endsWith(".tmp")) + assert.deepEqual(leftovers, []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("contains artifact names inside the artifacts directory", async () => { + const dir = await tempDir("agent-evals-artifacts-") + try { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: dir }) + await runtime.writeArtifact("../escape.json", { ok: true }) + assert.ok((await fs.stat(path.join(dir, "escape.json"))).isFile()) + assert.equal( + await fs.stat(path.join(dir, "..", "escape.json")).then( + () => true, + () => false, + ), + false, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("reports run identity from the repository, or bounded unavailability", async () => { + const dir = await tempDir("agent-evals-identity-") + try { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: dir }) + const identity = await runtime.readIdentity?.() + assert.ok(identity) + assert.match(identity.nodeVersion, /^v\d+\./) + assert.equal(["linux", "darwin", "win32"].includes(identity.platform), true) + if (identity.commit.available) assert.match(identity.commit.value, /^[0-9a-f]{40}$/) + if (identity.extensionVersion.available) assert.match(identity.extensionVersion.value, /^\d+\.\d+\.\d+/) + if (identity.vscodeTarget.available) assert.match(identity.vscodeTarget.value, /^\d+\.\d+\.\d+/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts new file mode 100644 index 0000000000..1edd2917d2 --- /dev/null +++ b/scripts/agent-evals/launcher.ts @@ -0,0 +1,189 @@ +import { execFile, spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { promisify } from "node:util" +import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { IpcClient } from "@roo-code/ipc" +import { IpcMessageType } from "@roo-code/types" +import { buildChildEnvironment, buildLaunchArgs } from "./isolation.ts" +import type { AgentClient, AgentHost, AgentRuntime } from "./runner.ts" +import { identityUnavailable, measured, type MeasuredString, type TraceRunIdentity } from "./trace.ts" + +const execFileAsync = promisify(execFile) + +export interface HostRuntimeOptions { + root: string + artifactsDir: string + now?: () => number + pause?: (ms: number) => Promise +} + +const readCommit = async (root: string): Promise => { + try { + const fromCi = process.env.GITHUB_SHA + if (fromCi && /^[0-9a-f]{40}$/i.test(fromCi)) return { available: true, value: fromCi } + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) + const sha = stdout.trim() + return /^[0-9a-f]{40}$/i.test(sha) + ? { available: true, value: sha } + : { available: false, reason: "commitUnreadable" } + } catch { + return { available: false, reason: "gitUnavailable" } + } +} + +const readJsonField = async ( + file: string, + read: (parsed: Record) => unknown, +): Promise => { + try { + const parsed = JSON.parse(await fs.readFile(file, "utf8")) as Record + const value = read(parsed) + return measured(typeof value === "string" ? value : undefined, "fieldMissing") + } catch { + return { available: false, reason: "fileUnreadable" } + } +} + +// The only place where VS Code is downloaded, spawned, isolated and torn +// down. Both the smoke driver and future adapters get the same launcher, so +// process and credential isolation cannot drift between entry points. +export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { + const { root, artifactsDir } = options + let temp: string | undefined + + return { + async launch(workspaceFiles) { + const extension = path.join(root, "src") + await fs.access(path.join(extension, "package.json")) + temp = await fs.mkdtemp(path.join(os.tmpdir(), "agent-evals-")) + const workspace = path.join(temp, "workspace") + const isolatedHome = path.join(temp, "home") + await Promise.all([workspace, isolatedHome].map((folder) => fs.mkdir(folder))) + const workspaceRoot = await fs.realpath(workspace) + for (const file of workspaceFiles) { + const target = path.resolve(workspaceRoot, file.path) + if (target !== workspaceRoot && !target.startsWith(workspaceRoot + path.sep)) + throw new Error("workspace file path escapes the workspace") + await fs.mkdir(path.dirname(target), { recursive: true }) + await fs.writeFile(target, file.content) + if (!(await fs.realpath(target)).startsWith(workspaceRoot + path.sep)) + throw new Error("workspace file escapes the workspace") + } + const socket = path.join(temp, "ipc.sock") + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies: Record + } + const executable = await downloadAndUnzipVSCode({ + version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", + cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + }) + const args = buildLaunchArgs( + workspace, + path.join(temp, "userdata"), + path.join(temp, "extensions"), + extension, + ) + const child = spawn( + process.env.DISPLAY ? executable : "xvfb-run", + process.env.DISPLAY ? args : ["-a", executable, ...args], + { + detached: true, + env: buildChildEnvironment(process.env, isolatedHome, socket), + stdio: "ignore", + }, + ) + let exited = false + let onExit = () => {} + const closed = () => { + exited = true + onExit() + } + child.once("error", () => { + // Spawn failure has no live PID; a failed kill must not claim exit. + if (!child.pid) closed() + else onExit() + }) + child.once("close", closed) + const host: AgentHost = { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + if (exited) listener() + }, + signal(signal) { + if (!child.pid || exited) return + try { + process.kill(-child.pid, signal) + } catch { + child.kill(signal) + } + }, + } + return host + }, + connect(): AgentClient { + const socket = path.join(temp ?? "", "ipc.sock") + const client = new IpcClient(socket, () => {}) + return { + get isReady() { + return client.isReady + }, + onTaskEvent(listener) { + client.on(IpcMessageType.TaskEvent, listener) + }, + onDisconnect(listener) { + client.on(IpcMessageType.Disconnect, listener) + }, + sendCommand(command) { + client.sendCommand(command) + }, + disconnect() { + client.disconnect() + }, + } + }, + async cleanup() { + if (temp) await fs.rm(temp, { recursive: true, force: true }) + }, + async writeArtifact(fileName, value) { + // Atomic, whitelisted metadata only; no provider text or host logs. + const safeName = path.basename(fileName) + const pending = path.join(artifactsDir, `${safeName}.${process.pid}.tmp`) + await fs.writeFile(pending, JSON.stringify(value, null, 2)) + await fs.rename(pending, path.join(artifactsDir, safeName)) + }, + onInterrupt(listener) { + process.once("SIGINT", listener) + process.once("SIGTERM", listener) + return () => { + process.removeListener("SIGINT", listener) + process.removeListener("SIGTERM", listener) + } + }, + now: options.now ?? Date.now, + pause: options.pause ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + async readIdentity(): Promise { + const fallback = identityUnavailable() + return { + commit: await readCommit(root), + extensionVersion: await readJsonField( + path.join(root, "src", "package.json"), + (parsed) => parsed["version"], + ), + vscodeTarget: await readJsonField(path.join(root, "apps", "vscode-e2e", "package.json"), (parsed) => { + const devDependencies = parsed["devDependencies"] + return typeof devDependencies === "object" && devDependencies !== null + ? (devDependencies as Record)["@types/vscode"] + : undefined + }), + nodeVersion: fallback.nodeVersion, + platform: fallback.platform, + arch: fallback.arch, + } + }, + } +} diff --git a/scripts/agent-evals/package.json b/scripts/agent-evals/package.json new file mode 100644 index 0000000000..c4fb2812b9 --- /dev/null +++ b/scripts/agent-evals/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/scripts/agent-evals/registry.test.ts b/scripts/agent-evals/registry.test.ts new file mode 100644 index 0000000000..a8ddf213e8 --- /dev/null +++ b/scripts/agent-evals/registry.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { getFixture, listFixtures, registerFixture } from "./registry.ts" +import { buildSmokeFixture, ensureSmokeFixtureRegistered, SMOKE_FIXTURE_NAME } from "../solheim-smoke/fixture.ts" + +describe("trusted fixture registry", () => { + it("registers, lists, and returns the smoke fixture by exact name", () => { + ensureSmokeFixtureRegistered() + assert.ok(listFixtures().includes(SMOKE_FIXTURE_NAME)) + assert.equal(getFixture(SMOKE_FIXTURE_NAME).identity.name, SMOKE_FIXTURE_NAME) + }) + + it("rejects duplicate registration and unknown names", () => { + assert.throws(() => registerFixture(buildSmokeFixture()), /already registered/) + assert.throws(() => getFixture("does-not-exist"), /unknown fixture/) + }) + + it("validates fixtures at the registry boundary", () => { + const broken = buildSmokeFixture() + const invalid = { ...broken, identity: { ...broken.identity, name: "UPPER" } } + assert.throws(() => registerFixture(invalid), /name/) + }) +}) diff --git a/scripts/agent-evals/registry.ts b/scripts/agent-evals/registry.ts new file mode 100644 index 0000000000..c1806cc737 --- /dev/null +++ b/scripts/agent-evals/registry.ts @@ -0,0 +1,22 @@ +import { validateFixture, type AgentFixture } from "./contract.ts" + +// Trusted in-memory registry. Fixtures enter only through validated +// registration from repo code. There is no file loading, no URL loading, and +// no dynamic import, so a registry name can never select untrusted content. +const fixtures = new Map() + +export function registerFixture(fixture: AgentFixture): void { + validateFixture(fixture) + if (fixtures.has(fixture.identity.name)) throw new Error(`fixture already registered: ${fixture.identity.name}`) + fixtures.set(fixture.identity.name, fixture) +} + +export function getFixture(name: string): AgentFixture { + const fixture = fixtures.get(name) + if (!fixture) throw new Error(`unknown fixture: ${name}`) + return fixture +} + +export function listFixtures(): string[] { + return [...fixtures.keys()].sort() +} diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts new file mode 100644 index 0000000000..70721f9075 --- /dev/null +++ b/scripts/agent-evals/runner.test.ts @@ -0,0 +1,958 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" +import { registerFixture } from "./registry.ts" +import { runAgentFixture, type AgentRuntime, type AgentRunResult } from "./runner.ts" +import { digestProjection, type AgentTrace } from "./trace.ts" +import type { AgentFixture } from "./contract.ts" + +const CREDENTIAL = "sk-sentinel-credential-123" + +const allow = { + readOnly: false, + readOnlyOutsideWorkspace: false, + write: false, + writeOutsideWorkspace: false, + writeProtected: false, + execute: false, + mcp: false, + modeSwitch: false, + subtasks: false, + followupQuestions: false, +} + +function registerTestFixture(name: string, overrides: Partial = {}): AgentFixture { + const fixture: AgentFixture = { + identity: { name, version: "1.0.0", description: "Synthetic controller test fixture." }, + prompt: `synthetic prompt for ${name}`, + workspace: { files: [] }, + toolPolicy: { + mode: "test", + roleDefinition: "Run one synthetic task.", + groups: [], + disabledTools: ["new_task"], + allow, + }, + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 5_000, + teardownStepMs: 5_000, + }, + settingsProjection: { provider: "test", model: `model-${name}` }, + buildConfiguration: () => ({ + apiProvider: "openai", + openAiApiKey: CREDENTIAL, + mode: "test", + customModes: [{ slug: "test", name: "Test", roleDefinition: "Run one synthetic task.", groups: [] }], + disabledTools: ["new_task"], + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + }), + ...overrides, + } + registerFixture(fixture) + return fixture +} + +type Scenario = "success" | "setup" | "activation" | "start-failure" | "disconnect" | "no-events" | "foreign" + +function harness( + scenario: Scenario, + options: { + stuck?: boolean + closeThrows?: boolean + timeout?: () => void + onPause?: (count: number) => void + disconnectOnPause?: number + } = {}, +) { + let now = 0 + let pauseCount = 0 + let exited = false + let onExit = () => {} + let onDisconnect = () => {} + let onEvent: (event: unknown) => void = () => {} + let interrupt = () => {} + let cleaned = false + let disconnected = false + let unsubscribed = false + let launched = false + let workspaceFiles: readonly { path: string; content: string }[] = [] + const commands: TaskCommand[] = [] + const signals: NodeJS.Signals[] = [] + const artifacts: { name: string; value: unknown }[] = [] + // Events emitted before the run registers its listener; they flush when + // the task command is sent, matching real post-connect delivery. + const pendingEvents: unknown[] = [] + const emit = (event: unknown) => { + if (launched) onEvent(event) + else pendingEvents.push(event) + } + const close = () => { + exited = true + onExit() + } + const runtime: AgentRuntime = { + async launch(files) { + launched = true + workspaceFiles = files + if (scenario === "setup") throw new Error("private setup detail") + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + }, + signal(signal) { + signals.push(signal) + if (!options.stuck) close() + }, + } + }, + connect() { + return { + isReady: scenario !== "activation", + onTaskEvent(listener) { + onEvent = listener + }, + onDisconnect(listener) { + onDisconnect = listener + }, + disconnect() { + disconnected = true + }, + sendCommand(command) { + commands.push(command) + if (command.commandName === TaskCommandName.CloseTask) { + if (options.closeThrows) throw new Error("private close error") + if (!options.stuck) close() + return + } + if (command.commandName !== TaskCommandName.StartNewTask) return + for (const event of pendingEvents.splice(0)) onEvent(event) + if (options.timeout) { + options.timeout() + return + } + onEvent({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + scenario === "start-failure" + ? { + requestId: command.data.requestId, + success: false, + errorCode: "task_start_failed", + errorMessage: "Task start failed", + } + : { requestId: command.data.requestId, success: true, taskId: "root" }, + ], + }) + const taskId = scenario === "foreign" ? "other" : "root" + if (scenario === "foreign") { + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 1, totalTokensOut: 1, totalCost: 0, contextTokens: 1 }, + {}, + ], + }) + } + if (scenario === "success") { + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "created", + message: { + ts: 1, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai" }), + }, + }, + ], + }) + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "updated", + message: { + ts: 1, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai", tokensIn: 7, tokensOut: 3 }), + }, + }, + ], + }) + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + ], + }) + } + if (scenario === "disconnect") onDisconnect() + }, + } + }, + async cleanup() { + cleaned = true + }, + async writeArtifact(name, value) { + artifacts.push({ name, value }) + }, + onInterrupt(listener) { + interrupt = listener + return () => { + unsubscribed = true + } + }, + now: () => now, + async pause(ms) { + now += ms + pauseCount++ + // The callback runs first so a test can emit before the loss lands. + options.onPause?.(pauseCount) + if (options.disconnectOnPause === pauseCount) onDisconnect() + }, + } + return { + runtime, + commands, + signals, + artifacts, + emit, + interrupt: () => interrupt(), + get cleaned() { + return cleaned + }, + get disconnected() { + return disconnected + }, + get unsubscribed() { + return unsubscribed + }, + get launched() { + return launched + }, + get workspaceFiles() { + return workspaceFiles + }, + } +} + +const writtenTrace = (f: ReturnType): AgentTrace => { + const entry = f.artifacts.find((artifact) => artifact.name === "trace.json") + assert.ok(entry) + return entry.value as AgentTrace +} + +describe("agent fixture controller", () => { + it("runs a fixture, writes a metadata-only trace, and keeps credentials and prompts out", async () => { + registerTestFixture("controller-success") + const f = harness("success") + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-success", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.completionMessageSeen, false) + assert.equal(result.execution.activated, true) + assert.equal(result.execution.taskAccepted, true) + assert.equal(result.teardown.status, "ok") + assert.equal(result.teardown.hostExited, true) + assert.deepEqual( + f.artifacts.map((artifact) => artifact.name), + ["trace.json"], + ) + const trace = writtenTrace(f) + assert.equal(trace.traceSchemaVersion, 1) + assert.equal(trace.kind, "agent-evals-trace") + assert.equal(trace.settings.available, true) + assert.equal(trace.settings.digest, digestProjection({ provider: "test", model: "model-controller-success" })) + assert.equal(trace.accounting.requestCount, 1) + assert.deepEqual(trace.accounting.requests[0]?.tokensIn, { available: true, value: 7 }) + assert.equal(trace.accounting.requests[0]?.messageCount, 2) + assert.equal(trace.accounting.usage.snapshotsObserved, 1) + assert.equal(trace.accounting.completeness.status, "complete") + assert.deepEqual(trace.accounting.completeness.reasons, []) + const serialized = JSON.stringify(trace) + assert.ok(!serialized.includes(CREDENTIAL)) + assert.ok(!serialized.includes("synthetic prompt")) + assert.ok(!serialized.includes("root")) + assert.ok(f.cleaned && f.disconnected && f.unsubscribed) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + }) + + it("buffers events before the start ack, replays only accepted task events, and counts the rest", async () => { + registerTestFixture("controller-buffer", { + limits: { + activationTimeoutMs: 100, + taskStartTimeoutMs: 100, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("foreign") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-buffer", credential: CREDENTIAL }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + const trace = writtenTrace(f) + assert.equal(trace.eventFlow.observed, 2) + assert.equal(trace.eventFlow.foreignTaskEvents, 1) + assert.equal(trace.eventFlow.replayedAfterAck, 1) + assert.ok(trace.accounting.completeness.reasons.includes("foreignTaskEventsDropped")) + assert.ok(trace.accounting.completeness.reasons.includes("noUsageEventsObserved")) + + const mixed = harness("success") + mixed.emit({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId: "nope", success: true, taskId: "x" }], + }) + const accepted: TaskEvent[] = [] + const result2 = await runAgentFixture(mixed.runtime, { + fixtureName: "controller-buffer", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => accepted.push(event), + shouldStop: () => accepted.length >= 3, + }, + }) + assert.equal(result2.execution.code, "STOPPED") + assert.equal(writtenTrace(mixed).eventFlow.replayedAfterAck, 3) + assert.equal(accepted.length, 3) + }) + + it("bounds the run timeout, the activation wait, and the usage drain", async () => { + registerTestFixture("controller-fast", { + limits: { + activationTimeoutMs: 200, + taskStartTimeoutMs: 200, + taskRunTimeoutMs: 200, + usageDrainMs: 500, + teardownStepMs: 100, + }, + }) + const timedOut = harness("no-events") + const timeoutResult = await runAgentFixture(timedOut.runtime, { + fixtureName: "controller-fast", + credential: CREDENTIAL, + }) + assert.equal(timeoutResult.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(timeoutResult.execution.stopReason, "runTimeout") + assert.equal(writtenTrace(timedOut).execution.timings.taskRunMs, 200) + assert.equal(writtenTrace(timedOut).accounting.usage.available, false) + + const neverReady = harness("activation") + const activationResult = await runAgentFixture(neverReady.runtime, { + fixtureName: "controller-fast", + credential: CREDENTIAL, + }) + assert.equal(activationResult.execution.code, "ACTIVATION_FAILED") + assert.equal(writtenTrace(neverReady).execution.timings.activationMs, 200) + assert.equal(neverReady.commands.length, 0) + assert.ok(neverReady.cleaned) + + registerTestFixture("controller-drain", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 500, + teardownStepMs: 5_000, + }, + }) + let drained = false + const withCompletion = harness("success", { + onPause: (count) => { + if (count >= 2 && !drained) { + drained = true + withCompletion.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 40, totalTokensOut: 20, totalCost: 0, contextTokens: 30 }, + { attempt_completion: { attempts: 1, failures: 0 } }, + ], + }) + } + }, + }) + withCompletion.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + const drainResult = await runAgentFixture(withCompletion.runtime, { + fixtureName: "controller-drain", + credential: CREDENTIAL, + }) + assert.equal(drainResult.execution.code, "COMPLETED") + assert.equal(drainResult.execution.stopReason, "taskCompleted") + const drainTrace = writtenTrace(withCompletion) + assert.equal(drainTrace.execution.timings.usageDrainMs, 500) + assert.equal(drainTrace.accounting.usage.snapshotsObserved, 3) + assert.equal(drainTrace.accounting.usage.lastSnapshot?.source, "taskTokenUsageUpdated") + assert.equal(drainTrace.accounting.usage.lastSnapshot?.tokensIn, 40) + assert.deepEqual(drainTrace.accounting.toolSnapshot.snapshot, { + attempt_completion: { attempts: 1, failures: 0 }, + }) + }) + + it("records teardown failure separately from execution, and keeps live-host storage out of cleanup", async () => { + registerTestFixture("controller-stuck") + const f = harness("success", { stuck: true }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-stuck", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.teardown.status, "failed") + assert.equal(result.teardown.hostExited, false) + assert.deepEqual(result.teardown.escalatedSignals, ["SIGTERM", "SIGKILL"]) + assert.equal(result.teardown.cleanup, "skippedLiveHost") + assert.equal(f.cleaned, false) + assert.equal(f.disconnected, true) + const trace = writtenTrace(f) + assert.equal(trace.teardown.status, "failed") + assert.equal(trace.execution.timings.teardownMs, 15_000) + }) + + it("keeps teardown moving after a CloseTask exception", async () => { + registerTestFixture("controller-closerror") + const f = harness("success", { closeThrows: true }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-closerror", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.teardown.status, "failed") + assert.deepEqual(result.teardown.failures, ["closeTask"]) + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + }) + + it("fails closed on setup errors and still writes a trace", async () => { + registerTestFixture("controller-setup") + const f = harness("setup") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-setup", credential: CREDENTIAL }) + assert.equal(result.execution.code, "SETUP_FAILED") + assert.equal(result.execution.stopReason, "setupError") + const trace = writtenTrace(f) + assert.ok(trace.accounting.completeness.reasons.includes("taskNotAccepted")) + assert.ok(f.cleaned && f.unsubscribed) + }) + + it("fails closed without a credential before launching", async () => { + registerTestFixture("controller-nocred") + const f = harness("success") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-nocred", credential: "" }) + assert.equal(result.execution.code, "SETUP_FAILED") + assert.equal(result.execution.stopReason, "credentialMissing") + assert.equal(f.launched, false) + const trace = writtenTrace(f) + assert.ok(trace) + // The settings section must name the real cause, not claim a + // fixture rejection that never happened. + assert.deepEqual(trace.settings, { available: false, reason: "credentialMissing" }) + }) + + it("rejects a configuration that widens the declared tool policy, and never launches", async () => { + const base = registerTestFixture("controller-policy-base") + registerTestFixture("controller-policy", { + buildConfiguration: (credential) => ({ ...base.buildConfiguration(credential), alwaysAllowExecute: true }), + }) + const f = harness("success") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-policy", credential: CREDENTIAL }) + assert.equal(result.execution.stopReason, "fixtureRejected") + assert.equal(f.launched, false) + assert.deepEqual(writtenTrace(f).settings, { available: false, reason: "configurationViolatesToolPolicy" }) + }) + + it("rejects a fixture whose projection carries the credential, and never launches", async () => { + registerTestFixture("controller-leak", { settingsProjection: { apiKey: CREDENTIAL } }) + const f = harness("success") + const result = await runAgentFixture(f.runtime, { fixtureName: "controller-leak", credential: CREDENTIAL }) + assert.equal(result.execution.code, "SETUP_FAILED") + assert.equal(result.execution.stopReason, "fixtureRejected") + assert.equal(f.launched, false) + const trace = writtenTrace(f) + assert.equal(trace.settings.available, false) + assert.equal(trace.settings.reason, "credentialInProjection") + assert.ok(!JSON.stringify(trace).includes(CREDENTIAL)) + }) + + it("counts malformed events and tool failures without storing payloads", async () => { + registerTestFixture("controller-noise") + const f = harness("success") + f.emit("garbage") + f.emit({ + eventName: RooCodeEventName.TaskToolFailed, + payload: ["root", "read_file", "private failure payload"], + }) + await runAgentFixture(f.runtime, { + fixtureName: "controller-noise", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + const trace = writtenTrace(f) + assert.equal(trace.eventFlow.malformed, 1) + assert.equal(trace.accounting.toolFailureEvents, 1) + assert.equal(trace.accounting.toolFailuresByTool["read_file"], 1) + assert.ok(!JSON.stringify(trace).includes("private failure payload")) + }) + + it("captures injected identity and reports unavailable identity with a reason", async () => { + registerTestFixture("controller-identity") + const withIdentity = harness("success") + withIdentity.runtime.readIdentity = async () => ({ + commit: { available: true, value: "a".repeat(40) }, + extensionVersion: { available: true, value: "3.0.0" }, + vscodeTarget: { available: true, value: "1.100.0" }, + nodeVersion: "v22.0.0", + platform: "linux", + arch: "x64", + }) + await runAgentFixture(withIdentity.runtime, { + fixtureName: "controller-identity", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.deepEqual(writtenTrace(withIdentity).run.identity.commit, { available: true, value: "a".repeat(40) }) + + const without = harness("success") + const plain = await runAgentFixture(without.runtime, { + fixtureName: "controller-identity", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.deepEqual(writtenTrace(without).run.identity.commit, { available: false, reason: "identityUnavailable" }) + assert.equal(plain.trace.run.identity.nodeVersion, process.version) + }) + + it("caveats accounting when the session dies inside the usage drain", async () => { + registerTestFixture("controller-drainloss", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 500, + teardownStepMs: 5_000, + }, + }) + let interruptCalled = false + const withCompletion = harness("success", { + onPause: (count) => { + // The drain is the first pause: the run loop exits on the + // pre-emitted completion without polling. + if (count === 1 && !interruptCalled) { + interruptCalled = true + withCompletion.interrupt() + } + }, + }) + withCompletion.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + const result = await runAgentFixture(withCompletion.runtime, { + fixtureName: "controller-drainloss", + credential: CREDENTIAL, + }) + // Completion was observed, so the run code stays COMPLETED and the + // run-phase session flag stays false. + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.sessionLost, false) + const trace = writtenTrace(withCompletion) + assert.equal(trace.execution.timings.usageDrainMs, 100) + assert.ok(trace.accounting.completeness.reasons.includes("sessionLostDuringUsageDrain")) + assert.ok(!trace.accounting.completeness.reasons.includes("sessionLost")) + assert.equal(trace.accounting.completeness.status, "partial") + }) + + it("passes fixture workspace files to the launcher and records session loss", async () => { + registerTestFixture("controller-workspace", { + workspace: { files: [{ path: "src/task.md", content: "fixture data" }] }, + }) + const f = harness("success") + await runAgentFixture(f.runtime, { + fixtureName: "controller-workspace", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.deepEqual([...f.workspaceFiles], [{ path: "src/task.md", content: "fixture data" }]) + + const lost = harness("disconnect") + const result = await runAgentFixture(lost.runtime, { + fixtureName: "controller-workspace", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "SESSION_LOST") + assert.equal(result.execution.stopReason, "sessionLost") + assert.equal(result.execution.sessionLost, true) + }) + + it("reproduces the synthetic PR 1909 topology: an observer stop after the completion message stays distinct", async () => { + // SYNTHETIC FIXTURE. The event topology mirrors one user-reported + // local live smoke run: two events before the start ack, one accounted + // request, two cumulative usage snapshots, one final completion_result + // message, then the observer stops. This is not a recorded provider + // artifact. Token counts are the reported aggregates. Timings are not + // reproduced. + registerTestFixture("pr1909-smoke-topology-synthetic", { + identity: { + name: "pr1909-smoke-topology-synthetic", + version: "0.1.0", + description: + "SYNTHETIC reproduction of one reported live smoke topology. Not a recorded provider artifact.", + }, + }) + const f = harness("no-events", { + onPause: (count) => { + if (count !== 1) return + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { + ts: 2, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai" }), + }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "updated", + message: { + ts: 2, + type: "say", + say: "api_req_started", + text: JSON.stringify({ apiProtocol: "openai", tokensIn: 2719, tokensOut: 29 }), + }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 2719, totalTokensOut: 12, totalCost: 0, contextTokens: 2719 }, + {}, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 2719, totalTokensOut: 29, totalCost: 0, contextTokens: 2748 }, + { attempt_completion: { attempts: 1, failures: 0 } }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 3, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + }, + }) + f.emit({ eventName: RooCodeEventName.TaskStarted, payload: ["root"] }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "text", text: "synthetic early event" }, + }, + ], + }) + let completionOutput = false + const result = await runAgentFixture(f.runtime, { + fixtureName: "pr1909-smoke-topology-synthetic", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => { + if ( + event.eventName === RooCodeEventName.Message && + event.payload[0].message.say === "completion_result" + ) + completionOutput = true + }, + shouldStop: () => completionOutput, + }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStopAfterCompletionMessage") + assert.equal(result.execution.completionMessageSeen, true) + assert.equal(result.execution.sessionLost, false) + assert.equal(result.execution.taskAccepted, true) + assert.equal(result.teardown.status, "ok") + assert.equal(result.teardown.cleanup, "done") + const trace = writtenTrace(f) + // Seven task events plus the counted TaskStartResponse envelope. + assert.equal(trace.eventFlow.observed, 8) + assert.equal(trace.eventFlow.bufferedBeforeAck, 2) + assert.equal(trace.eventFlow.replayedAfterAck, 2) + assert.equal(trace.eventFlow.malformed, 0) + assert.equal(trace.eventFlow.foreignTaskEvents, 0) + assert.equal(trace.accounting.requestCount, 1) + assert.deepEqual(trace.accounting.requests[0]?.tokensIn, { available: true, value: 2719 }) + assert.deepEqual(trace.accounting.requests[0]?.tokensOut, { available: true, value: 29 }) + assert.equal(trace.accounting.requests[0]?.messageCount, 2) + assert.equal(trace.accounting.usage.snapshotsObserved, 2) + assert.equal(trace.accounting.usage.lastSnapshot?.source, "taskTokenUsageUpdated") + assert.equal(trace.accounting.usage.lastSnapshot?.tokensOut, 29) + assert.deepEqual(trace.accounting.toolSnapshot.snapshot, { + attempt_completion: { attempts: 1, failures: 0 }, + }) + assert.equal(trace.accounting.completeness.status, "complete") + assert.deepEqual(trace.accounting.completeness.reasons, []) + // The drain is bound to the accepted-completion event. Without it, no + // drain window runs. + assert.equal(trace.execution.timings.usageDrainMs, undefined) + const serialized = JSON.stringify(trace) + assert.ok(serialized.includes("pr1909-smoke-topology-synthetic")) + assert.ok(!serialized.includes(CREDENTIAL)) + assert.ok(!serialized.includes("synthetic early event")) + assert.ok(!serialized.includes("synthetic final answer")) + assert.ok(!serialized.includes("root")) + }) + + it("keeps invalid completion_result variants observed but not marked as the completion message", async () => { + registerTestFixture("controller-invalid-completion") + const f = harness("no-events") + // One message per rejection condition of the completion filter: a + // partial stream, whitespace-only final text, and the ask-side + // completion_result prompt. + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "partial output", partial: true }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 2, type: "say", say: "completion_result", text: " \n\t " }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 3, type: "ask", ask: "completion_result", text: "awaiting feedback" }, + }, + ], + }) + const accepted: TaskEvent[] = [] + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-invalid-completion", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => accepted.push(event), + shouldStop: () => true, + }, + }) + const trace = writtenTrace(f) + // Three task events plus the counted TaskStartResponse envelope. + assert.equal(accepted.length, 3) + assert.equal(trace.eventFlow.observed, 4) + assert.equal(trace.eventFlow.replayedAfterAck, 3) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.completionMessageSeen, false) + assert.equal(trace.execution.stopReason, "adapterStop") + assert.equal(trace.execution.completionMessageSeen, false) + assert.ok(JSON.stringify(trace).includes('"completionMessageSeen":false')) + }) + + it("keeps completion output non-completing on a run timeout", async () => { + registerTestFixture("controller-output-timeout", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 200, + usageDrainMs: 100, + teardownStepMs: 100, + }, + }) + const f = harness("no-events", { + onPause: (count) => { + if (count !== 1) return + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + }, + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-output-timeout", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "TASK_RUN_TIMEOUT") + assert.equal(result.execution.stopReason, "runTimeout") + assert.equal(result.execution.completionMessageSeen, true) + assert.equal(writtenTrace(f).accounting.usage.snapshotsObserved, 0) + }) + + it("keeps a manual interrupt a session loss even after completion output", async () => { + registerTestFixture("controller-manual-stop") + const f = harness("no-events", { + onPause: (count) => { + if (count === 1) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "synthetic final answer", + }, + }, + ], + }) + if (count === 2) f.interrupt() + }, + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-manual-stop", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "SESSION_LOST") + assert.equal(result.execution.stopReason, "sessionLost") + assert.equal(result.execution.completionMessageSeen, true) + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + const trace = writtenTrace(f) + assert.ok(trace.accounting.completeness.reasons.includes("sessionLost")) + }) + + it("keeps a confirmed completion when the session dies in the same window", async () => { + registerTestFixture("controller-completion-outranks-loss") + const f = harness("success", { + // The completion event lands, then the session dies, before the run + // loop can poll again. + onPause: (count) => { + if (count === 1) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + }, + disconnectOnPause: 1, + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-completion-outranks-loss", + credential: CREDENTIAL, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.stopReason, "taskCompleted") + assert.equal(result.execution.sessionLost, true) + const trace = writtenTrace(f) + assert.ok(trace.accounting.completeness.reasons.includes("sessionLost")) + // A loss in the same window skips the drain and keeps the caveat. + assert.equal(trace.execution.timings.usageDrainMs, undefined) + }) + + it("keeps a confirmed completion when the observer requests a stop", async () => { + registerTestFixture("controller-completion-outranks-stop") + const f = harness("no-events") + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" }, + }, + ], + }) + f.emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "root", + { totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 }, + {}, + { isSubtask: false }, + ], + }) + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-completion-outranks-stop", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "COMPLETED") + assert.equal(result.execution.stopReason, "taskCompleted") + assert.equal(result.execution.completionMessageSeen, true) + assert.equal(result.execution.sessionLost, false) + assert.equal(result.teardown.status, "ok") + }) +}) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts new file mode 100644 index 0000000000..5c12961db3 --- /dev/null +++ b/scripts/agent-evals/runner.ts @@ -0,0 +1,421 @@ +import { randomUUID } from "node:crypto" +import { RooCodeEventName, TaskCommandName, taskEventSchema, type TaskCommand, type TaskEvent } from "@roo-code/types" +import { configurationMatchesToolPolicy, type AgentFixtureWorkspaceFile } from "./contract.ts" +import { getFixture } from "./registry.ts" +import { createTaskStartWaiter, type TaskStartWaiter } from "./task-start.ts" +import { + TRACE_KIND, + TRACE_NOTES, + TRACE_SCHEMA_VERSION, + createAccountant, + digestProjection, + eventTaskId, + identityUnavailable, + type Accountant, + type AgentEventFlow, + type AgentExecutionCode, + type AgentStopReason, + type AgentTeardown, + type AgentTrace, + type TraceRunIdentity, +} from "./trace.ts" + +export interface AgentHost { + readonly exited: boolean + onExit(listener: () => void): void + signal(signal: NodeJS.Signals): void +} + +export interface AgentClient { + readonly isReady: boolean + onTaskEvent(listener: (event: unknown) => void): void + onDisconnect(listener: () => void): void + sendCommand(command: TaskCommand): void + disconnect(): void +} + +// The adapter owns processes, sockets, files and credentials. The controller +// only sees these operations, so tests can drive it with synthetic IPC and a +// deterministic clock. +export interface AgentRuntime { + launch(workspaceFiles: readonly AgentFixtureWorkspaceFile[]): Promise + connect(): AgentClient + cleanup(): Promise + writeArtifact(fileName: string, value: unknown): Promise + onInterrupt(listener: () => void): () => void + now(): number + pause(ms: number): Promise + readIdentity?(): Promise +} + +export interface AgentRunObservers { + onAcceptedEvent?(event: TaskEvent, taskId: string): void + shouldStop?(): boolean +} + +export interface AgentRunOptions { + fixtureName: string + credential: string + observers?: AgentRunObservers +} + +export interface AgentRunResult { + execution: { + code: AgentExecutionCode + stopReason: AgentStopReason + activated: boolean + taskAccepted: boolean + sessionLost: boolean + completionMessageSeen: boolean + } + teardown: AgentTeardown + trace: AgentTrace +} + +const EVENT_BUFFER_LIMIT = 256 + +type BufferedEvent = { kind: "parsed"; event: TaskEvent } + +const traceSettings = (fixtureProjection: Record, credential: string): AgentTrace["settings"] => { + try { + const projectionJson = JSON.stringify(fixtureProjection) + if (projectionJson === undefined) return { available: false, reason: "projectionNotSerializable" } + if (credential.length > 0 && projectionJson.includes(credential)) + return { available: false, reason: "credentialInProjection" } + return { + available: true, + projection: JSON.parse(projectionJson) as Record, + digest: digestProjection(fixtureProjection), + } + } catch { + return { available: false, reason: "projectionNotSerializable" } + } +} + +export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOptions): Promise { + const fixture = getFixture(options.fixtureName) + const started = runtime.now() + let code: AgentExecutionCode = "SETUP_FAILED" + let stopReason: AgentStopReason = "setupError" + let activated = false + let accepted = false + let sessionLost = false + // Frozen at the end of the task run phase, before teardown can terminate + // the host. Teardown exits are reported in the teardown section only. + let runSessionLost = false + // Set when the session dies inside the post-completion usage drain. + let lostDuringDrain = false + let acceptedTaskId: string | undefined + let completedSeen = false + // True once the final completion_result say message for the accepted task + // arrived. This is model output, not the accepted-completion event. It + // never produces COMPLETED; it only refines how a stop is reported. + let completionMessageSeen = false + // The reason must name what actually stopped evaluation. "notEvaluated" + // means the projection was never inspected; it is replaced below. + let settings: AgentTrace["settings"] = { available: false, reason: "notEvaluated" } + let host: AgentHost | undefined + let client: AgentClient | undefined + let waiter: TaskStartWaiter | undefined + let teardown: AgentTeardown = { + status: "ok", + hostExited: true, + closeTaskSent: false, + escalatedSignals: [], + cleanup: "notNeeded", + failures: [], + } + const timings: AgentTrace["execution"]["timings"] = { totalMs: 0 } + const flow: AgentEventFlow = { + observed: 0, + malformed: 0, + foreignTaskEvents: 0, + bufferedBeforeAck: 0, + replayedAfterAck: 0, + eventBufferLimit: EVENT_BUFFER_LIMIT, + bufferOverflow: false, + droppedByOverflow: 0, + } + const buffer: BufferedEvent[] = [] + let buffering = false + const accountant: Accountant = createAccountant(() => runtime.now() - started) + + async function until(check: () => boolean, timeoutMs: number, stopOnLoss = true): Promise { + const deadline = runtime.now() + timeoutMs + while (!check() && (!stopOnLoss || !sessionLost) && runtime.now() < deadline) await runtime.pause(100) + } + + const markLost = () => { + sessionLost = true + waiter?.onExit() + } + + const unsubscribe = runtime.onInterrupt(() => { + markLost() + try { + host?.signal("SIGTERM") + } catch { + /* Teardown retries independently. */ + } + }) + + const dispatch = (event: TaskEvent): void => { + const taskId = eventTaskId(event) + if (acceptedTaskId === undefined || taskId !== acceptedTaskId) { + flow.foreignTaskEvents++ + return + } + accountant.observe(event) + if (event.eventName === RooCodeEventName.Message) { + const message = event.payload[0].message + if ( + message.type === "say" && + message.say === "completion_result" && + message.partial !== true && + typeof message.text === "string" && + message.text.trim().length > 0 + ) + completionMessageSeen = true + } + if (event.eventName === RooCodeEventName.TaskCompleted) completedSeen = true + options.observers?.onAcceptedEvent?.(event, acceptedTaskId) + } + + const pushBuffer = (item: BufferedEvent): void => { + if (buffer.length >= EVENT_BUFFER_LIMIT) { + flow.bufferOverflow = true + flow.droppedByOverflow++ + return + } + buffer.push(item) + flow.bufferedBeforeAck++ + } + + const handleEvent = (raw: unknown): void => { + flow.observed++ + if ( + typeof raw === "object" && + raw !== null && + (raw as { eventName?: unknown }).eventName === RooCodeEventName.TaskStartResponse + ) { + waiter?.onTaskEvent(raw) + return + } + const parsed = taskEventSchema.safeParse(raw) + if (!parsed.success) { + flow.malformed++ + return + } + if (buffering) { + pushBuffer({ kind: "parsed", event: parsed.data }) + return + } + dispatch(parsed.data) + } + + const replay = (): void => { + buffering = false + for (const item of buffer.splice(0)) { + flow.replayedAfterAck++ + dispatch(item.event) + } + } + + try { + const credential = options.credential + if (!credential) { + stopReason = "credentialMissing" + settings = { available: false, reason: "credentialMissing" } + } else { + settings = traceSettings(fixture.settingsProjection, credential) + const configuration = fixture.buildConfiguration(credential) + if (settings.available && !configurationMatchesToolPolicy(configuration, fixture.toolPolicy)) + settings = { available: false, reason: "configurationViolatesToolPolicy" } + if (!settings.available) { + stopReason = "fixtureRejected" + } else { + host = await runtime.launch(fixture.workspace.files) + host.onExit(markLost) + // An interrupt during launch must not connect or send a task. + if (sessionLost) throw new Error("startup interrupted") + client = runtime.connect() + client.onDisconnect(() => { + sessionLost = true + waiter?.onDisconnect() + }) + client.onTaskEvent(handleEvent) + buffering = true + const activationStart = runtime.now() + await until(() => client?.isReady === true, fixture.limits.activationTimeoutMs) + activated = client.isReady + timings.activationMs = runtime.now() - activationStart + if (!activated) { + code = "ACTIVATION_FAILED" + stopReason = "activationFailed" + } else { + if (sessionLost) throw new Error("startup interrupted") + const requestId = randomUUID() + waiter = createTaskStartWaiter({ requestId, timeoutMs: fixture.limits.taskStartTimeoutMs }) + // A synchronous send failure must not leave a rejected timer promise. + void waiter.result.catch(() => {}) + const sentAt = runtime.now() + client.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { + requestId, + text: fixture.prompt, + newTab: false, + configuration, + }, + }) + try { + acceptedTaskId = await waiter.result + accepted = true + } catch { + code = "TASK_START_FAILED" + stopReason = "startNotAccepted" + } + if (accepted) { + timings.taskStartMs = runtime.now() - sentAt + replay() + const runStart = runtime.now() + await until( + () => sessionLost || completedSeen || options.observers?.shouldStop?.() === true, + fixture.limits.taskRunTimeoutMs, + ) + runSessionLost = sessionLost + timings.taskRunMs = runtime.now() - runStart + if (completedSeen && !runSessionLost) { + // Fixed bound; ends early only on session loss. + const drainStart = runtime.now() + await until(() => false, fixture.limits.usageDrainMs) + timings.usageDrainMs = runtime.now() - drainStart + // A loss inside the drain window can truncate late + // usage events. The run code stays COMPLETED, but the + // accounting must carry the caveat. + lostDuringDrain = sessionLost + } + const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true + if (completedSeen) { + // A confirmed accepted completion outranks a stop + // request or a loss observed in the same window. + code = "COMPLETED" + stopReason = "taskCompleted" + } else if (runSessionLost) { + code = "SESSION_LOST" + stopReason = "sessionLost" + } else if (stopped && completionMessageSeen) { + // The observer stopped after the model's final + // completion message. The accepted-completion event + // never fired, so this is not COMPLETED. + code = "STOPPED" + stopReason = "adapterStopAfterCompletionMessage" + } else if (stopped) { + code = "STOPPED" + stopReason = "adapterStop" + } else { + code = "TASK_RUN_TIMEOUT" + stopReason = "runTimeout" + } + } + } + } + } + } catch { + // Never expose thrown provider, filesystem or host messages. + } finally { + waiter?.onExit() + const teardownStart = runtime.now() + const attempt = async (step: string, operation: () => unknown) => { + try { + await operation() + } catch { + teardown.failures.push(step) + } + } + if (client?.isReady) { + teardown.closeTaskSent = true + await attempt("closeTask", () => client?.sendCommand({ commandName: TaskCommandName.CloseTask })) + } + if (host && !host.exited) { + await attempt("hostTermination", () => + until(() => host?.exited === true, fixture.limits.teardownStepMs, false), + ) + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + if (host.exited) break + await attempt("hostTermination", () => host?.signal(signal)) + teardown.escalatedSignals.push(signal) + await attempt("hostTermination", () => + until(() => host?.exited === true, fixture.limits.teardownStepMs, false), + ) + } + } + teardown.hostExited = host ? host.exited : true + await attempt("disconnect", () => client?.disconnect()) + if (!host || host.exited) { + teardown.cleanup = "done" + await attempt("cleanup", () => runtime.cleanup()) + } else { + teardown.cleanup = "skippedLiveHost" + } + await attempt("unsubscribe", () => unsubscribe()) + if (teardown.failures.length > 0 || (host !== undefined && !host.exited)) teardown.status = "failed" + timings.teardownMs = runtime.now() - teardownStart + } + + const finishedAt = runtime.now() + timings.totalMs = finishedAt - started + let identity: TraceRunIdentity + try { + identity = (await runtime.readIdentity?.()) ?? identityUnavailable() + } catch { + identity = identityUnavailable() + } + const accounting = accountant.snapshot({ + accepted, + sessionLost: runSessionLost, + sessionLostDuringDrain: lostDuringDrain, + bufferOverflow: flow.bufferOverflow, + malformedEvents: flow.malformed, + foreignTaskEvents: flow.foreignTaskEvents, + }) + const trace: AgentTrace = { + traceSchemaVersion: TRACE_SCHEMA_VERSION, + kind: TRACE_KIND, + run: { + runId: randomUUID(), + fixture: { name: fixture.identity.name, version: fixture.identity.version }, + startedAt: new Date(started).toISOString(), + finishedAt: new Date(finishedAt).toISOString(), + durationMs: timings.totalMs, + identity, + }, + settings, + execution: { + code, + stopReason, + activated, + taskAccepted: accepted, + sessionLost: runSessionLost, + completionMessageSeen, + timings, + }, + eventFlow: flow, + accounting, + teardown, + notes: TRACE_NOTES, + } + await runtime.writeArtifact("trace.json", trace) + return { + execution: { + code, + stopReason, + activated, + taskAccepted: accepted, + sessionLost: runSessionLost, + completionMessageSeen, + }, + teardown, + trace, + } +} diff --git a/scripts/agent-evals/task-start.ts b/scripts/agent-evals/task-start.ts new file mode 100644 index 0000000000..2aafaaea57 --- /dev/null +++ b/scripts/agent-evals/task-start.ts @@ -0,0 +1,96 @@ +import { RooCodeEventName, taskStartResponseSchema } from "@roo-code/types" + +export type TaskStartWaiter = { + onTaskEvent: (event: unknown) => string | undefined + onDisconnect: () => void + onExit: () => void + result: Promise +} + +const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null + +// Correlates one StartNewTask request with its sanitized response, similar to +// JSON-RPC: the sender generates the requestId and the reply copies it back. +// The waiter settles exactly once. A matched success resolves with the +// accepted task ID. A matched failure, a timeout, an IPC disconnect, or a +// VS Code exit rejects. Unmatched or malformed responses never settle it. +export function createTaskStartWaiter({ + requestId, + timeoutMs, +}: { + requestId: string + timeoutMs: number +}): TaskStartWaiter { + let resolveResult!: (taskId: string) => void + let rejectResult!: (error: Error) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + + let settled = false + let timer: NodeJS.Timeout | undefined + + const settle = (finish: () => void): void => { + if (settled) { + return + } + settled = true + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + finish() + } + + const rejectWith = (message: string): void => { + settle(() => rejectResult(new Error(message))) + } + + timer = setTimeout(() => { + rejectWith(`task start response missing after ${timeoutMs}ms (requestId=${requestId})`) + }, timeoutMs) + + return { + onTaskEvent(event: unknown): string | undefined { + if (settled) return undefined + if (!isRecord(event) || event["eventName"] !== RooCodeEventName.TaskStartResponse) { + return undefined + } + + const payload = event["payload"] + if (!Array.isArray(payload) || payload.length === 0) { + return undefined + } + + const parsed = taskStartResponseSchema.safeParse(payload[0]) + if (!parsed.success || parsed.data.requestId !== requestId) { + return undefined + } + + const response = parsed.data + if (response.success) { + const acceptedTaskId = response.taskId + settle(() => resolveResult(acceptedTaskId)) + return acceptedTaskId + } + + const { errorCode, errorMessage, stage } = response + // The stage is a fixed sanitized token from the response schema; it + // names the startup phase that stopped responding. + const stageDetail = stage === undefined ? "" : ` (stage=${stage})` + rejectWith(`${errorCode}: ${errorMessage}${stageDetail}`) + return undefined + }, + + onDisconnect(): void { + rejectWith("IPC disconnected before the task start response arrived") + }, + + onExit(): void { + rejectWith("VS Code exited before the task start response arrived") + }, + + result, + } +} diff --git a/scripts/agent-evals/trace.test.ts b/scripts/agent-evals/trace.test.ts new file mode 100644 index 0000000000..d669fced05 --- /dev/null +++ b/scripts/agent-evals/trace.test.ts @@ -0,0 +1,227 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, type TaskEvent } from "@roo-code/types" +import { + TRACE_NOTES, + createAccountant, + digestProjection, + eventTaskId, + parseTaskEvent, + stableStringify, +} from "./trace.ts" + +let clock = 0 +const fresh = () => createAccountant(() => ++clock) + +const message = (ts: number, text: string | undefined): TaskEvent => ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId: "root", action: "created", message: { ts, type: "say", say: "api_req_started", text } }], +}) + +const usageEvent = (tokensIn: number, tokensOut: number, cacheWrites?: number): TaskEvent => ({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { + totalTokensIn: tokensIn, + totalTokensOut: tokensOut, + totalCost: 0, + contextTokens: 11, + ...(cacheWrites === undefined ? {} : { totalCacheWrites: cacheWrites }), + }, + {}, + ], +}) + +describe("trace request accounting", () => { + it("upserts one request row per message timestamp across created and updated events", () => { + const accountant = fresh() + accountant.observe(message(1, JSON.stringify({ apiProtocol: "openai" }))) + accountant.observe( + message( + 1, + JSON.stringify({ + apiProtocol: "openai", + tokensIn: 10, + tokensOut: 4, + cacheWrites: 2, + cacheReads: 3, + cost: 0.5, + }), + ), + ) + accountant.observe( + message( + 1, + JSON.stringify({ + apiProtocol: "openai", + tokensIn: 10, + tokensOut: 4, + cancelReason: "streaming_failed", + streamingFailedMessage: "private provider error", + }), + ), + ) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.requestCount, 1) + const row = accounting.requests[0] + assert.ok(row) + assert.equal(row.messageCount, 3) + assert.deepEqual(row.tokensIn, { available: true, value: 10 }) + assert.deepEqual(row.tokensOut, { available: true, value: 4 }) + assert.deepEqual(row.cacheWrites, { available: true, value: 2 }) + assert.deepEqual(row.cacheReads, { available: true, value: 3 }) + assert.deepEqual(row.cost, { available: true, value: 0.5 }) + assert.equal(row.cancelReason, "streaming_failed") + assert.equal(row.streamingFailed, true) + assert.ok(!JSON.stringify(accounting).includes("private provider error")) + }) + + it("marks requests without usage updates instead of filling zeros", () => { + const accountant = fresh() + accountant.observe(message(1, JSON.stringify({ tokensIn: 10, tokensOut: 2 }))) + accountant.observe(message(2, JSON.stringify({ apiProtocol: "openai" }))) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.requestsMissingUsage, 1) + assert.equal(accounting.completeness.status, "partial") + assert.ok(accounting.completeness.reasons.includes("finalRequestMissingUsage")) + const row = accounting.requests[1] + assert.ok(row) + assert.deepEqual(row.tokensIn, { available: false, reason: "notReported" }) + assert.deepEqual(row.tokensOut, { available: false, reason: "notReported" }) + }) + + it("keeps only the latest cumulative snapshot and never sums snapshots", () => { + const accountant = fresh() + accountant.observe(usageEvent(10, 5)) + accountant.observe(usageEvent(25, 9)) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.usage.snapshotsObserved, 2) + assert.equal(accounting.usage.lastSnapshot?.tokensIn, 25) + assert.equal(accounting.usage.lastSnapshot?.tokensOut, 9) + assert.equal(accounting.usage.available, true) + assert.ok(!("tokensInSum" in (accounting.usage as unknown as Record))) + }) + + it("reports missing cache fields and zero tokens as caveated, not measured", () => { + const accountant = fresh() + accountant.observe(usageEvent(0, 0)) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.usage.lastSnapshot?.tokensIn, 0) + assert.deepEqual(accounting.usage.lastSnapshot?.cacheWrites, { available: false, reason: "notReported" }) + assert.ok(TRACE_NOTES.some((note) => note.includes("Zero token or cost values"))) + }) + + it("counts tool failure events by name without storing error payloads", () => { + const accountant = fresh() + accountant.observe({ + eventName: RooCodeEventName.TaskToolFailed, + payload: ["root", "execute_command", "private failure detail that must never persist"], + }) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.toolFailureEvents, 1) + assert.deepEqual(accounting.toolFailuresByTool, { execute_command: 1 }) + assert.ok(!JSON.stringify(accounting).includes("private failure detail")) + }) + + it("counts retry marker messages separately from usage rows", () => { + const accountant = fresh() + accountant.observe({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { ts: 9, type: "say", say: "api_req_retried", text: "[retry marker]" }, + }, + ], + }) + const accounting = accountant.snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.apiReqRetriedMarkers, 1) + assert.equal(accounting.requestCount, 0) + assert.ok(TRACE_NOTES.some((note) => note.includes("not an HTTP retry count"))) + }) + + it("reports no-usage completeness with a fixed reason", () => { + const accounting = fresh().snapshot({ + accepted: true, + sessionLost: false, + bufferOverflow: false, + malformedEvents: 0, + foreignTaskEvents: 0, + }) + assert.equal(accounting.usage.available, false) + assert.equal(accounting.usage.unavailableReason, "noUsageEventsObserved") + assert.ok(accounting.completeness.reasons.includes("noUsageEventsObserved")) + }) +}) + +describe("trace helpers", () => { + it("documents the settings projection as the only configuration content", () => { + assert.ok(TRACE_NOTES.some((note) => note.includes("nonsecret settings projection"))) + }) + + it("documents the two completion signals and the observer stop reasons", () => { + assert.ok(TRACE_NOTES.some((note) => note.includes("taskCompleted event"))) + assert.ok(TRACE_NOTES.some((note) => note.includes("completion_result"))) + assert.ok(TRACE_NOTES.some((note) => note.includes("adapterStopAfterCompletionMessage"))) + }) + + it("digests projections stably regardless of key order", () => { + assert.equal(digestProjection({ a: 1, b: { c: 2 } }), digestProjection({ b: { c: 2 }, a: 1 })) + assert.notEqual(digestProjection({ a: 1 }), digestProjection({ a: 2 })) + assert.equal( + stableStringify({ b: 1, a: [2, { z: 3, y: 4 }] }), + stableStringify({ a: [2, { y: 4, z: 3 }], b: 1 }), + ) + }) + + it("parses events defensively and reads task identities", () => { + assert.equal(eventTaskId(parseTaskEvent(usageEvent(1, 1)) as TaskEvent), "root") + assert.equal( + eventTaskId(parseTaskEvent({ eventName: RooCodeEventName.TaskAborted, payload: ["root"] }) as TaskEvent), + "root", + ) + assert.equal( + eventTaskId(parseTaskEvent({ eventName: RooCodeEventName.ModelsResponse, payload: [{}] }) as TaskEvent), + undefined, + ) + assert.equal(parseTaskEvent({ eventName: "bogus" }), undefined) + }) +}) diff --git a/scripts/agent-evals/trace.ts b/scripts/agent-evals/trace.ts new file mode 100644 index 0000000000..2cdb2299ca --- /dev/null +++ b/scripts/agent-evals/trace.ts @@ -0,0 +1,409 @@ +import { createHash } from "node:crypto" +import { RooCodeEventName, taskEventSchema, type TaskEvent } from "@roo-code/types" + +export const TRACE_SCHEMA_VERSION = 1 +export const TRACE_KIND = "agent-evals-trace" + +export type Measured = { available: true; value: T } | { available: false; reason: string } +export type MeasuredString = Measured +export type Accountant = ReturnType + +export const measured = (value: T | undefined | null, reason: string): Measured => + value === undefined || value === null ? { available: false, reason } : { available: true, value } + +export interface TraceRunIdentity { + commit: MeasuredString + extensionVersion: MeasuredString + vscodeTarget: MeasuredString + nodeVersion: string + platform: string + arch: string +} + +export const identityUnavailable = (reason = "identityUnavailable"): TraceRunIdentity => ({ + commit: { available: false, reason }, + extensionVersion: { available: false, reason }, + vscodeTarget: { available: false, reason }, + nodeVersion: process.version, + platform: process.platform, + arch: process.arch, +}) + +export type AgentExecutionCode = + | "COMPLETED" + | "STOPPED" + | "SETUP_FAILED" + | "ACTIVATION_FAILED" + | "TASK_START_FAILED" + | "SESSION_LOST" + | "TASK_RUN_TIMEOUT" + +export type AgentStopReason = + | "taskCompleted" + | "adapterStop" + | "adapterStopAfterCompletionMessage" + | "runTimeout" + | "sessionLost" + | "startNotAccepted" + | "activationFailed" + | "credentialMissing" + | "fixtureRejected" + | "setupError" + +export interface AgentEventFlow { + observed: number + malformed: number + foreignTaskEvents: number + bufferedBeforeAck: number + replayedAfterAck: number + eventBufferLimit: number + bufferOverflow: boolean + droppedByOverflow: number +} + +export interface TraceRequestRow { + index: number + ts: number + apiProtocol: MeasuredString + tokensIn: Measured + tokensOut: Measured + cacheWrites: Measured + cacheReads: Measured + cost: Measured + cancelReason?: string + streamingFailed: boolean + firstObservedMs: number + lastUpdatedMs: number + messageCount: number +} + +export interface CumulativeUsageSnapshot { + source: "taskTokenUsageUpdated" | "taskCompleted" + observedAtMs: number + tokensIn: number + tokensOut: number + cacheWrites: Measured + cacheReads: Measured + cost: number + contextTokens: number +} + +export interface TraceUsage { + available: boolean + unavailableReason?: string + snapshotsObserved: number + lastSnapshot?: CumulativeUsageSnapshot +} + +export interface TraceToolSnapshot { + snapshot?: Record + observedAtMs?: number +} + +export interface TraceAccounting { + requests: TraceRequestRow[] + requestCount: number + requestsMissingUsage: number + apiReqRetriedMarkers: number + usage: TraceUsage + toolSnapshot: TraceToolSnapshot + toolFailureEvents: number + toolFailuresByTool: Record + completeness: { status: "complete" | "partial"; reasons: string[] } +} + +export interface AgentTeardown { + status: "ok" | "failed" + hostExited: boolean + closeTaskSent: boolean + escalatedSignals: string[] + cleanup: "done" | "skippedLiveHost" | "failed" | "notNeeded" + failures: string[] +} + +export interface AgentTrace { + traceSchemaVersion: typeof TRACE_SCHEMA_VERSION + kind: typeof TRACE_KIND + run: { + runId: string + fixture: { name: string; version: string } + startedAt: string + finishedAt: string + durationMs: number + identity: TraceRunIdentity + } + settings: { + available: boolean + reason?: string + projection?: Record + digest?: string + } + execution: { + code: AgentExecutionCode + stopReason: AgentStopReason + activated: boolean + taskAccepted: boolean + sessionLost: boolean + // True when the final completion_result message for the accepted task + // was observed. This is model output, not acceptance. + completionMessageSeen: boolean + timings: { + activationMs?: number + taskStartMs?: number + taskRunMs?: number + usageDrainMs?: number + teardownMs?: number + totalMs: number + } + } + eventFlow: AgentEventFlow + accounting: TraceAccounting + teardown: AgentTeardown + notes: string[] +} + +// Fixed vocabulary. Reviewers can rely on these sentences across trace versions. +export const TRACE_NOTES = [ + "Usage numbers come from extension events. The trace never sums cumulative snapshots.", + "Zero token or cost values can be defaults from the adapter or task layer. They do not prove the provider measured zero.", + "contextTokens is an extension context estimate, not a provider measurement.", + "Request rows use per-request fields from api_req_started message updates, keyed by message timestamp.", + "apiReqRetriedMarkers counts retry marker messages. It is not an HTTP retry count.", + "The trace is metadata only. Prompts, model output, reasoning, tool arguments, error payloads, and storage paths are excluded. Provider settings appear only as the nonsecret settings projection.", + "Missing measurements carry a reason. They are never filled with zero.", + "execution.code COMPLETED requires the extension taskCompleted event. The extension emits it only after the completion result is accepted, or a delegated subtask hands back, and the turn is persisted.", + "The final completion_result message is model output, not acceptance. execution.completionMessageSeen records it and never produces COMPLETED.", + "An observer stop after that message is stopReason adapterStopAfterCompletionMessage with code STOPPED. An observer stop without it stays adapterStop.", +] + +const sortValue = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sortValue) + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sortValue(entry)]), + ) + } + return value +} + +export const stableStringify = (value: unknown): string => JSON.stringify(sortValue(value)) + +export const digestProjection = (projection: Record): string => + createHash("sha256").update(stableStringify(projection)).digest("hex") + +interface RequestRowState { + index: number + ts: number + apiProtocol?: string + tokensIn?: number + tokensOut?: number + cacheWrites?: number + cacheReads?: number + cost?: number + cancelReason?: string + streamingFailed: boolean + firstObservedMs: number + lastUpdatedMs: number + messageCount: number + hasUsage: boolean +} + +const MAX_REQUEST_TEXT_CHARS = 4_096 +const MAX_CANCEL_REASON_CHARS = 32 +const MAX_PROTOCOL_CHARS = 16 + +export interface AccountingInputs { + accepted: boolean + sessionLost: boolean + // True when the session died inside the post-completion usage drain. Late + // usage may be truncated, so the accounting carries its own caveat. + sessionLostDuringDrain?: boolean + bufferOverflow: boolean + malformedEvents: number + foreignTaskEvents: number +} + +export function createAccountant(nowMs: () => number): { + observe(event: TaskEvent): void + snapshot(inputs: AccountingInputs): TraceAccounting +} { + const rows = new Map() + let snapshotsObserved = 0 + let lastUsageSnapshot: CumulativeUsageSnapshot | undefined + let lastToolSnapshot: TraceToolSnapshot = {} + let apiReqRetriedMarkers = 0 + let toolFailureEvents = 0 + const toolFailuresByTool: Record = {} + + const usageSnapshot = ( + source: CumulativeUsageSnapshot["source"], + usage: { + totalTokensIn: number + totalTokensOut: number + totalCacheWrites?: number + totalCacheReads?: number + totalCost: number + contextTokens: number + }, + ): CumulativeUsageSnapshot => ({ + source, + observedAtMs: nowMs(), + tokensIn: usage.totalTokensIn, + tokensOut: usage.totalTokensOut, + cacheWrites: measured(usage.totalCacheWrites, "notReported"), + cacheReads: measured(usage.totalCacheReads, "notReported"), + cost: usage.totalCost, + contextTokens: usage.contextTokens, + }) + + const observe = (event: TaskEvent): void => { + if (event.eventName === RooCodeEventName.Message) { + const entry = event.payload[0] + const message = entry.message + if (message.type !== "say") return + if (message.say === "api_req_retried") { + apiReqRetriedMarkers++ + return + } + if (message.say !== "api_req_started") return + const ts = message.ts + const at = nowMs() + const row: RequestRowState = rows.get(ts) ?? { + index: rows.size, + ts, + streamingFailed: false, + firstObservedMs: at, + lastUpdatedMs: at, + messageCount: 0, + hasUsage: false, + } + row.messageCount++ + row.lastUpdatedMs = at + if (typeof message.text === "string" && message.text.length <= MAX_REQUEST_TEXT_CHARS) { + try { + const info = JSON.parse(message.text) as { + apiProtocol?: string + tokensIn?: number + tokensOut?: number + cacheWrites?: number + cacheReads?: number + cost?: number + cancelReason?: string + streamingFailedMessage?: string + } + if (typeof info.apiProtocol === "string" && info.apiProtocol.length <= MAX_PROTOCOL_CHARS) + row.apiProtocol = info.apiProtocol + if (typeof info.tokensIn === "number") { + row.tokensIn = info.tokensIn + row.hasUsage = true + } + if (typeof info.tokensOut === "number") { + row.tokensOut = info.tokensOut + row.hasUsage = true + } + if (typeof info.cacheWrites === "number") { + row.cacheWrites = info.cacheWrites + row.hasUsage = true + } + if (typeof info.cacheReads === "number") { + row.cacheReads = info.cacheReads + row.hasUsage = true + } + if (typeof info.cost === "number") row.cost = info.cost + if (typeof info.cancelReason === "string" && info.cancelReason.length <= MAX_CANCEL_REASON_CHARS) + row.cancelReason = info.cancelReason + // Presence only. The failure text itself is never stored. + if (typeof info.streamingFailedMessage === "string") row.streamingFailed = true + } catch { + // Unreadable request text stays an availability gap, not an error payload. + } + } + rows.set(ts, row) + return + } + if (event.eventName === RooCodeEventName.TaskTokenUsageUpdated) { + snapshotsObserved++ + lastUsageSnapshot = usageSnapshot("taskTokenUsageUpdated", event.payload[1]) + lastToolSnapshot = { snapshot: event.payload[2], observedAtMs: nowMs() } + return + } + if (event.eventName === RooCodeEventName.TaskCompleted) { + snapshotsObserved++ + lastUsageSnapshot = usageSnapshot("taskCompleted", event.payload[1]) + lastToolSnapshot = { snapshot: event.payload[2], observedAtMs: nowMs() } + return + } + if (event.eventName === RooCodeEventName.TaskToolFailed) { + const tool = event.payload[1] + toolFailureEvents++ + toolFailuresByTool[tool] = (toolFailuresByTool[tool] ?? 0) + 1 + } + } + + const snapshot = (inputs: AccountingInputs): TraceAccounting => { + const requests: TraceRequestRow[] = [...rows.values()] + .sort((a, b) => a.index - b.index) + .map((row) => ({ + index: row.index, + ts: row.ts, + apiProtocol: measured(row.apiProtocol, "notReported"), + tokensIn: measured(row.tokensIn, "notReported"), + tokensOut: measured(row.tokensOut, "notReported"), + cacheWrites: measured(row.cacheWrites, "notReported"), + cacheReads: measured(row.cacheReads, "notReported"), + cost: measured(row.cost, "notReported"), + cancelReason: row.cancelReason, + streamingFailed: row.streamingFailed, + firstObservedMs: row.firstObservedMs, + lastUpdatedMs: row.lastUpdatedMs, + messageCount: row.messageCount, + })) + const requestsMissingUsage = [...rows.values()].filter((row) => !row.hasUsage).length + const reasons: string[] = [] + if (!inputs.accepted) reasons.push("taskNotAccepted") + if (snapshotsObserved === 0) reasons.push("noUsageEventsObserved") + if (requestsMissingUsage > 0) reasons.push("finalRequestMissingUsage") + if (inputs.bufferOverflow) reasons.push("eventBufferOverflow") + if (inputs.malformedEvents > 0) reasons.push("malformedEventsObserved") + if (inputs.foreignTaskEvents > 0) reasons.push("foreignTaskEventsDropped") + if (inputs.sessionLost) reasons.push("sessionLost") + if (inputs.sessionLostDuringDrain) reasons.push("sessionLostDuringUsageDrain") + return { + requests, + requestCount: requests.length, + requestsMissingUsage, + apiReqRetriedMarkers, + usage: { + available: snapshotsObserved > 0, + unavailableReason: snapshotsObserved > 0 ? undefined : "noUsageEventsObserved", + snapshotsObserved, + lastSnapshot: lastUsageSnapshot, + }, + toolSnapshot: lastToolSnapshot, + toolFailureEvents, + toolFailuresByTool, + completeness: { status: reasons.length === 0 ? "complete" : "partial", reasons }, + } + } + + return { observe, snapshot } +} + +// Defense in depth for replay: schema-valid events only, and only their task +// identity is read. No payload content passes through this helper. +export const parseTaskEvent = (raw: unknown): TaskEvent | undefined => { + const parsed = taskEventSchema.safeParse(raw) + return parsed.success ? parsed.data : undefined +} + +export const eventTaskId = (event: TaskEvent): string | undefined => { + const first: unknown = (event.payload as readonly unknown[])[0] + if (typeof first === "string") return first + if (typeof first === "object" && first !== null && "taskId" in first) { + const taskId = (first as { taskId?: unknown }).taskId + if (typeof taskId === "string") return taskId + } + return undefined +} diff --git a/scripts/agent-evals/tsconfig.json b/scripts/agent-evals/tsconfig.json new file mode 100644 index 0000000000..55bddebe10 --- /dev/null +++ b/scripts/agent-evals/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "lib": ["ES2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./*.ts", "./*.mts"] +} diff --git a/scripts/solheim-smoke-workflow.test.mjs b/scripts/solheim-smoke-workflow.test.mjs new file mode 100644 index 0000000000..e14ec4a320 --- /dev/null +++ b/scripts/solheim-smoke-workflow.test.mjs @@ -0,0 +1,454 @@ +import assert from "node:assert/strict" +import fs from "node:fs" +import { URL, fileURLToPath } from "node:url" +import { describe, it } from "node:test" +import { spawn, spawnSync } from "node:child_process" +import os from "node:os" +import path from "node:path" +import { parse } from "yaml" + +// Parse configuration so comments, indentation, and key order are irrelevant. +// GitHub expressions remain strings: these checks protect the configured policy, +// while the recorder tests below exercise process behavior. +const workflow = parse( + fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8"), +) +const smoke = workflow.jobs.smoke +const steps = smoke.steps +const actionSteps = (action) => steps.filter((step) => step.uses?.startsWith(action + "@")) + +describe("Solheim provider smoke workflow policy", () => { + it("allows only manual runs on main behind the protected environment", () => { + assert.deepEqual(Object.keys(workflow.on), ["workflow_dispatch"]) + assert.deepEqual(Object.keys(workflow.jobs), ["smoke"]) + assert.equal(smoke.if, "github.ref == 'refs/heads/main'") + assert.equal(smoke.environment, "final-vscode-review-smoke") + assert.deepEqual(workflow.permissions, { contents: "read" }) + assert.equal(smoke.permissions, undefined) + }) + + it("checks out the triggering revision without persisting credentials", () => { + const checkouts = actionSteps("actions/checkout") + assert.equal(checkouts.length, 1) + assert.equal(checkouts[0].with.ref, "${{ github.sha }}") + assert.equal(checkouts[0].with["persist-credentials"], false) + }) + + it("exposes the provider credential only to the bounded provider step", () => { + const provider = steps.find((step) => step.id === "provider") + assert.ok(provider, "missing provider step") + assert.equal(provider.env.SOLHEIM_API_KEY, "${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }}") + assert.equal(workflow.env, undefined) + assert.equal(smoke.env, undefined) + // Inspect parsed values, excluding comments and display names. + const credentialSteps = steps.filter((step) => + JSON.stringify({ env: step.env, with: step.with, run: step.run }).includes("secrets."), + ) + assert.deepEqual(credentialSteps, [provider]) + assert.ok(provider["timeout-minutes"] > 0 && provider["timeout-minutes"] <= 8) + assert.ok(smoke["timeout-minutes"] > provider["timeout-minutes"] && smoke["timeout-minutes"] <= 25) + assert.equal(workflow.concurrency.group, "solheim-provider-smoke") + assert.equal(workflow.concurrency["cancel-in-progress"], false) + }) + + it("uploads metadata and the final recording separately, including after provider failure", () => { + const uploads = actionSteps("actions/upload-artifact") + assert.equal(uploads.length, 2) + const output = "${{ runner.temp }}/solheim-provider-smoke" + const paths = uploads.map((step) => + step.with.path + .trim() + .split("\n") + .map((line) => line.trim()), + ) + assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]]) + for (const step of uploads) { + assert.equal( + step.if, + "${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}", + ) + assert.equal(step.with["if-no-files-found"], "error") + assert.equal(step.with["retention-days"], 7) + assert.ok(steps.indexOf(step) > steps.findIndex((candidate) => candidate.id === "provider")) + } + assert.equal(new Set(uploads.map((step) => step.with.name)).size, 2) + }) + + it("pins remote actions to immutable commit revisions", () => { + for (const step of steps.filter((step) => step.uses && !step.uses.startsWith("./"))) { + const revision = step.uses.split("@")[1] + assert.match(revision ?? "", /^[0-9a-f]{40}$/, step.uses) + } + }) +}) + +// Runs record.sh with stub ffmpeg and ffprobe, so no display or codec is needed. +const defaultFfmpeg = (writes) => + `${writes ? 'for a; do :; done; echo data > "$a"' : ":"}\ntrap 'exit 0' TERM\nwhile true; do sleep 0.05; done` + +function stubRecorderBin(dir, { ffmpeg, ffmpegWrites = true, ffprobeOk = true } = {}) { + const bin = path.join(dir, "bin") + fs.mkdirSync(bin, { recursive: true }) + const stub = (name, body) => + fs.writeFileSync(path.join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 }) + // The stub ffmpeg writes the last argument (the recorder's output path), + // stops on TERM, and exits. + stub("ffmpeg", ffmpeg ?? defaultFfmpeg(ffmpegWrites)) + stub("ffprobe", ffprobeOk ? "exit 0" : "exit 1") + return bin +} + +function recorderPaths(dir) { + const outDir = path.join(dir, "out") + return { + outDir, + finalRecording: path.join(outDir, "smoke.mp4"), + } +} + +function recorderEnv(bin, outDir, providerMarker, extraEnv) { + return { + PATH: `${bin}:${process.env.PATH}`, + DISPLAY: ":99", + SOLHEIM_SMOKE_OUT_DIR: outDir, + ...(providerMarker ? { PROVIDER_MARKER: providerMarker } : {}), + ...extraEnv, + } +} + +const stopBudgetEnv = (stopBudget) => (stopBudget ? { SOLHEIM_SMOKE_STOP_BUDGET_S: String(stopBudget) } : {}) + +const recorderScript = fileURLToPath(new URL("./solheim-smoke/record.sh", import.meta.url)) + +function runRecorder( + args, + { + ffmpeg, + ffmpegWrites = true, + ffprobeOk = true, + seedFinal, + seedMetadata, + providerMarker, + stopBudget, + extraEnv, + } = {}, +) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir, { ffmpeg, ffmpegWrites, ffprobeOk }) + const paths = recorderPaths(dir) + fs.mkdirSync(paths.outDir, { recursive: true }) + if (seedFinal !== undefined) fs.writeFileSync(paths.finalRecording, seedFinal) + for (const name of seedMetadata ?? []) fs.writeFileSync(path.join(paths.outDir, name), "stale") + const result = spawnSync("bash", [recorderScript, ...args], { + env: recorderEnv(bin, paths.outDir, providerMarker, { ...stopBudgetEnv(stopBudget), ...extraEnv }), + encoding: "utf8", + timeout: 20_000, + }) + return { result, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) } +} + +function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, providerMarker, stopBudget } = {}) { + return new Promise((resolve) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir, { ffmpeg, ffprobeOk }) + const paths = recorderPaths(dir) + // The output directory is created here, before the recorder runs, so + // every process in the test observes the same directory object. + fs.mkdirSync(paths.outDir, { recursive: true }) + const child = spawn("bash", [recorderScript, ...args], { + env: recorderEnv(bin, paths.outDir, providerMarker, stopBudgetEnv(stopBudget)), + }) + let stdout = "" + child.stdout.on("data", (chunk) => { + stdout += chunk + }) + const watchdog = setTimeout(() => child.kill("SIGKILL"), 15_000) + const timer = setTimeout(() => child.kill("SIGTERM"), delayMs) + child.on("close", (code) => { + clearTimeout(watchdog) + clearTimeout(timer) + resolve({ code, stdout, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) }) + }) + }) +} + +const tempFilesLeft = (run) => fs.readdirSync(run.outDir).filter((name) => name.includes(".tmp")) + +const pidAlive = (pid) => { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +describe("record.sh exit and validation behavior", () => { + it("passes when the command succeeds and the recording is valid", () => { + const run = runRecorder(["true"]) + try { + assert.equal(run.result.status, 0) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + } + }) + it("keeps the exit code of a failing command and still publishes a valid recording", () => { + const run = runRecorder(["bash", "-c", "exit 7"]) + try { + assert.equal(run.result.status, 7) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + } finally { + run.cleanup() + } + }) + it("removes stale verdict and trace metadata at start", () => { + const run = runRecorder(["true"], { + seedMetadata: ["verdict.json", "trace.json", "verdict.json.1.tmp"], + }) + try { + for (const name of ["verdict.json", "trace.json", "verdict.json.1.tmp"]) + assert.equal(fs.existsSync(path.join(run.outDir, name)), false, name) + } finally { + run.cleanup() + } + }) + + it("refuses a supplied command while the provider credential is set", () => { + const run = runRecorder(["true"], { extraEnv: { SOLHEIM_API_KEY: "sk-test" } }) + try { + assert.equal(run.result.status, 2) + assert.match(run.result.stderr, /Refusing/) + } finally { + run.cleanup() + } + }) + + it("fails when the recording is missing", () => { + const run = runRecorder(["true"], { ffmpegWrites: false }) + try { + assert.equal(run.result.status, 1) + assert.match(run.result.stdout, /recording failed/) + assert.ok(!fs.existsSync(run.finalRecording)) + } finally { + run.cleanup() + } + }) + it("fails and cleans the temporary file when ffprobe rejects the recording", () => { + const run = runRecorder(["true"], { ffprobeOk: false }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + } + }) + it("keeps a failing command code when the recording is also invalid", () => { + const run = runRecorder(["bash", "-c", "exit 7"], { ffprobeOk: false }) + try { + assert.equal(run.result.status, 7) + assert.ok(!fs.existsSync(run.finalRecording)) + } finally { + run.cleanup() + } + }) + it("removes a stale final recording when the new recording fails", () => { + const run = runRecorder(["true"], { ffmpegWrites: false, seedFinal: "STALE" }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(run.finalRecording)) + } finally { + run.cleanup() + } + }) + it("replaces a stale final recording with the validated new recording", () => { + const run = runRecorder(["true"], { seedFinal: "STALE" }) + try { + assert.equal(run.result.status, 0) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + } finally { + run.cleanup() + } + }) + it("aborts before the command when the recorder dies at startup", () => { + const marker = path.join(os.tmpdir(), `record-sh-marker-${process.pid}-${Math.random()}`) + const run = runRecorder(["bash", "-c", 'echo ran > "$PROVIDER_MARKER"'], { + ffmpeg: 'for a; do :; done; echo partial > "$a"\nexit 1', + providerMarker: marker, + }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(marker)) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.match(run.result.stdout, /stopped on its own/) + } finally { + run.cleanup() + fs.rmSync(marker, { force: true }) + } + }) + it("rejects recorder output that stopped on its own with a failure status", () => { + const run = runRecorder(["bash", "-c", "sleep 2"], { + // The content is readable and would pass ffprobe, so the rejection + // must come from the recorder's own non-zero exit status. + ffmpeg: 'for a; do :; done\nsleep 0.7\necho data > "$a"\nexit 1', + }) + try { + assert.equal(run.result.status, 1) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.deepEqual(tempFilesLeft(run), []) + assert.match(run.result.stdout, /stopped on its own/) + } finally { + run.cleanup() + } + }) + it("publishes when the recorder stops on its own with success (size or time limit)", () => { + const run = runRecorder(["bash", "-c", "sleep 2"], { + ffmpeg: 'for a; do :; done\nsleep 0.7\necho data > "$a"\nexit 0', + }) + try { + assert.equal(run.result.status, 0) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + } finally { + run.cleanup() + } + }) +}) + +describe("record.sh signal teardown behavior", () => { + it("stops the recorder and the command on SIGTERM and keeps the signal exit code", async () => { + const marker = path.join(os.tmpdir(), `record-sh-marker-${process.pid}-${Math.random()}`) + const run = await runRecorderSignalled( + // The inner wait makes the stub command respond to TERM promptly, the + // way a signal-safe driver does. The trap exits so the "normal" write + // cannot run after a handled signal. + [ + "bash", + "-c", + `trap 'printf killed > "$PROVIDER_MARKER"; exit 143' TERM; sleep 5 & wait; printf normal > "$PROVIDER_MARKER"`, + ], + { delayMs: 800, providerMarker: marker }, + ) + try { + assert.equal(run.code, 143) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.equal(fs.readFileSync(marker, "utf8"), "killed") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + fs.rmSync(marker, { force: true }) + } + }) + it("leaves no final recording when the interrupted output fails validation", async () => { + const run = await runRecorderSignalled(["bash", "-c", "sleep 5"], { delayMs: 800, ffprobeOk: false }) + try { + assert.equal(run.code, 143) + assert.ok(!fs.existsSync(run.finalRecording)) + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + } + }) +}) + +describe("record.sh provider stop behavior", () => { + it("lets a driver-style stub finish teardown and atomic artifact writes on SIGTERM", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir) + const paths = recorderPaths(dir) + // The artifact lives beside the stub, not inside the recorder output + // directory, so the assertion only observes the driver-style write. + const artifact = path.join(dir, "verdict.json") + const stub = path.join(dir, "driver-stub.mjs") + fs.writeFileSync( + stub, + [ + `import fs from "node:fs"`, + `const artifact = process.argv[2]`, + `process.on("SIGTERM", () => {`, + ` // Simulate driver teardown before the atomic artifact write.`, + ` setTimeout(() => {`, + ` fs.writeFileSync(artifact + ".tmp", "complete")`, + ` fs.renameSync(artifact + ".tmp", artifact)`, + ` process.exit(143)`, + ` }, 1500)`, + `})`, + `setInterval(() => {}, 1000)`, + ].join("\n"), + ) + const run = await runRecorderSignalled(["node", stub, artifact], { delayMs: 800, stopBudget: 10 }) + try { + assert.equal(run.code, 143) + // The wrapper waited for the driver teardown instead of killing the + // driver mid-write, so the atomic artifact write completed. + assert.equal(fs.readFileSync(artifact, "utf8"), "complete") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + it("escalates to KILL and reaps a hung driver within the stop budget", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-")) + const bin = stubRecorderBin(dir) + const paths = recorderPaths(dir) + // The quote and backslash prove the paths reach the driver as argv, + // never interpolated into the driver source. + const hungPidFile = path.join(dir, "hung\\pid") + const termSeenFile = path.join(dir, 'term"seen') + const hungStub = path.join(dir, "hung-driver.mjs") + fs.writeFileSync( + hungStub, + [ + `import fs from "node:fs"`, + `process.on("SIGTERM", () => fs.writeFileSync(process.argv[2], "seen"))`, + `fs.writeFileSync(process.argv[3], String(process.pid))`, + `setInterval(() => {}, 1000)`, + ].join("\n"), + ) + const startedAt = Date.now() + // The TERM lands well after the child booted, so the ignored TERM is a + // handled signal, not a boot-time default kill. + const run = await runRecorderSignalled(["node", hungStub, termSeenFile, hungPidFile], { + delayMs: 900, + stopBudget: 2, + }) + try { + assert.equal(run.code, 143) + // TERM reached the real driver child, the wrapper waited the + // documented budget, then escalated. Teardown stays bounded. + assert.equal(fs.readFileSync(termSeenFile, "utf8"), "seen") + assert.ok(Date.now() - startedAt < 10_000) + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + const hungPid = Number(fs.readFileSync(hungPidFile, "utf8").trim()) + assert.ok(Number.isInteger(hungPid) && hungPid > 0) + let gone = false + for (let attempt = 0; attempt < 50 && !gone; attempt++) { + gone = !pidAlive(hungPid) + if (!gone) await new Promise((resolve) => setTimeout(resolve, 100)) + } + assert.ok(gone, "hung driver survived the KILL escalation") + } finally { + run.cleanup() + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + it("runs the real Node driver with no arguments and keeps metadata on failure", () => { + // The credential-free driver fails fast with a SETUP_FAILED verdict, so + // this exercises the exact CI spawn path end to end. + const run = runRecorder([]) + try { + assert.equal(run.result.status, 1) + const verdict = JSON.parse(fs.readFileSync(path.join(run.outDir, "verdict.json"), "utf8")) + assert.equal(verdict.code, "SETUP_FAILED") + const trace = JSON.parse(fs.readFileSync(path.join(run.outDir, "trace.json"), "utf8")) + assert.equal(trace.execution.code, "SETUP_FAILED") + assert.equal(fs.readFileSync(run.finalRecording, "utf8"), "data\n") + assert.deepEqual(tempFilesLeft(run), []) + } finally { + run.cleanup() + } + }) +}) diff --git a/scripts/solheim-smoke/README.md b/scripts/solheim-smoke/README.md new file mode 100644 index 0000000000..30b331ae90 --- /dev/null +++ b/scripts/solheim-smoke/README.md @@ -0,0 +1,119 @@ +# Project A: Solheim provider smoke + +Infrastructure validation only: a real VS Code extension activates, accepts a correlated IPC task, +receives output from Solheim's OpenAI-compatible provider, and produces a completion result. +It does not read a PR, assess an answer's meaning, select checks, find defects, or post to GitHub. + +The smoke is the first registered fixture of the shared substrate in +[`scripts/agent-evals`](../agent-evals/README.md). That substrate owns the fixture contract, the +trusted registry, the single VS Code launcher, startup correlation, bounded teardown, and the +metadata-only structured trace. [`run.ts`](./run.ts) is a thin adapter: it maps the shared +controller result onto the smoke evidence gates and verdict. The generic controller knows nothing +about Solheim, the prompt, or the expected completion. + +The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to +`refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the +single credential-bearing step. There is one job, no PR input, no review/posting job, +and `contents: read` is the only GitHub permission. This deliberately validates the provider integration +on main, not arbitrary PR artifacts. It is not a required PR review check. + +The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOKE_OPENAI_API_KEY` +retain their existing names to avoid an unnecessary repository-settings migration. The environment +requires the same reviewer approval as the `marketplace-production` deploy environment: a member of +the `Zoo-Code-Org/admins` team must approve each deployment before GitHub releases the secret. The +environment also allows deployments from one selected branch, `main`, and no tag policies. The +workflow keeps its own guard: the smoke job runs only on `refs/heads/main`. Workflow files cannot +declare reviewer rules; repository settings own both protections. + +The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool +groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary +workspace with isolated VS Code storage and an allowlisted host environment. The provider credential is +supplied over IPC, not inherited by the child or passed on its command line. Temporary extension storage +can contain the configured credential while the session runs. It is removed when no child remains alive; +if termination fails, storage cleanup is skipped and `gates.childExited` reports that separately from the +original smoke failure. Storage is never uploaded. +This is process/environment isolation, not a sandbox for malicious extension code—CI runs trusted main only. + +Activation and completion each have a 120-second limit; task-start acknowledgment has a 35-second limit, +in the controller. The extension awaits startup and configuration persistence without racing shared-profile +writes against a timer. If the controller times out, it tears down its isolated VS Code process rather than +trying to roll back shared settings or remove a possibly unrelated task. The live CI step has an eight-minute outer timeout. +Teardown escalates through close, terminate, and kill within independent five-second windows. A non-empty +completion and actual output-token usage must belong to the accepted task. Activation alone cannot pass. +The driver never approves dialogs or tool execution; it closes the task during teardown. + +The `completionSeen` gate is the final non-empty `completion_result` message for the accepted task. That +message is model output. It is not the extension `taskCompleted` event, which fires only after a user +accepts the result. The driver never approves dialogs, so `taskCompleted` does not fire in a smoke run. +A passing smoke therefore ends with `execution.code: "STOPPED"` and +`stopReason: "adapterStopAfterCompletionMessage"` in `trace.json`. This is the expected terminal state. +It does not contradict the `OK` verdict. See +[`scripts/agent-evals/README.md`](../agent-evals/README.md) for the terminal code semantics. + +The action uploads `verdict.json` (fixed failure code, boolean gates, duration and provider/model names), +`trace.json` (versioned metadata-only structured trace from the shared substrate; see +[`scripts/agent-evals/README.md`](../agent-evals/README.md) for its accounting limits), and `smoke.mp4`, +a 1280×720, 12 fps screen recording of the fresh CI virtual display. It records up to +eight minutes / approximately 120 MiB, with bounded shutdown and seven-day artifact retention. Video +contains the visible smoke UI, potentially including the fixed prompt and model answer; unlike the verdict +and the trace, it is not text-redacted. No settings screen is opened, and no host logs, storage, task +configuration or credential files are uploaded. Fragmented MP4 preserves completed fragments on abrupt +interruption. + +The recorder writes ffmpeg output to a unique temporary file inside the smoke output directory, with the +MP4 muxer forced because the temporary name has no mp4 suffix. After bounded shutdown it publishes the +recording only when the file is non-empty and `ffprobe` accepts it, through an atomic same-directory +rename onto `smoke.mp4`. Every failed check removes the temporary file and leaves no final recording, and +a stale `smoke.mp4` from an earlier attempt in a reused directory is removed before recording starts. +ffmpeg exits non-zero after its graceful INT or TERM shutdown, so a recorder that was still alive at +shutdown is judged by content validation alone. A recorder that stopped on its own before shutdown must +have exited zero (the `-t` time or `-fs` size limits); any other self-chosen status fails the recording +even when the fragments would still probe. The wrapper reports failure reasons on stdout and keeps the +provider exit code unchanged. There is no separate persisted status marker: a published, non-empty +`smoke.mp4` is the only success marker, and the workflow checks exactly that. + +The wrapper starts the Node driver directly, not through a package-manager wrapper, so its stop routine +signals and monitors the driver PID that owns teardown and the atomic verdict and trace writes. It sends +`TERM`, waits a budget, then escalates to `KILL` for that PID only. The budget defaults to 30 seconds and +covers the driver teardown worst case: three 5 second host termination windows plus cleanup and writes. +`SOLHEIM_SMOKE_STOP_BUDGET_S` overrides the budget for tests. A `KILL`ed driver can leave host processes +behind; no wrapper cleanup can repair that boundary. + +Metadata (`verdict.json`, `trace.json`) and the video are uploaded as two separate artifacts, each with +`if-no-files-found: error`. A missing recording therefore fails its own upload step and the explicit +integrity check step, instead of hiding behind a multi-path upload. Verdict and trace evidence stay +available when the provider failed and the video is missing. Cancellation may prevent finalization and +upload. + +Checks, without a provider key or network calls: + +```sh +pnpm test:solheim-smoke:unit +pnpm test:solheim-smoke-ci +pnpm solheim-smoke:check-types +``` + +For an explicitly authorized local live run, build the extension/webview and supply `SOLHEIM_API_KEY` +through the environment, never an argument. Set `SOLHEIM_SMOKE_OUT_DIR` to a scratch directory and run +`pnpm solheim:smoke`. Linux needs Xvfb or an existing display. Do not run an untrusted extension with a key. +To record locally, install ffmpeg and run `xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" +bash scripts/solheim-smoke/record.sh` instead. Do not attach the recorder to your existing desktop. + +The new simplified driver/workflow has not yet had a real-provider CI run. Earlier question-lane runs +established the underlying integration, not this refactor's live status. Review qualification stopped +after the frozen Stage 1B pilot produced zero incremental verified defects. Its research harness and +receipts are archived outside this smoke-only change; the chronology, results, caveats and preservation +details are recorded in [tracking issue #1896](https://github.com/Zoo-Code-Org/Zoo-Code/issues/1896). + +Validation status, kept as two separate records. One user-reported local live run passed the smoke +verdict with `completionSeen: true`: total about 6390 ms, activation about 3010 ms, task about 3110 ms, +2719 input and 29 output tokens, two early buffered and replayed events, two cumulative usage snapshots, +and a clean teardown with no malformed, foreign, tool failure, or retry events. That trace reported +`STOPPED` with `adapterStopAfterCompletionMessage`, which matches the signal distinction above. This +report is not independently verified. The trusted-main workflow run is still pending. Only a +`.github/workflows/solheim-provider-smoke.yml` run on trusted main counts as verified CI evidence. + +Planned follow-ups on top of the shared substrate, in order: five frozen real coding tasks, independent +credential-free graders, paired exact trusted commit comparisons, repeated runs with uncertainty +reporting, then advisory release integration aligned to the actual release workflows. They are tracked +in [`scripts/agent-evals/README.md`](../agent-evals/README.md). diff --git a/scripts/solheim-smoke/config.test.ts b/scripts/solheim-smoke/config.test.ts new file mode 100644 index 0000000000..345062f0d3 --- /dev/null +++ b/scripts/solheim-smoke/config.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + buildChildEnvironment, + buildLaunchArgs, + buildSmokeConfiguration, + SOLHEIM_BASE_URL, + SOLHEIM_MODEL, + SMOKE_PROMPT, +} from "./config.ts" + +describe("Solheim provider smoke configuration", () => { + it("uses one fixed provider and a small output budget", () => { + const config = buildSmokeConfiguration("test-credential") + assert.equal(config.openAiBaseUrl, SOLHEIM_BASE_URL) + assert.equal(config.openAiModelId, SOLHEIM_MODEL) + assert.equal(config.openAiApiKey, "test-credential") + assert.equal(config.modelMaxTokens, 512) + assert.deepEqual(JSON.parse(config.openAiExtraBody ?? ""), { chat_template_kwargs: { enable_thinking: false } }) + }) + it("is a no-tools infrastructure task, with no delegation or reviewer instructions", () => { + const config = buildSmokeConfiguration("test-credential") + assert.deepEqual(config.customModes?.[0]?.groups, []) + assert.equal(config.mode, "provider-smoke") + for (const name of [ + "alwaysAllowWrite", + "alwaysAllowExecute", + "alwaysAllowMcp", + "alwaysAllowSubtasks", + "alwaysAllowReadOnly", + ] as const) + assert.equal(config[name], false) + assert.ok(config.disabledTools?.includes("new_task")) + assert.ok(config.disabledTools?.includes("ask_followup_question")) + assert.match(SMOKE_PROMPT, /not a code review/) + }) + it("passes only allowlisted host variables into the isolated extension host", () => { + const env = buildChildEnvironment( + { + PATH: "/bin", + DISPLAY: ":1", + SOLHEIM_API_KEY: "secret", + GITHUB_TOKEN: "token", + NODE_OPTIONS: "--require=bad.js", + XDG_CONFIG_HOME: "/real-home", + EXTRA_CREDENTIAL: "other", + }, + "/tmp/isolated", + "/tmp/ipc", + ) + assert.equal(env.PATH, "/bin") + assert.equal(env.DISPLAY, ":1") + assert.equal(env.HOME, "/tmp/isolated") + assert.equal(env.XDG_CONFIG_HOME, "/tmp/isolated/.config") + for (const name of ["SOLHEIM_API_KEY", "GITHUB_TOKEN", "NODE_OPTIONS", "EXTRA_CREDENTIAL"]) + assert.equal(env[name], undefined) + }) + it("uses isolated storage and a keyring-free launch", () => { + const args = buildLaunchArgs("/empty-workspace", "/data", "/extensions", "/extension") + assert.equal(args[0], "/empty-workspace") + assert.ok(args.includes("--password-store=basic")) + assert.ok(args.includes("--extensionDevelopmentPath=/extension")) + assert.ok(args.includes("--user-data-dir=/data")) + }) +}) diff --git a/scripts/solheim-smoke/config.ts b/scripts/solheim-smoke/config.ts new file mode 100644 index 0000000000..b394d12cf6 --- /dev/null +++ b/scripts/solheim-smoke/config.ts @@ -0,0 +1,50 @@ +import type { RooCodeSettings } from "@roo-code/types" + +export { buildChildEnvironment, buildLaunchArgs } from "../agent-evals/isolation.ts" + +export const SOLHEIM_BASE_URL = "https://api.solheim.ai/v1" +export const SOLHEIM_MODEL = "qwen3.8-27b" +export const SMOKE_TIMEOUT_MS = 120_000 +export const SMOKE_PROMPT = + "This is a provider connectivity smoke test, not a code review. Call attempt_completion once with the result: Provider smoke completed. Do not read files, use other tools, delegate, or judge code." + +export function buildSmokeConfiguration(apiKey: string): RooCodeSettings { + return { + apiProvider: "openai", + openAiBaseUrl: SOLHEIM_BASE_URL, + openAiApiKey: apiKey, + openAiModelId: SOLHEIM_MODEL, + openAiR1FormatEnabled: true, + openAiExtraBody: JSON.stringify({ chat_template_kwargs: { enable_thinking: false } }), + modelMaxTokens: 512, + autoCondenseContext: false, + mode: "provider-smoke", + customModes: [ + { + slug: "provider-smoke", + name: "Provider smoke", + roleDefinition: "Complete one connectivity smoke task. You are not a reviewer.", + groups: [], + }, + ], + disabledTools: [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", + ], + autoApprovalEnabled: true, + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + } +} diff --git a/scripts/solheim-smoke/driver.mts b/scripts/solheim-smoke/driver.mts new file mode 100644 index 0000000000..df10cd5566 --- /dev/null +++ b/scripts/solheim-smoke/driver.mts @@ -0,0 +1,19 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { createHostRuntime } from "../agent-evals/launcher.ts" +import { runSmoke, smokeExitCode } from "./run.ts" + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const out = process.env.SOLHEIM_SMOKE_OUT_DIR +if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory") +await fs.mkdir(out, { recursive: true }) +const runtime = createHostRuntime({ root, artifactsDir: out }) +try { + const verdict = await runSmoke(runtime, process.env.SOLHEIM_API_KEY ?? "") + console.log(`Solheim provider smoke: ${verdict.code}`) + process.exit(smokeExitCode(verdict)) +} catch { + console.log("Solheim provider smoke: SETUP_FAILED") + process.exit(1) +} diff --git a/scripts/solheim-smoke/evidence.test.ts b/scripts/solheim-smoke/evidence.test.ts new file mode 100644 index 0000000000..b3be3faf4f --- /dev/null +++ b/scripts/solheim-smoke/evidence.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, type TaskEvent } from "@roo-code/types" +import { collectEvidence, smokeCode, type SmokeEvidence } from "./evidence.ts" +const fresh = (): SmokeEvidence => ({ + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, +}) +const usage = ( + taskId: string, + totalTokensOut: number, +): Extract => ({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [taskId, { totalTokensIn: 1, totalTokensOut, totalCost: 0, contextTokens: 1 }, {}], +}) +const message = (taskId: string, value: Record) => ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId, action: "created", message: { ts: 1, ...value } }], +}) + +describe("provider smoke evidence", () => { + it("rejects malformed and uncorrelated events, including activity before task acceptance", () => { + const state = fresh() + collectEvidence(state, usage("root", 5), undefined) + collectEvidence(state, usage("other", 5), "root") + collectEvidence( + state, + { eventName: RooCodeEventName.TaskTokenUsageUpdated, payload: ["root", { totalTokensOut: "5" }, {}] }, + "root", + ) + assert.deepEqual(state, fresh()) + }) + it("requires actual output usage; echoed task text is not provider evidence", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "say", say: "text", text: "task prompt" }), "root") + collectEvidence(state, usage("root", 0), "root") + assert.equal(state.providerResponseSeen, false) + collectEvidence(state, usage("root", 5), "root") + assert.equal(state.providerResponseSeen, true) + }) + it("accepts any non-empty completed response, without reviewing its meaning", () => { + const state = fresh() + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion", partial: true }), + "root", + ) + assert.equal(state.completionSeen, false) + collectEvidence( + state, + message("root", { type: "say", say: "completion_result", text: "any completion" }), + "root", + ) + assert.equal(state.completionSeen, true) + assert.ok(!JSON.stringify(state).includes("any completion")) + }) + it("allows the final completion dialog but rejects other tools", () => { + const state = fresh() + collectEvidence(state, message("root", { type: "ask", ask: "completion_result" }), "root") + assert.equal(state.completionSeen, false) + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("other", { type: "ask", ask: "command", text: "ignored" }), "root") + assert.equal(state.unexpectedTool, false) + collectEvidence(state, message("root", { type: "ask", ask: "command", text: "never stored" }), "root") + assert.equal(state.unexpectedTool, true) + const noAsk = fresh() + const attempted = usage("root", 5) + attempted.payload[2] = { execute_command: { attempts: 1, failures: 1 } } + collectEvidence(noAsk, attempted, "root") + assert.equal(noAsk.unexpectedTool, true) + }) + it("cannot pass on activation alone or ignore a lost session", () => { + assert.equal(smokeCode(fresh(), false), "PROVIDER_TIMEOUT") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true }, false), "COMPLETION_MISSING") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, false), "OK") + assert.equal(smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true }, true), "SESSION_LOST") + assert.equal( + smokeCode({ ...fresh(), providerResponseSeen: true, completionSeen: true, unexpectedTool: true }, false), + "UNEXPECTED_TOOL", + ) + }) + it("rejects delegation and aborts on the accepted root", () => { + const state = fresh() + collectEvidence(state, { eventName: RooCodeEventName.TaskDelegated, payload: ["root", "child"] }, "root") + assert.equal(state.unexpectedTool, true) + collectEvidence(state, { eventName: RooCodeEventName.TaskAborted, payload: ["root"] }, "root") + assert.equal(smokeCode(state, false), "SESSION_LOST") + }) +}) diff --git a/scripts/solheim-smoke/evidence.ts b/scripts/solheim-smoke/evidence.ts new file mode 100644 index 0000000000..3e436ca8c4 --- /dev/null +++ b/scripts/solheim-smoke/evidence.ts @@ -0,0 +1,62 @@ +import { RooCodeEventName, taskEventSchema, type TaskEvent } from "@roo-code/types" + +export type SmokeEvidence = { + providerResponseSeen: boolean + completionSeen: boolean + unexpectedTool: boolean + aborted: boolean +} + +// Only schema-valid events for the accepted root task can satisfy smoke gates. +// No model text, credentials, tool arguments, or task IDs are persisted. +export function collectEvidence(state: SmokeEvidence, raw: unknown, rootTaskId: string | undefined): void { + if (!rootTaskId) return + const parsed = taskEventSchema.safeParse(raw) + if (!parsed.success) return + const event: TaskEvent = parsed.data + if (event.eventName === RooCodeEventName.Message) { + const entry = event.payload[0] + if (entry.taskId !== rootTaskId || entry.message.partial === true) return + const message = entry.message + if (message.type === "say" && message.say === "completion_result" && message.text?.trim()) + state.completionSeen = true + if (message.type === "ask" && message.ask !== "completion_result") state.unexpectedTool = true + } else if (event.payload[0] === rootTaskId) { + if ( + event.eventName === RooCodeEventName.TaskTokenUsageUpdated || + event.eventName === RooCodeEventName.TaskCompleted + ) { + if ( + Object.entries(event.payload[2]).some( + ([tool, usage]) => tool !== "attempt_completion" && usage.attempts > 0, + ) + ) + state.unexpectedTool = true + } + if (event.eventName === RooCodeEventName.TaskTokenUsageUpdated && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskCompleted && event.payload[1].totalTokensOut > 0) + state.providerResponseSeen = true + if (event.eventName === RooCodeEventName.TaskAborted) state.aborted = true + if (event.eventName === RooCodeEventName.TaskDelegated || event.eventName === RooCodeEventName.TaskSpawned) + state.unexpectedTool = true + } +} + +export type SmokeCode = + | "OK" + | "SETUP_FAILED" + | "ACTIVATION_FAILED" + | "TASK_START_FAILED" + | "SESSION_LOST" + | "UNEXPECTED_TOOL" + | "PROVIDER_TIMEOUT" + | "COMPLETION_MISSING" + | "TEARDOWN_FAILED" +export function smokeCode(state: SmokeEvidence, sessionLost: boolean): SmokeCode { + if (sessionLost || state.aborted) return "SESSION_LOST" + if (state.unexpectedTool) return "UNEXPECTED_TOOL" + if (!state.providerResponseSeen) return "PROVIDER_TIMEOUT" + if (!state.completionSeen) return "COMPLETION_MISSING" + return "OK" +} diff --git a/scripts/solheim-smoke/fixture.ts b/scripts/solheim-smoke/fixture.ts new file mode 100644 index 0000000000..37916b54dd --- /dev/null +++ b/scripts/solheim-smoke/fixture.ts @@ -0,0 +1,89 @@ +import type { AgentFixture } from "../agent-evals/contract.ts" +import { registerFixture } from "../agent-evals/registry.ts" +import { SMOKE_TIMEOUT_MS, SMOKE_PROMPT, SOLHEIM_BASE_URL, SOLHEIM_MODEL, buildSmokeConfiguration } from "./config.ts" + +export const SMOKE_FIXTURE_NAME = "solheim-smoke" +const SMOKE_FIXTURE_VERSION = "1.0.0" +const ROLE_DEFINITION = "Complete one connectivity smoke task. You are not a reviewer." +const DISABLED_TOOLS = [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", +] as const + +// The first trusted fixture: the no-tools deterministic provider smoke on an +// empty workspace. The projection is the explicit nonsecret mirror of the +// built configuration; the trace digests it and never sees the credential. +export function buildSmokeFixture(): AgentFixture { + return { + identity: { + name: SMOKE_FIXTURE_NAME, + version: SMOKE_FIXTURE_VERSION, + description: "No-tools deterministic provider connectivity smoke on an empty workspace.", + }, + prompt: SMOKE_PROMPT, + workspace: { files: [] }, + toolPolicy: { + mode: "provider-smoke", + roleDefinition: ROLE_DEFINITION, + groups: [], + disabledTools: [...DISABLED_TOOLS], + allow: { + readOnly: false, + readOnlyOutsideWorkspace: false, + write: false, + writeOutsideWorkspace: false, + writeProtected: false, + execute: false, + mcp: false, + modeSwitch: false, + subtasks: false, + followupQuestions: false, + }, + }, + limits: { + activationTimeoutMs: SMOKE_TIMEOUT_MS, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: SMOKE_TIMEOUT_MS, + usageDrainMs: 5_000, + teardownStepMs: 5_000, + }, + settingsProjection: { + apiProvider: "openai", + openAiBaseUrl: SOLHEIM_BASE_URL, + openAiModelId: SOLHEIM_MODEL, + openAiR1FormatEnabled: true, + openAiExtraBody: { chat_template_kwargs: { enable_thinking: false } }, + modelMaxTokens: 512, + autoCondenseContext: false, + mode: "provider-smoke", + customModes: [ + { slug: "provider-smoke", name: "Provider smoke", roleDefinition: ROLE_DEFINITION, groups: [] }, + ], + disabledTools: [...DISABLED_TOOLS], + autoApprovalEnabled: true, + alwaysAllowReadOnly: false, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: false, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: false, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + }, + buildConfiguration: buildSmokeConfiguration, + } +} + +let registered = false + +export function ensureSmokeFixtureRegistered(): void { + if (registered) return + registered = true + registerFixture(buildSmokeFixture()) +} diff --git a/scripts/solheim-smoke/package.json b/scripts/solheim-smoke/package.json new file mode 100644 index 0000000000..c4fb2812b9 --- /dev/null +++ b/scripts/solheim-smoke/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/scripts/solheim-smoke/record.sh b/scripts/solheim-smoke/record.sh new file mode 100644 index 0000000000..436c57c0b8 --- /dev/null +++ b/scripts/solheim-smoke/record.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Run inside a fresh xvfb-run display, never record an existing desktop. +set -euo pipefail +: "${DISPLAY:?Run this wrapper under xvfb-run}" +: "${SOLHEIM_SMOKE_OUT_DIR:?Set a scratch output directory}" +mkdir -p "$SOLHEIM_SMOKE_OUT_DIR" +final_recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Stale output from an earlier attempt must never survive into this attempt's +# upload. Remove only these exact paths and pending artifact writes. +rm -f "$final_recording" "$SOLHEIM_SMOKE_OUT_DIR/verdict.json" "$SOLHEIM_SMOKE_OUT_DIR/trace.json" \ + "$SOLHEIM_SMOKE_OUT_DIR"/verdict.json.*.tmp "$SOLHEIM_SMOKE_OUT_DIR"/trace.json.*.tmp + +# The recorder needs display access, not provider/GitHub credentials. Fragmented +# MP4 keeps completed fragments readable if the action is forcibly interrupted. +# The output muxer is forced with -f mp4 because the temporary file name has no +# mp4 suffix for ffmpeg to infer the format from. +temp_recording="$SOLHEIM_SMOKE_OUT_DIR/.smoke.mp4.$$.tmp" +env -i "PATH=$PATH" "DISPLAY=$DISPLAY" "XAUTHORITY=${XAUTHORITY:-}" \ + ffmpeg -nostdin -hide_banner -loglevel error -y \ + -f x11grab -video_size 1280x720 -framerate 12 -i "$DISPLAY" \ + -an -c:v libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p -g 24 \ + -t 480 -fs 125829120 -movflags +frag_keyframe+empty_moov \ + -f mp4 "$temp_recording" >/dev/null 2>&1 & +recorder_pid=$! + +provider_pid="" + +# Accepted normal shutdown semantics: ffmpeg exits non-zero after its graceful +# INT or TERM shutdown, so a recorder that was still alive when shutdown started +# is judged by content validation, not by its interrupted exit status. A recorder +# that stopped on its own before shutdown must have exited zero (the -t or -fs +# limits); any other self-chosen non-zero status fails the recording even when +# ffprobe can still read the fragments. +report_recording_failure() { + echo "Solheim smoke recording failed: $1" + [[ "$result" -ne 0 ]] || result=1 +} + +# The provider child is the Node driver itself, so TERM reaches the one process +# that owns teardown and the atomic verdict and trace writes. The 30 second +# budget covers the driver teardown worst case: three 5 second host termination +# windows plus cleanup and artifact writes. KILL is the last resort for a hung +# driver; a killed driver can leave host processes behind, and no wrapper +# cleanup can repair that boundary. +provider_stop_budget="${SOLHEIM_SMOKE_STOP_BUDGET_S:-30}" +[[ "$provider_stop_budget" =~ ^[0-9]+$ ]] || provider_stop_budget=30 + +stop_provider() { + [[ -n "$provider_pid" ]] || return 0 + kill -TERM "$provider_pid" 2>/dev/null || return 0 + for ((attempt = 0; attempt < provider_stop_budget * 10; attempt++)); do + kill -0 "$provider_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$provider_pid" 2>/dev/null || true + wait "$provider_pid" 2>/dev/null || true +} + +finish() { + result=$? + trap - EXIT + # Teardown must run to completion once it starts. + trap '' INT TERM + local recorder_status=0 recorder_was_alive=0 + if kill -0 "$recorder_pid" 2>/dev/null; then + recorder_was_alive=1 + fi + # TERM is a graceful stop for ffmpeg: it finalizes the fragments before exit. + kill -TERM "$recorder_pid" 2>/dev/null || true + # A hung recorder must not hold the action open indefinitely. + for ((attempt = 0; attempt < 50; attempt++)); do + kill -0 "$recorder_pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$recorder_pid" 2>/dev/null || true + wait "$recorder_pid" 2>/dev/null || recorder_status=$? + # An interrupted provider must not outlive the wrapper. + stop_provider + if [[ "$recorder_was_alive" -eq 0 ]] && [[ "$recorder_status" -ne 0 ]]; then + rm -f "$temp_recording" + report_recording_failure "recorder stopped on its own with exit status $recorder_status" + elif [[ ! -s "$temp_recording" ]]; then + rm -f "$temp_recording" + report_recording_failure "recorder output is missing or empty" + elif ! env -i "PATH=$PATH" ffprobe -v error "$temp_recording" >/dev/null 2>&1; then + rm -f "$temp_recording" + report_recording_failure "recorder output failed ffprobe validation" + elif mv -f "$temp_recording" "$final_recording"; then + # Same-directory rename: atomic, so no partially written final exists. + # The published, nonempty final is the only success marker. + : + else + rm -f "$temp_recording" + report_recording_failure "validated recording could not be published" + fi + exit "$result" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +sleep 0.5 +kill -0 "$recorder_pid" 2>/dev/null || exit 1 + +# Arguments are used for credential-free recorder tests; CI supplies none. +# An injected command would inherit the provider credential, so reject it. +# Backgrounding the provider keeps the signal traps responsive during the run. +if [[ "$#" -gt 0 ]]; then + if [[ -n "${SOLHEIM_API_KEY:-}" ]]; then + echo "Refusing to run a supplied command while SOLHEIM_API_KEY is set" >&2 + exit 2 + fi + "$@" & +else + # Invoke the Node driver directly: a package-manager wrapper would sit + # between this wrapper and the driver, and stop_provider must signal and + # monitor the driver PID that performs teardown and artifact writes. + node --import tsx "$script_dir/driver.mts" & +fi +provider_pid=$! +provider_status=0 +wait "$provider_pid" || provider_status=$? +exit "$provider_status" diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts new file mode 100644 index 0000000000..9be74b3d75 --- /dev/null +++ b/scripts/solheim-smoke/run.test.ts @@ -0,0 +1,245 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand } from "@roo-code/types" +import { runSmoke, smokeExitCode, type SmokeRuntime, type SmokeVerdict } from "./run.ts" + +type Scenario = + | "success" + | "activation" + | "start-failure" + | "no-provider" + | "no-completion" + | "tool" + | "disconnect" + | "uncorrelated" + | "aborted" + | "setup" +function fixture( + scenario: Scenario = "success", + options: { stuck?: boolean; closeThrows?: boolean; timeout?: () => void } = {}, +) { + let now = 0 + let exited = false + let onExit = () => {} + let onDisconnect = () => {} + let onEvent: (event: unknown) => void = () => {} + let interrupt = () => {} + let cleaned = false + let disconnected = false + let unsubscribed = false + const commands: TaskCommand[] = [] + const signals: NodeJS.Signals[] = [] + const written: SmokeVerdict[] = [] + const close = () => { + exited = true + onExit() + } + const runtime: SmokeRuntime = { + async launch() { + if (scenario === "setup") throw new Error("private setup detail") + return { + get exited() { + return exited + }, + onExit(listener) { + onExit = listener + }, + signal(signal) { + signals.push(signal) + if (!options.stuck) close() + }, + } + }, + connect() { + return { + isReady: scenario !== "activation", + onTaskEvent(listener) { + onEvent = listener + }, + onDisconnect(listener) { + onDisconnect = listener + }, + disconnect() { + disconnected = true + }, + sendCommand(command) { + commands.push(command) + if (command.commandName === TaskCommandName.CloseTask) { + if (options.closeThrows) throw new Error("private close error") + if (!options.stuck) close() + return + } + if (command.commandName !== TaskCommandName.StartNewTask) return + if (options.timeout) { + options.timeout() + return + } + onEvent({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [ + scenario === "start-failure" + ? { + requestId: command.data.requestId, + success: false, + errorCode: "task_start_failed", + errorMessage: "Task start failed", + } + : { requestId: command.data.requestId, success: true, taskId: "root" }, + ], + }) + const taskId = scenario === "uncorrelated" ? "other" : "root" + if (scenario !== "no-provider") + onEvent({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + taskId, + { totalTokensIn: 1, totalTokensOut: 5, totalCost: 0, contextTokens: 1 }, + {}, + ], + }) + if (scenario !== "no-completion") + onEvent({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId, + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "private synthetic answer", + }, + }, + ], + }) + if (scenario === "tool") + onEvent({ + eventName: RooCodeEventName.TaskSpawned, + payload: ["root", "child"], + }) + if (scenario === "disconnect") onDisconnect() + if (scenario === "aborted") onEvent({ eventName: RooCodeEventName.TaskAborted, payload: ["root"] }) + }, + } + }, + async cleanup() { + cleaned = true + }, + async writeArtifact(fileName, value) { + if (fileName === "verdict.json") written.push(value as SmokeVerdict) + }, + onInterrupt(listener) { + interrupt = listener + return () => { + unsubscribed = true + } + }, + now: () => now, + async pause(ms) { + now += ms + }, + } + return { + runtime, + commands, + signals, + written, + interrupt: () => interrupt(), + get cleaned() { + return cleaned + }, + get disconnected() { + return disconnected + }, + get unsubscribed() { + return unsubscribed + }, + } +} + +describe("provider smoke orchestration", () => { + it("correlates startup and evidence, writes a metadata-only verdict, and exits zero", async () => { + const f = fixture() + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "OK") + assert.equal(smokeExitCode(verdict), 0) + assert.deepEqual(verdict.gates, { + extensionActivated: true, + ipcConnected: true, + taskAccepted: true, + providerResponseSeen: true, + completionSeen: true, + childExited: true, + }) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.disconnected && f.unsubscribed) + assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask) + assert.ok(!JSON.stringify(verdict).includes("private")) + }) + for (const [scenario, code] of [ + ["activation", "ACTIVATION_FAILED"], + ["start-failure", "TASK_START_FAILED"], + ["no-provider", "PROVIDER_TIMEOUT"], + ["no-completion", "COMPLETION_MISSING"], + ["tool", "UNEXPECTED_TOOL"], + ["disconnect", "SESSION_LOST"], + ["uncorrelated", "PROVIDER_TIMEOUT"], + ["aborted", "SESSION_LOST"], + ["setup", "SETUP_FAILED"], + ] as const) { + it(`fails closed for ${scenario} and still writes the verdict`, async () => { + const f = fixture(scenario) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, code) + assert.equal(verdict.passed, false) + assert.equal(smokeExitCode(verdict), 1) + assert.deepEqual(f.written, [verdict]) + assert.ok(f.cleaned && f.unsubscribed) + }) + } + it("bounds startup at the controller and closes its host when no reply arrives", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }) + const f = fixture("success", { timeout: () => t.mock.timers.tick(35_000) }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.taskAccepted, false) + assert.ok(f.cleaned) + }) + it("bounds TERM/KILL teardown and keeps live-host storage out of cleanup", async () => { + const f = fixture("success", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.equal(verdict.durationMs, 15_000) + assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + assert.equal(f.disconnected, true) + }) + it("preserves the original failure but reports failed teardown separately", async () => { + const f = fixture("start-failure", { stuck: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TASK_START_FAILED") + assert.equal(verdict.gates.childExited, false) + assert.equal(f.cleaned, false) + }) + it("interrupts an evidence wait, terminates only its host, and fails closed", async () => { + const f = fixture("no-completion") + const pause = f.runtime.pause + f.runtime.pause = async (ms) => { + f.interrupt() + await pause(ms) + } + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "SESSION_LOST") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.unsubscribed) + }) + it("continues host termination and writes a failing verdict after a CloseTask exception", async () => { + const f = fixture("success", { closeThrows: true }) + const verdict = await runSmoke(f.runtime, "private-test-key") + assert.equal(verdict.code, "TEARDOWN_FAILED") + assert.deepEqual(f.signals, ["SIGTERM"]) + assert.ok(f.cleaned && f.disconnected) + }) +}) diff --git a/scripts/solheim-smoke/run.ts b/scripts/solheim-smoke/run.ts new file mode 100644 index 0000000000..8cbf1e9fb5 --- /dev/null +++ b/scripts/solheim-smoke/run.ts @@ -0,0 +1,84 @@ +import { runAgentFixture, type AgentRuntime } from "../agent-evals/runner.ts" +import { getFixture } from "../agent-evals/registry.ts" +import { SOLHEIM_BASE_URL, SOLHEIM_MODEL } from "./config.ts" +import { collectEvidence, smokeCode, type SmokeCode, type SmokeEvidence } from "./evidence.ts" +import { SMOKE_FIXTURE_NAME, ensureSmokeFixtureRegistered } from "./fixture.ts" + +export type SmokeRuntime = AgentRuntime + +export type SmokeVerdict = { + kind: "provider-smoke" + model: string + provider: string + passed: boolean + code: SmokeCode + gates: { + extensionActivated: boolean + ipcConnected: boolean + taskAccepted: boolean + providerResponseSeen: boolean + completionSeen: boolean + childExited: boolean + } + durationMs: number +} + +// Thin adapter: the shared controller runs the fixture and writes the trace; +// this function keeps the smoke-specific evidence gates and verdict shape. +export async function runSmoke(runtime: SmokeRuntime, apiKey: string): Promise { + ensureSmokeFixtureRegistered() + getFixture(SMOKE_FIXTURE_NAME) + const started = runtime.now() + const evidence: SmokeEvidence = { + providerResponseSeen: false, + completionSeen: false, + unexpectedTool: false, + aborted: false, + } + const result = await runAgentFixture(runtime, { + fixtureName: SMOKE_FIXTURE_NAME, + credential: apiKey, + observers: { + onAcceptedEvent: (event, taskId) => collectEvidence(evidence, event, taskId), + shouldStop: () => + evidence.unexpectedTool || + evidence.aborted || + (evidence.providerResponseSeen && evidence.completionSeen), + }, + }) + let code: SmokeCode + switch (result.execution.code) { + case "SETUP_FAILED": + code = "SETUP_FAILED" + break + case "ACTIVATION_FAILED": + code = "ACTIVATION_FAILED" + break + case "TASK_START_FAILED": + code = "TASK_START_FAILED" + break + default: + code = smokeCode(evidence, result.execution.sessionLost) + } + if (result.teardown.status !== "ok" && code === "OK") code = "TEARDOWN_FAILED" + const verdict: SmokeVerdict = { + kind: "provider-smoke", + model: SOLHEIM_MODEL, + provider: SOLHEIM_BASE_URL, + passed: code === "OK", + code, + gates: { + extensionActivated: result.execution.activated, + ipcConnected: result.execution.activated, + taskAccepted: result.execution.taskAccepted, + providerResponseSeen: evidence.providerResponseSeen, + completionSeen: evidence.completionSeen, + childExited: result.teardown.hostExited, + }, + durationMs: runtime.now() - started, + } + await runtime.writeArtifact("verdict.json", verdict) + return verdict +} + +export const smokeExitCode = (verdict: SmokeVerdict): number => (verdict.passed ? 0 : 1) diff --git a/scripts/solheim-smoke/task-start.test.ts b/scripts/solheim-smoke/task-start.test.ts new file mode 100644 index 0000000000..58ebb91fac --- /dev/null +++ b/scripts/solheim-smoke/task-start.test.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { RooCodeEventName, TASK_START_FAILURE_ERROR_CODE, TASK_START_FAILURE_ERROR_MESSAGE } from "@roo-code/types" + +import { createTaskStartWaiter } from "./task-start.ts" + +const REQUEST_ID = "final-smoke-0123abcdef456789" + +const startResponse = (payload: unknown): unknown => ({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [payload], +}) + +const successPayload = (requestId = REQUEST_ID): unknown => ({ + requestId, + success: true, + taskId: "accepted-task-1", +}) + +const failurePayload = (requestId = REQUEST_ID): Record => ({ + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, +}) + +const settledOutcome = async (promise: Promise): Promise<"resolved" | "rejected" | "pending"> => { + const probe = promise.then( + () => "resolved" as const, + () => "rejected" as const, + ) + return Promise.race([probe, new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 10))]) +} + +describe("createTaskStartWaiter", () => { + it("resolves with the accepted task ID on a matching success response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload())) + + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("rejects with the fixed sanitized failure values on a matching failure response", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal(error.message, `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE}`) + return true + }) + }) + + it("rejects with the stage in the sanitized error on a failure response carrying a stage", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "taskCreation", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.equal( + error.message, + `${TASK_START_FAILURE_ERROR_CODE}: ${TASK_START_FAILURE_ERROR_MESSAGE} (stage=taskCreation)`, + ) + return true + }) + }) + + it("rejects with the stage unknown in the sanitized error when the response names it", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "unknown", + }), + ) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /\(stage=unknown\)$/) + return true + }) + }) + + it("ignores a failure response carrying an unknown stage value", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + ...failurePayload(), + stage: "secretStore", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("ignores responses for a different requestId", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(successPayload("some-other-request"))) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores malformed and non-start events", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskCreated, payload: ["task-x"] }) + waiter.onTaskEvent(startResponse({ requestId: REQUEST_ID, success: true })) + waiter.onTaskEvent(startResponse("not-an-object")) + waiter.onTaskEvent({ eventName: RooCodeEventName.TaskStartResponse }) + waiter.onTaskEvent(undefined) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("ignores a success response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: true, + taskId: "accepted-task-1", + providerError: "key sk-secret-123 rejected", + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(successPayload())) + assert.equal(await waiter.result, "accepted-task-1") + }) + + it("cannot settle on a failure response carrying extra top-level fields", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent( + startResponse({ + requestId: REQUEST_ID, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + detail: { apiKey: "sk-secret-123", url: "https://api.example.com" }, + }), + ) + assert.equal(await settledOutcome(waiter.result), "pending") + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("settles only once: later responses cannot change the outcome", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 1_000 }) + + waiter.onTaskEvent(startResponse(failurePayload())) + await assert.rejects(waiter.result, /task_start_failed/) + + assert.equal(waiter.onTaskEvent(startResponse(successPayload())), undefined) + waiter.onDisconnect() + waiter.onExit() + await assert.rejects(waiter.result, /task_start_failed/) + }) + + it("rejects on timeout", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + await assert.rejects(waiter.result, (error: Error) => { + assert.match(error.message, /task start response missing after 10ms/) + assert.match(error.message, new RegExp(REQUEST_ID)) + return true + }) + }) + + it("rejects on IPC disconnect", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onDisconnect() + + await assert.rejects(waiter.result, /IPC disconnected before the task start response arrived/) + }) + + it("rejects on VS Code exit", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10_000 }) + + waiter.onExit() + + await assert.rejects(waiter.result, /VS Code exited before the task start response arrived/) + }) + + it("stops the timeout once settled", async () => { + const waiter = createTaskStartWaiter({ requestId: REQUEST_ID, timeoutMs: 10 }) + + waiter.onTaskEvent(startResponse(successPayload())) + await waiter.result + + // A pending timer must not reject after success. Wait past the + // original deadline; a leak would reject the already-settled promise + // handler chain here. + await new Promise((resolve) => setTimeout(resolve, 30)) + assert.equal(await waiter.result, "accepted-task-1") + }) +}) diff --git a/scripts/solheim-smoke/task-start.ts b/scripts/solheim-smoke/task-start.ts new file mode 100644 index 0000000000..39f8ffa97c --- /dev/null +++ b/scripts/solheim-smoke/task-start.ts @@ -0,0 +1 @@ +export { createTaskStartWaiter, type TaskStartWaiter } from "../agent-evals/task-start.ts" diff --git a/scripts/solheim-smoke/tsconfig.json b/scripts/solheim-smoke/tsconfig.json new file mode 100644 index 0000000000..55bddebe10 --- /dev/null +++ b/scripts/solheim-smoke/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "lib": ["ES2022"], + "module": "esnext", + "moduleResolution": "bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022", + "types": ["node"] + }, + "include": ["./*.ts", "./*.mts"] +} diff --git a/src/extension/__tests__/api-start-task-logging.spec.ts b/src/extension/__tests__/api-start-task-logging.spec.ts new file mode 100644 index 0000000000..014991128d --- /dev/null +++ b/src/extension/__tests__/api-start-task-logging.spec.ts @@ -0,0 +1,234 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import * as vscode from "vscode" + +import { + TaskCommandName, + RooCodeEventName, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, + taskStartResponseSchema, + providerIdentifiers, + type RooCodeSettings, +} from "@roo-code/types" + +import { API } from "../api" +import { Package } from "../../shared/package" +import type { ClineProvider } from "../../core/webview/ClineProvider" +import { makeClineProviderFactory } from "../../test-utils/provider" + +const { commandHandlers, sentMessages } = vi.hoisted(() => ({ + commandHandlers: [] as Array<(clientId: string, command: unknown) => Promise>, + sentMessages: [] as Array<{ clientId: string; message: unknown }>, +})) + +vi.mock("@roo-code/ipc", () => ({ + IpcServer: class { + listen(): void {} + on(_messageType: unknown, handler: (clientId: string, command: unknown) => Promise): void { + commandHandlers.push(handler) + } + send(clientId: string, message: unknown): void { + sentMessages.push({ clientId, message }) + } + }, +})) + +type SentTaskEvent = { data: { eventName: RooCodeEventName; payload: unknown[] } } + +const sentTaskEvents = (): SentTaskEvent[] => sentMessages.map(({ message }) => message as SentTaskEvent) + +const buildStartCommand = (overrides?: { requestId?: string; text?: string }) => ({ + commandName: TaskCommandName.StartNewTask, + data: { + configuration: { + apiProvider: providerIdentifiers.openai, + openAiApiKey: "opaque-solheim-secret-0123456789abcdef", + openAiModelId: "qwen3.8-27b", + openAiBaseUrl: "https://api.solheim.ai/v1", + autoApprovalEnabled: true, + } as RooCodeSettings, + text: overrides?.text ?? "provider connectivity smoke", + requestId: overrides?.requestId, + }, +}) + +function buildApi(createTaskImpl?: () => Promise<{ taskId: string }>, postStateToWebviewImpl?: () => Promise) { + const outputChannel = { appendLine: vi.fn() } as unknown as vscode.OutputChannel + const provider = { + context: {}, + on: vi.fn(), + contextProxy: { setValues: vi.fn().mockResolvedValue(undefined) }, + providerSettingsManager: { saveConfig: vi.fn().mockResolvedValue("default"), setModeConfig: vi.fn() }, + postStateToWebview: postStateToWebviewImpl + ? vi.fn().mockImplementation(postStateToWebviewImpl) + : vi.fn().mockResolvedValue(undefined), + evictCurrentTask: vi.fn().mockResolvedValue(undefined), + postMessageToWebview: vi.fn().mockResolvedValue(undefined), + removeClineFromStack: vi.fn().mockResolvedValue(undefined), + createTask: createTaskImpl + ? vi.fn().mockImplementation(createTaskImpl) + : vi.fn().mockResolvedValue({ taskId: "task-1" }), + } as unknown as ClineProvider + + const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}) + const executeCommand = vi.spyOn(vscode.commands, "executeCommand").mockResolvedValue(undefined) + const api = new API(outputChannel, provider, makeClineProviderFactory(), "/tmp/final-smoke-test.sock", true) + + const collectLogs = () => + [ + ...(outputChannel.appendLine as ReturnType).mock.calls.flat().map(String), + ...consoleSpy.mock.calls.flat().map((arg) => JSON.stringify(arg)), + ].join("\n") + + return { api, provider, executeCommand, collectLogs } +} + +describe("API StartNewTask IPC", () => { + afterEach(() => { + commandHandlers.length = 0 + sentMessages.length = 0 + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it("never logs the API key, base URL, or prompt text from the StartNewTask command", async () => { + const { collectLogs } = buildApi() + + const handler = commandHandlers.at(-1) + expect(handler).toBeDefined() + + await handler!("client-1", buildStartCommand()) + + const logged = collectLogs() + + expect(logged).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(logged).not.toContain('"openAiApiKey":"') + // Configuration is omitted entirely, even nominally non-secret fields. + expect(logged).not.toContain("openAiModelId") + expect(logged).not.toContain("qwen3.8-27b") + expect(logged).not.toContain("openAiBaseUrl") + expect(logged).not.toContain("openAiApiKey") + // Prompt content is metadata only: length, never text. + expect(logged).not.toContain("provider connectivity smoke") + expect(logged).toContain("promptLength") + }) + + it("omits the sidebar focus for an IPC start, so an unresolved focus command cannot block it", async () => { + const { executeCommand } = buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-legacy" })) + + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) + + it("answers a correlated start with exactly one sanitized success response", async () => { + buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-1" })) + + expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + expect(taskStartResponseSchema.parse(events[0]?.data.payload[0])).toEqual({ + requestId: "req-1", + success: true, + taskId: "task-1", + }) + }) + + it("answers a failed correlated start with exactly one fixed safe failure response", async () => { + const rawError = new Error( + "openai rejected key opaque-solheim-secret-0123456789abcdef at https://api.solheim.ai/v1 for prompt review the pull request diff", + ) + buildApi(() => Promise.reject(rawError)) + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand({ requestId: "req-2" })) + + expect(sentMessages).toHaveLength(1) + expect(sentMessages[0]?.clientId).toBe("client-1") + const events = sentTaskEvents() + expect(events[0]?.data.eventName).toBe(RooCodeEventName.TaskStartResponse) + + const parsed = taskStartResponseSchema.parse(events[0]?.data.payload[0]) + expect(parsed).toEqual({ + requestId: "req-2", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + + const sent = JSON.stringify(sentMessages) + expect(sent).not.toContain("opaque-solheim-secret-0123456789abcdef") + expect(sent).not.toContain("api.solheim.ai") + expect(sent).not.toContain("review the pull request diff") + }) + + it("does not race pending configuration/startup against a timeout", async () => { + vi.useFakeTimers() + let finishCreation!: (task: { taskId: string }) => void + const { provider } = buildApi( + () => + new Promise((resolve) => { + finishCreation = resolve + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-pending" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(provider.createTask).toHaveBeenCalledTimes(1) + expect(sentMessages).toHaveLength(0) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() + finishCreation({ taskId: "finished-task" }) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-pending", + success: true, + taskId: "finished-task", + }) + }) + + it("reports a delayed startup rejection only after it settles, without late task removal", async () => { + vi.useFakeTimers() + let rejectCreation!: (error: Error) => void + const { provider } = buildApi( + () => + new Promise((_resolve, reject) => { + rejectCreation = reject + }), + ) + const running = commandHandlers.at(-1)!("client-1", buildStartCommand({ requestId: "req-reject" })) + await vi.advanceTimersByTimeAsync(60_000) + expect(sentMessages).toHaveLength(0) + rejectCreation(new Error("private startup error")) + await running + expect(taskStartResponseSchema.parse(sentTaskEvents()[0]?.data.payload[0])).toEqual({ + requestId: "req-reject", + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + }) + expect(provider.removeClineFromStack).not.toHaveBeenCalled() + }) + + it("sends no start response for legacy IPC starts without requestId", async () => { + const { executeCommand } = buildApi() + + const handler = commandHandlers.at(-1) + await handler!("client-1", buildStartCommand()) + + expect(sentMessages).toHaveLength(0) + expect(executeCommand).not.toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) + + it("keeps the sidebar focus for direct API callers that do not opt out", async () => { + const { api, executeCommand } = buildApi() + + await api.startNewTask({ configuration: {} as RooCodeSettings, text: "direct caller" }) + + expect(executeCommand).toHaveBeenCalledWith(`${Package.name}.SidebarProvider.focus`) + }) +}) diff --git a/src/extension/api.ts b/src/extension/api.ts index 6a8ed8dfdd..3a33510346 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -13,6 +13,7 @@ import { type ProviderSettings, type ProviderSettingsEntry, type TaskEvent, + type TaskStartResponse, type CreateTaskOptions, type TaskApiConversationHistorySequence, type WebviewThemeFixture, @@ -21,7 +22,10 @@ import { isSecretStateKey, IpcOrigin, IpcMessageType, + TASK_START_FAILURE_ERROR_CODE, + TASK_START_FAILURE_ERROR_MESSAGE, } from "@roo-code/types" + import { IpcServer } from "@roo-code/ipc" import { Package } from "../shared/package" @@ -81,12 +85,45 @@ export class API extends EventEmitter implements RooCodeAPI { } switch (command.commandName) { - case TaskCommandName.StartNewTask: - this.log( - `[API] StartNewTask -> ${command.data.text}, ${JSON.stringify(command.data.configuration)}`, - ) - await this.startNewTask(command.data) + case TaskCommandName.StartNewTask: { + // Neither prompt nor settings values enter the log. + this.log("[API] StartNewTask", { + promptLength: command.data.text.length, + imageCount: command.data.images?.length ?? 0, + newTab: command.data.newTab === true, + requestId: command.data.requestId, + }) + + const { requestId } = command.data + + if (requestId === undefined) { + await this.startNewTask(command.data, { focusSidebar: false }) + break + } + + // Exactly one sanitized, client-scoped response per + // correlated start. The reply never echoes the failure. + // Await actual startup, including configuration writes. + // The smoke controller owns its timeout and terminates + // its isolated host; shared-profile writes are not raced. + let response: TaskStartResponse + try { + const taskId = await this.startNewTask(command.data, { + focusSidebar: false, + }) + response = { requestId, success: true, taskId } + } catch { + this.log("[API] StartNewTask failed", { requestId }) + response = { + requestId, + success: false, + errorCode: TASK_START_FAILURE_ERROR_CODE, + errorMessage: TASK_START_FAILURE_ERROR_MESSAGE, + } + } + sendResponse(RooCodeEventName.TaskStartResponse, [response]) break + } case TaskCommandName.CancelTask: this.log(`[API] CancelTask`) await this.cancelCurrentTask() @@ -169,17 +206,20 @@ export class API extends EventEmitter implements RooCodeAPI { return super.emit(eventName, ...args) } - public async startNewTask({ - configuration, - text, - images, - newTab, - }: { - configuration: RooCodeSettings - text?: string - images?: string[] - newTab?: boolean - }) { + public async startNewTask( + { + configuration, + text, + images, + newTab, + }: { + configuration: RooCodeSettings + text?: string + images?: string[] + newTab?: boolean + }, + startOptions?: { focusSidebar?: boolean }, + ) { let provider: ClineProvider if (newTab) { @@ -189,7 +229,11 @@ export class API extends EventEmitter implements RooCodeAPI { provider = await this.providerFactory.createInNewTab() this.registerListeners(provider) } else { - await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) + // IPC callers omit focus: executeCommand cannot be cancelled and a + // stalled webview launch would block the whole start. + if (startOptions?.focusSidebar ?? true) { + await vscode.commands.executeCommand(`${Package.name}.SidebarProvider.focus`) + } provider = this.sidebarProvider }