feat(evals): add trusted fixture runner and execution traces (substrate only)

.github/workflows/code-qa.yml

@@ -98,6 +98,16 @@ jobs:
               run: pnpm check-types
             - name: Validate Code QA workflow
               run: pnpm test:code-qa-ci
+            - name: Validate Solheim provider smoke workflow
+              run: pnpm test:solheim-smoke-ci
+            - name: Test Solheim provider smoke units
+              run: pnpm test:solheim-smoke:unit
+            - name: Check Solheim provider smoke types
+              run: pnpm solheim-smoke:check-types
+            - name: Test agent evals units
+              run: pnpm test:agent-evals:unit
+            - name: Check agent evals types
+              run: pnpm agent-evals:check-types
             - name: Model-check task lifecycle protocols
               run: pnpm lifecycle:model-check
             - name: Validate MCP OAuth integration

.github/workflows/solheim-provider-smoke.yml

@@ -0,0 +1,88 @@
+name: Solheim provider smoke
+
+# Project A: infrastructure validation only. No PR diff, review judgment, router,
+# deterministic review rule, or GitHub posting. Only trusted main-branch code runs.
+on:
+    workflow_dispatch:
+
+permissions:
+    contents: read
+
+concurrency:
+    group: solheim-provider-smoke
+    cancel-in-progress: false
+
+jobs:
+    smoke:
+        if: github.ref == 'refs/heads/main'
+        runs-on: ubuntu-24.04
+        timeout-minutes: 25
+        # Retain the existing protected environment and secret names for compatibility.
+        # The environment requires reviewer approval, matching the deploy
+        # environment. The job guard above restricts runs to main, so the
+        # secret is released only for an approved main run. See
+        # scripts/solheim-smoke/README.md.
+        environment: final-vscode-review-smoke
+        steps:
+            - name: Checkout exact trusted main revision
+              uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+              with:
+                  ref: ${{ github.sha }}
+                  persist-credentials: false
+
+            - name: Setup Node.js and pnpm
+              uses: ./.github/actions/setup-node-pnpm
+              with:
+                  install-args: "--frozen-lockfile"
+
+            - name: Build trusted extension and webview
+              run: |
+                  pnpm -w bundle
+                  pnpm --filter @roo-code/vscode-webview build
+
+            - name: Install display dependencies
+              run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64
+
+            - name: Run Solheim provider smoke
+              id: provider
+              timeout-minutes: 8
+              env:
+                  SOLHEIM_API_KEY: ${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }}
+                  SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke
+              run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh
+
+            # The recorder publishes smoke.mp4 only through an atomic rename
+            # after a nonempty check and ffprobe validation, so the published
+            # final is the commit marker: present and nonempty means complete.
+            # There is no separate persisted status marker.
+            - name: Verify recording integrity
+              if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}
+              env:
+                  SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke
+              run: |
+                  test -s "$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4"
+                  echo "Recording published and nonempty: $SOLHEIM_SMOKE_OUT_DIR/smoke.mp4"
+
+            # Verdict and trace evidence must survive even when the recording
+            # is missing, so metadata and video are uploaded separately.
+            - name: Upload provider smoke verdict and trace
+              if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}
+              uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+              with:
+                  name: solheim-provider-smoke-${{ github.sha }}
+                  path: |
+                      ${{ runner.temp }}/solheim-provider-smoke/verdict.json
+                      ${{ runner.temp }}/solheim-provider-smoke/trace.json
+                  if-no-files-found: error
+                  retention-days: 7
+
+            # Exact validated final path only; the recorder leaves no temporary
+            # file behind, so a failed upload here reports the missing recording.
+            - name: Upload provider smoke recording
+              if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}
+              uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+              with:
+                  name: solheim-provider-smoke-recording-${{ github.sha }}
+                  path: ${{ runner.temp }}/solheim-provider-smoke/smoke.mp4
+                  if-no-files-found: error
+                  retention-days: 7

package.json

@@ -13,6 +13,12 @@
 		"check-types": "turbo check-types --log-order grouped --output-logs new-only",
 		"test": "turbo test --log-order grouped --output-logs new-only",
 		"test:code-qa-ci": "node --test scripts/code-qa-workflow.test.mjs",
+		"solheim:smoke": "node --import tsx scripts/solheim-smoke/driver.mts",
+		"test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs",
+		"test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts",
+		"solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json",
+		"test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts",
+		"agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json",
 		"test:mutation-ci": "node --test scripts/stryker-diff.test.mjs",
 		"lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check",
 		"fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts",
@@ -38,6 +44,9 @@
 	"devDependencies": {
 		"@changesets/cli": "2.31.1",
 		"@roo-code/config-typescript": "workspace:^",
+		"@roo-code/ipc": "workspace:^",
+		"@roo-code/types": "workspace:^",
+		"@vscode/test-electron": "2.5.2",
 		"@stryker-mutator/core": "10.0.0",
 		"@stryker-mutator/vitest-runner": "10.0.0",
 		"@types/node": "22.20.1",
@@ -52,7 +61,8 @@
 		"rimraf": "6.0.1",
 		"tsx": "4.23.15",
 		"turbo": "2.11.8",
-		"typescript": "5.9.3"
+		"typescript": "5.9.3",
+		"yaml": "2.9.1"
 	},
 	"lint-staged": {
 		"src/**/*.ts": [

packages/ipc/package.json

@@ -5,6 +5,7 @@
 	"type": "module",
 	"exports": "./src/index.ts",
 	"scripts": {
+		"test": "node --import tsx --test src/__tests__/*.test.ts",
 		"lint": "eslint src --ext=ts --max-warnings=0",
 		"check-types": "tsc --noEmit",
 		"clean": "rimraf .turbo"

packages/ipc/src/__tests__/ipc-server.test.ts

@@ -0,0 +1,65 @@
+import assert from "node:assert/strict"
+import { describe, it } from "node:test"
+import { IpcMessageType, IpcOrigin, TaskCommandName, ipcMessageSchema } from "@roo-code/types"
+import { IpcServer } from "../ipc-server.js"
+
+function fixture() {
+	const logs: unknown[][] = []
+	const commands: unknown[] = []
+	const server = new IpcServer("/unused-test.sock", (...args) => logs.push(args))
+	server.on(IpcMessageType.TaskCommand, (clientId, command) => commands.push({ clientId, command }))
+	return { logs, commands, receive: (data: unknown) => server["onMessage"](data) }
+}
+const key = "dummy-api-key-not-for-public-logging"
+const command = (requestId: string) => ({
+	type: IpcMessageType.TaskCommand,
+	origin: IpcOrigin.Client,
+	clientId: "client-1",
+	data: {
+		commandName: TaskCommandName.StartNewTask,
+		data: { requestId, text: "private prompt", configuration: { apiProvider: "openai", openAiApiKey: key } },
+	},
+})
+
+describe("IPC rejection logging", () => {
+	it("rejects an invalid request ID without logging its key, prompt or schema issue values", () => {
+		const f = fixture()
+		f.receive(command(`invalid request containing ${key}`))
+		assert.deepEqual(f.commands, [])
+		assert.equal(f.logs.length, 1)
+		assert.equal(f.logs[0]?.[0], "[server#onMessage] invalid payload")
+		assert.ok(!JSON.stringify(f.logs).includes(key))
+		assert.ok(!JSON.stringify(f.logs).includes("private prompt"))
+		assert.ok(!JSON.stringify(f.logs).includes("invalid request containing"))
+	})
+	it("does not log credential-shaped primitive input or dynamic issue paths", () => {
+		const f = fixture()
+		const valid = command("valid-id")
+		const malformedHeaders = {
+			...valid,
+			data: {
+				...valid.data,
+				data: {
+					...valid.data.data,
+					configuration: { ...valid.data.data.configuration, openAiHeaders: { [key]: null } },
+				},
+			},
+		}
+		const rejected = ipcMessageSchema.safeParse(malformedHeaders)
+		assert.ok(!rejected.success)
+		assert.ok(rejected.error.issues.some((issue) => issue.path.includes(key)))
+		for (const data of [key, null, { type: key }, malformedHeaders]) {
+			f.receive(data)
+		}
+		assert.equal(f.logs.length, 4)
+		assert.ok(!JSON.stringify(f.logs).includes(key))
+		assert.deepEqual(f.commands, [])
+	})
+	it("still dispatches a valid correlated command to its client without logging its settings", () => {
+		const f = fixture()
+		f.receive(command("valid-id"))
+		assert.equal(f.commands.length, 1)
+		assert.deepEqual(f.commands[0], { clientId: "client-1", command: command("valid-id").data })
+		assert.deepEqual(f.logs, [])
+	})
+})

packages/ipc/src/ipc-server.ts

@@ -76,16 +76,15 @@ export class IpcServer extends EventEmitter<IpcServerEvents> implements RooCodeI
 
 	private onMessage(data: unknown) {
 		if (typeof data !== "object") {
-			this.log(`[server#onMessage] invalid data -> ${JSON.stringify(data)}`)
+			this.log("[server#onMessage] invalid data")
 			return
 		}
 
 		const result = ipcMessageSchema.safeParse(data)
 
 		if (!result.success) {
-			this.log(
-				`[server#onMessage] invalid payload -> ${JSON.stringify(result.error.issues)} -> ${JSON.stringify(data)}`,
-			)
+			// Both the input and Zod issue values/paths may contain credentials.
+			this.log("[server#onMessage] invalid payload", { issueCount: result.error.issues.length })
 
 			return
 		}
@@ -98,7 +97,7 @@ export class IpcServer extends EventEmitter<IpcServerEvents> implements RooCodeI
 					this.emit(IpcMessageType.TaskCommand, payload.clientId, payload.data)
 					break
 				default:
-					this.log(`[server#onMessage] unhandled payload: ${JSON.stringify(payload)}`)
+					this.log("[server#onMessage] unhandled payload")
 					break
 			}
 		}