Skip to content

Commit bb46786

Browse files
edelaunaedelauna
committed
feat(ci): add recorded Solheim provider smoke test
1 parent f780647 commit bb46786

22 files changed

Lines changed: 1832 additions & 26 deletions

‎.github/workflows/code-qa.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,12 @@ jobs:
9292
run: pnpm check-types
9393
- name: Validate Code QA workflow
9494
run: pnpm test:code-qa-ci
95+
- name: Validate Solheim provider smoke workflow
96+
run: pnpm test:solheim-smoke-ci
97+
- name: Test Solheim provider smoke units
98+
run: pnpm test:solheim-smoke:unit
99+
- name: Check Solheim provider smoke types
100+
run: pnpm solheim-smoke:check-types
95101
- name: Model-check task lifecycle protocols
96102
run: pnpm lifecycle:model-check
97103
- name: Validate MCP OAuth integration

‎.github/workflows/solheim-provider-smoke.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Solheim provider smoke
2+
3+
# Project A: infrastructure validation only. No PR diff, review judgment, router,
4+
# deterministic review rule, or GitHub posting. Only trusted main-branch code runs.
5+
on:
6+
workflow_dispatch:
7+
8+
permissions:
9+
contents: read
10+
11+
concurrency:
12+
group: solheim-provider-smoke
13+
cancel-in-progress: false
14+
15+
jobs:
16+
smoke:
17+
if: github.ref == 'refs/heads/main'
18+
runs-on: ubuntu-24.04
19+
timeout-minutes: 25
20+
# Retain the existing protected environment and secret names for compatibility.
21+
# Repository policy must restrict deployments to main and require approval.
22+
environment: final-vscode-review-smoke
23+
steps:
24+
- name: Checkout exact trusted main revision
25+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
26+
with:
27+
ref: ${{ github.sha }}
28+
persist-credentials: false
29+
30+
- name: Setup Node.js and pnpm
31+
uses: ./.github/actions/setup-node-pnpm
32+
with:
33+
install-args: "--frozen-lockfile"
34+
35+
- name: Build trusted extension and webview
36+
run: |
37+
pnpm -w bundle
38+
pnpm --filter @roo-code/vscode-webview build
39+
40+
- name: Install display dependencies
41+
run: sudo apt-get update && sudo apt-get install -y xvfb ffmpeg libasound2t64
42+
43+
- name: Run Solheim provider smoke
44+
id: provider
45+
timeout-minutes: 8
46+
env:
47+
SOLHEIM_API_KEY: ${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }}
48+
SOLHEIM_SMOKE_OUT_DIR: ${{ runner.temp }}/solheim-provider-smoke
49+
run: xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp" bash scripts/solheim-smoke/record.sh
50+
51+
- name: Upload provider smoke verdict and recording
52+
if: ${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}
53+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
54+
with:
55+
name: solheim-provider-smoke-${{ github.sha }}
56+
path: |
57+
${{ runner.temp }}/solheim-provider-smoke/verdict.json
58+
${{ runner.temp }}/solheim-provider-smoke/smoke.mp4
59+
if-no-files-found: error
60+
retention-days: 7

‎package.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
"check-types": "turbo check-types --log-order grouped --output-logs new-only",
1414
"test": "turbo test --log-order grouped --output-logs new-only",
1515
"test:code-qa-ci": "node --test scripts/code-qa-workflow.test.mjs",
16+
"solheim:smoke": "node --import tsx scripts/solheim-smoke/driver.mts",
17+
"test:solheim-smoke-ci": "node --test scripts/solheim-smoke-workflow.test.mjs",
18+
"test:solheim-smoke:unit": "node --import tsx --test scripts/solheim-smoke/*.test.ts",
19+
"solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json",
1620
"test:mutation-ci": "node --test scripts/stryker-diff.test.mjs",
1721
"lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts",
1822
"fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts",
@@ -34,6 +38,9 @@
3438
"devDependencies": {
3539
"@changesets/cli": "2.31.0",
3640
"@roo-code/config-typescript": "workspace:^",
41+
"@roo-code/ipc": "workspace:^",
42+
"@roo-code/types": "workspace:^",
43+
"@vscode/test-electron": "2.5.2",
3744
"@stryker-mutator/core": "10.0.0",
3845
"@stryker-mutator/vitest-runner": "10.0.0",
3946
"@types/node": "22.20.1",

‎packages/types/src/__tests__/events.test.ts‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
import {
2+
RooCodeEventName,
3+
TASK_START_FAILURE_ERROR_CODE,
4+
TASK_START_FAILURE_ERROR_MESSAGE,
5+
TASK_START_STAGES,
6+
taskEventSchema,
7+
taskStartResponseSchema,
8+
} from "../events.js"
9+
10+
const requestId = "final-smoke-0123abcdef456789"
11+
12+
describe("taskStartResponseSchema", () => {
13+
it("accepts a success response with the accepted task ID", () => {
14+
const result = taskStartResponseSchema.safeParse({ requestId, success: true, taskId: "task-1" })
15+
expect(result.success).toBe(true)
16+
17+
if (result.success) {
18+
expect(result.data).toEqual({ requestId, success: true, taskId: "task-1" })
19+
}
20+
})
21+
22+
it("accepts the fixed failure response", () => {
23+
const result = taskStartResponseSchema.safeParse({
24+
requestId,
25+
success: false,
26+
errorCode: TASK_START_FAILURE_ERROR_CODE,
27+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
28+
})
29+
expect(result.success).toBe(true)
30+
31+
if (result.success && !result.data.success) {
32+
expect(result.data.success).toBe(false)
33+
expect(result.data.stage).toBeUndefined()
34+
}
35+
})
36+
37+
it("accepts a failure response carrying each allowed stage", () => {
38+
for (const stage of TASK_START_STAGES) {
39+
const result = taskStartResponseSchema.safeParse({
40+
requestId,
41+
success: false,
42+
errorCode: TASK_START_FAILURE_ERROR_CODE,
43+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
44+
stage,
45+
})
46+
expect(result.success).toBe(true)
47+
48+
if (result.success && !result.data.success) {
49+
expect(result.data.stage).toBe(stage)
50+
}
51+
}
52+
})
53+
54+
it("rejects a failure response carrying an unknown stage", () => {
55+
const result = taskStartResponseSchema.safeParse({
56+
requestId,
57+
success: false,
58+
errorCode: TASK_START_FAILURE_ERROR_CODE,
59+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
60+
stage: "secretStore",
61+
})
62+
expect(result.success).toBe(false)
63+
})
64+
65+
it("rejects a failure response carrying a non-string stage", () => {
66+
const result = taskStartResponseSchema.safeParse({
67+
requestId,
68+
success: false,
69+
errorCode: TASK_START_FAILURE_ERROR_CODE,
70+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
71+
stage: { name: "taskCreation" },
72+
})
73+
expect(result.success).toBe(false)
74+
})
75+
76+
it("rejects a success response carrying a stage", () => {
77+
const result = taskStartResponseSchema.safeParse({
78+
requestId,
79+
success: true,
80+
taskId: "task-1",
81+
stage: "taskCreation",
82+
})
83+
expect(result.success).toBe(false)
84+
})
85+
86+
it("rejects a success response without a taskId", () => {
87+
const result = taskStartResponseSchema.safeParse({ requestId, success: true })
88+
expect(result.success).toBe(false)
89+
})
90+
91+
it("rejects a failure response carrying an unfixed error code", () => {
92+
const result = taskStartResponseSchema.safeParse({
93+
requestId,
94+
success: false,
95+
errorCode: "provider_auth_failed",
96+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
97+
})
98+
expect(result.success).toBe(false)
99+
})
100+
101+
it("rejects a failure response carrying a raw error message", () => {
102+
const result = taskStartResponseSchema.safeParse({
103+
requestId,
104+
success: false,
105+
errorCode: TASK_START_FAILURE_ERROR_CODE,
106+
errorMessage: "openai rejected the key sk-secret-123 at https://api.example.com",
107+
})
108+
expect(result.success).toBe(false)
109+
})
110+
111+
it("rejects an empty requestId", () => {
112+
const result = taskStartResponseSchema.safeParse({ requestId: "", success: true, taskId: "task-1" })
113+
expect(result.success).toBe(false)
114+
})
115+
116+
it("rejects a success response with an extra top-level field", () => {
117+
const result = taskStartResponseSchema.safeParse({
118+
requestId,
119+
success: true,
120+
taskId: "task-1",
121+
providerError: "openai rejected the key sk-secret-123",
122+
})
123+
expect(result.success).toBe(false)
124+
})
125+
126+
it("rejects a failure response with an extra field on the error object", () => {
127+
const result = taskStartResponseSchema.safeParse({
128+
requestId,
129+
success: false,
130+
errorCode: TASK_START_FAILURE_ERROR_CODE,
131+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
132+
detail: { apiKey: "sk-secret-123", url: "https://api.example.com" },
133+
})
134+
expect(result.success).toBe(false)
135+
})
136+
})
137+
138+
describe("taskEventSchema TaskStartResponse", () => {
139+
it("routes a success response through the task event envelope", () => {
140+
const result = taskEventSchema.safeParse({
141+
eventName: RooCodeEventName.TaskStartResponse,
142+
payload: [{ requestId, success: true, taskId: "task-1" }],
143+
})
144+
expect(result.success).toBe(true)
145+
})
146+
147+
it("routes a failure response through the task event envelope", () => {
148+
const result = taskEventSchema.safeParse({
149+
eventName: RooCodeEventName.TaskStartResponse,
150+
payload: [
151+
{
152+
requestId,
153+
success: false,
154+
errorCode: TASK_START_FAILURE_ERROR_CODE,
155+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
156+
},
157+
],
158+
})
159+
expect(result.success).toBe(true)
160+
})
161+
162+
it("routes a failure response with a stage through the task event envelope", () => {
163+
const result = taskEventSchema.safeParse({
164+
eventName: RooCodeEventName.TaskStartResponse,
165+
payload: [
166+
{
167+
requestId,
168+
success: false,
169+
errorCode: TASK_START_FAILURE_ERROR_CODE,
170+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
171+
stage: "taskCreation",
172+
},
173+
],
174+
})
175+
expect(result.success).toBe(true)
176+
})
177+
178+
it("rejects a failure payload with an unknown stage through the task event envelope", () => {
179+
const result = taskEventSchema.safeParse({
180+
eventName: RooCodeEventName.TaskStartResponse,
181+
payload: [
182+
{
183+
requestId,
184+
success: false,
185+
errorCode: TASK_START_FAILURE_ERROR_CODE,
186+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
187+
stage: "keyring",
188+
},
189+
],
190+
})
191+
expect(result.success).toBe(false)
192+
})
193+
194+
it("rejects a failure payload with a stage plus an extra field through the task event envelope", () => {
195+
const result = taskEventSchema.safeParse({
196+
eventName: RooCodeEventName.TaskStartResponse,
197+
payload: [
198+
{
199+
requestId,
200+
success: false,
201+
errorCode: TASK_START_FAILURE_ERROR_CODE,
202+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
203+
stage: "taskCreation",
204+
rawError: "sk-secret-123",
205+
},
206+
],
207+
})
208+
expect(result.success).toBe(false)
209+
})
210+
211+
it("rejects a success payload with an extra field through the task event envelope", () => {
212+
const result = taskEventSchema.safeParse({
213+
eventName: RooCodeEventName.TaskStartResponse,
214+
payload: [{ requestId, success: true, taskId: "task-1", stack: "at run (api.ts:1:1)" }],
215+
})
216+
expect(result.success).toBe(false)
217+
})
218+
219+
it("rejects a failure payload with an extra error field through the task event envelope", () => {
220+
const result = taskEventSchema.safeParse({
221+
eventName: RooCodeEventName.TaskStartResponse,
222+
payload: [
223+
{
224+
requestId,
225+
success: false,
226+
errorCode: TASK_START_FAILURE_ERROR_CODE,
227+
errorMessage: TASK_START_FAILURE_ERROR_MESSAGE,
228+
rawError: "sk-secret-123",
229+
},
230+
],
231+
})
232+
expect(result.success).toBe(false)
233+
})
234+
})

‎packages/types/src/__tests__/ipc.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,4 +117,59 @@ describe("IPC Types", () => {
117117
expect(result.success).toBe(false)
118118
})
119119
})
120+
121+
describe("StartNewTask requestId", () => {
122+
const legacyCommand = {
123+
commandName: TaskCommandName.StartNewTask,
124+
data: {
125+
configuration: {},
126+
text: "review the diff",
127+
},
128+
}
129+
130+
it("should accept a legacy StartNewTask without requestId", () => {
131+
const result = taskCommandSchema.safeParse(legacyCommand)
132+
expect(result.success).toBe(true)
133+
134+
if (result.success && result.data.commandName === TaskCommandName.StartNewTask) {
135+
expect(result.data.data.requestId).toBeUndefined()
136+
}
137+
})
138+
139+
it("should accept and preserve a valid requestId", () => {
140+
const result = taskCommandSchema.safeParse({
141+
...legacyCommand,
142+
data: { ...legacyCommand.data, requestId: "final-smoke-0123abcdef456789" },
143+
})
144+
expect(result.success).toBe(true)
145+
146+
if (result.success && result.data.commandName === TaskCommandName.StartNewTask) {
147+
expect(result.data.data.requestId).toBe("final-smoke-0123abcdef456789")
148+
}
149+
})
150+
151+
it("should reject an empty requestId", () => {
152+
const result = taskCommandSchema.safeParse({
153+
...legacyCommand,
154+
data: { ...legacyCommand.data, requestId: "" },
155+
})
156+
expect(result.success).toBe(false)
157+
})
158+
159+
it("should reject a requestId longer than 128 characters", () => {
160+
const result = taskCommandSchema.safeParse({
161+
...legacyCommand,
162+
data: { ...legacyCommand.data, requestId: "a".repeat(129) },
163+
})
164+
expect(result.success).toBe(false)
165+
})
166+
167+
it("should reject a requestId with characters outside the allowed set", () => {
168+
const result = taskCommandSchema.safeParse({
169+
...legacyCommand,
170+
data: { ...legacyCommand.data, requestId: "id with spaces" },
171+
})
172+
expect(result.success).toBe(false)
173+
})
174+
})
120175
})

0 commit comments

Comments
 (0)