From 506a4b1c0fe935c48e450dfc35bbd929f2d34658 Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 4 Oct 2026 15:49:17 +0000 Subject: [PATCH 01/14] Fix interrupted child redelegation Amp-Thread-ID: https://ampcode.com/threads/T-01a1077e-5823-744c-ad1a-1653a3ccdac5 --- docs/architecture/task-lifecycle-model.md | 3 +- scripts/check-task-lifecycle.ts | 36 ++++++++++++-- .../ClineProvider.delegation.spec.ts | 32 +++++++++++++ src/core/task-persistence/TaskHistoryStore.ts | 45 ++++++++++++++++- .../TaskHistoryStore.realConcurrency.spec.ts | 48 +++++++++++++++++++ .../__tests__/taskLifecycle.spec.ts | 19 ++++++++ src/core/task-persistence/taskLifecycle.ts | 12 +++++ src/core/task/Task.ts | 10 ++++ .../task/__tests__/Task.persistence.spec.ts | 36 ++++++++++++++ src/core/webview/ClineProvider.ts | 26 ++++++++++ 10 files changed, 260 insertions(+), 7 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index eaeb61d987..50ada81c0c 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -47,13 +47,14 @@ TLA+/PlusCal or Quint with TLC becomes a better fit when the lifecycle needs tem | Task record and status | `HistoryItem` persisted by `TaskHistoryStore` | | `delegate(parent, child)` | `ClineProvider.delegateParentAndOpenChild` | | `interrupt(child)` | cancellation or eviction through `markDelegatedChildInterrupted` | +| `resume(child)` | accepted `resume_task` prompt through `TaskHistoryStore.resumeInterruptedTask` | | `complete(child)` | `ClineProvider.reopenParentFromDelegation` | | `abandon(child)` | `ClineProvider.abandonSubtask` | | Pending-action settlement | `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear in the rejected-delegation settlement path (#1714) | | Atomic event step | `atomicReadAndUpdate`, `atomicUpdatePair`, and per-parent delegation transition lock | | Event interleaving | Competing completion, cancellation, abandonment, and new delegation calls | -The model has three fixed task slots, enough to cover competing siblings and a nested parent-child-grandchild chain. It explores every reachable interleaving through depth 12, deduplicating canonical states. Representative checks also exercise rejected operations that do not create a new state: a second concurrent delegation while the first child is active, stale completion after re-delegation, late completion after abandonment, completion after interruption, and nested completion. Named semantic landmarks require the graph to retain interrupted-child re-delegation and nested delegation even when the raw state total changes. +The model has three fixed task slots, enough to cover competing siblings and a nested parent-child-grandchild chain. It explores every reachable interleaving through depth 13, deduplicating canonical states. Representative checks also exercise rejected operations that do not create a new state: a second concurrent delegation while the first child is active, stale completion after re-delegation, late completion after abandonment, completion after interruption, and nested completion. Named semantic landmarks require the graph to retain interrupted-child re-delegation, nested delegation, and an explicitly resumed interrupted child delegating to a grandchild even when the raw state total changes. The resume action calls the production reducer and carries a model-only provenance bit so the explorer can distinguish that path from a child that was never interrupted; generic persisted transitions still reject `interrupted → active`. Each task slot can also hold one of two pending `create_subtask` actions. A `stage` action mirrors `setPendingTaskAction` overwrite semantics, delegation clears the action its request carried, completion clears the child's action only when its event carries the matching action ID, and a `settle-rejected` action models the settlement that follows an authoritative delegation rejection (#1714). Production settles through the typed `LifecycleTransitionError` from the shared guards: the provider calls the disk-authoritative `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear under the per-file lock, then propagates the original rejection. Six named witnesses must remain reachable: settlement from an interrupted record after rejection, settlement through a successful active delegation, unrelated-action preservation during completion, stale-action protection where a settlement targeting one action ID leaves a replacement action intact, matching-ID completion clearing, and replacement-ID completion preservation. A mismatched pending-action request keeps its production behavior: the atomic update throws before any transition, and no settlement runs. diff --git a/scripts/check-task-lifecycle.ts b/scripts/check-task-lifecycle.ts index bf05ad68e9..411bb8175a 100644 --- a/scripts/check-task-lifecycle.ts +++ b/scripts/check-task-lifecycle.ts @@ -7,12 +7,16 @@ import { completeDelegatedChild, delegateTaskToChild, interruptDelegatedChild, + resumeInterruptedTask, settleRejectedCreateSubtaskAction, } from "../src/core/task-persistence/taskLifecycle" const taskIds = ["parent", "child-a", "child-b"] as const type TaskId = (typeof taskIds)[number] -type ModelState = Record +interface ModelTask extends HistoryItem { + modelWasResumed?: boolean +} +type ModelState = Record interface Transition { name: string @@ -33,10 +37,10 @@ interface WitnessContext { transition: Transition } -const MAX_DEPTH = 12 +const MAX_DEPTH = 13 const MAX_STATES = 10_000 const actionIds = ["action-1", "action-2"] as const -const expectedActions = ["delegate", "interrupt", "complete", "abandon", "stage", "settle-rejected"] as const +const expectedActions = ["delegate", "interrupt", "resume", "complete", "abandon", "stage", "settle-rejected"] as const const semanticLandmarks = { "interrupted-child-redelegation": (state: ModelState) => state.parent?.status === "delegated" && @@ -47,6 +51,12 @@ const semanticLandmarks = { state.parent.awaitingChildId === "child-a" && state["child-a"]?.status === "delegated" && state["child-a"].awaitingChildId === "child-b", + "resumed-interrupted-child-nested-delegation": (state: ModelState) => + state.parent?.status === "delegated" && + state.parent.awaitingChildId === "child-a" && + state["child-a"]?.status === "delegated" && + state["child-a"].awaitingChildId === "child-b" && + state["child-a"].modelWasResumed === true, } satisfies Record boolean> const semanticWitnesses = { "interrupted-pending-delegation-settled": ({ prev, next, transition }: WitnessContext) => @@ -101,7 +111,7 @@ const semanticWitnesses = { }, } satisfies Record boolean> -function task(id: TaskId, parentTaskId?: TaskId): HistoryItem { +function task(id: TaskId, parentTaskId?: TaskId): ModelTask { return { id, number: taskIds.indexOf(id), @@ -132,7 +142,7 @@ function initialState(): ModelState { return { parent: task("parent"), "child-a": undefined, "child-b": undefined } } -function replace(state: ModelState, ...updates: HistoryItem[]): ModelState { +function replace(state: ModelState, ...updates: ModelTask[]): ModelState { const next = { ...state } for (const update of updates) next[update.id as TaskId] = update return next @@ -176,6 +186,18 @@ function transitions(state: ModelState): Transition[] { }) } } + + const lineageParent = parent.parentTaskId ? state[parent.parentTaskId as TaskId] : undefined + const resumeValid = + parent.status === "interrupted" && + (!parent.parentTaskId || + (lineageParent?.status === "delegated" && lineageParent.awaitingChildId === parent.id)) + if (resumeValid) { + result.push({ + name: `resume(${parentId})`, + next: replace(state, { ...resumeInterruptedTask(parent), modelWasResumed: true }), + }) + } } for (const childId of taskIds) { @@ -416,6 +438,10 @@ function runRepresentativeScenarios(): void { assert.throws(() => delegateTaskToChild(delegated, "child-b", "active"), /not interrupted/) const interruptedA = interruptDelegatedChild(delegated, childA) + const resumedA = resumeInterruptedTask(interruptedA) + const resumedNested = delegateTaskToChild(resumedA, "child-b") + assert.equal(resumedNested.status, "delegated") + assert.equal(resumedNested.awaitingChildId, "child-b") const redelegated = delegateTaskToChild(delegated, "child-b", interruptedA.status) assert.throws(() => completeDelegatedChild(redelegated, interruptedA, "stale"), /not delegated to child/) diff --git a/src/__tests__/ClineProvider.delegation.spec.ts b/src/__tests__/ClineProvider.delegation.spec.ts index 68f39c6517..02b41a4d72 100644 --- a/src/__tests__/ClineProvider.delegation.spec.ts +++ b/src/__tests__/ClineProvider.delegation.spec.ts @@ -10,6 +10,7 @@ import { providerIdentifiers, RooCodeEventName } from "@roo-code/types" import { ClineProvider } from "../core/webview/ClineProvider" import { TaskScheduler } from "../core/task/TaskScheduler" import { LifecycleTransitionError } from "../core/task-persistence" +import { makeProviderStub } from "./helpers/provider-stub" const parentHistoryItem: HistoryItem = { id: "parent-1", @@ -51,6 +52,37 @@ const makeParentTask = () => retrySaveApiConversationHistory: vi.fn(), }) as any +describe("ClineProvider.resumeInterruptedTask()", () => { + it("resumes a delegated child only while its parent still awaits it", async () => { + const resumed = { id: "child-1", status: "active" } as HistoryItem + const taskHistoryStore = { + invalidate: vi.fn().mockResolvedValue(undefined), + get: vi.fn().mockReturnValue({ id: "parent-1", status: "delegated", awaitingChildId: "child-1" }), + resumeInterruptedTask: vi.fn().mockResolvedValue(resumed), + } + const provider = makeProviderStub({ taskHistoryStore, isViewLaunched: false }) + + await ClineProvider.prototype.resumeInterruptedTask.call(provider, "child-1", "parent-1") + + expect(taskHistoryStore.invalidate).toHaveBeenCalledWith("parent-1") + expect(taskHistoryStore.resumeInterruptedTask).toHaveBeenCalledWith("child-1") + }) + + it("rejects a stale interrupted child after its parent delegates elsewhere", async () => { + const taskHistoryStore = { + invalidate: vi.fn().mockResolvedValue(undefined), + get: vi.fn().mockReturnValue({ id: "parent-1", status: "delegated", awaitingChildId: "child-2" }), + resumeInterruptedTask: vi.fn(), + } + const provider = makeProviderStub({ taskHistoryStore, isViewLaunched: false }) + + await expect( + ClineProvider.prototype.resumeInterruptedTask.call(provider, "child-1", "parent-1"), + ).rejects.toThrow("parent parent-1 no longer awaits it") + expect(taskHistoryStore.resumeInterruptedTask).not.toHaveBeenCalled() + }) +}) + describe("ClineProvider.delegateParentAndOpenChild()", () => { it("rejects a stale restored action before delegation side effects", async () => { const parentTask = makeParentTask() diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index 346d697d9a..c6d811ad64 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -10,7 +10,12 @@ import { GlobalFileNames } from "../../shared/globalFileNames" import { LOCK_STALE_MS, withFileLock } from "../../utils/fileLock" import { safeWriteJson } from "../../utils/safeWriteJson" import { getStorageBasePath } from "../../utils/storage" -import { assertValidTransition, settleRejectedCreateSubtaskAction, type HistoryItemStatus } from "./taskLifecycle" +import { + assertValidTransition, + resumeInterruptedTask as resumeInterruptedTaskRecord, + settleRejectedCreateSubtaskAction, + type HistoryItemStatus, +} from "./taskLifecycle" import { computeHistoryDelta, DeltaRejectedError, mergeHistoryDelta } from "./taskStoreConcurrency" export { assertValidTransition, type HistoryItemStatus } from "./taskLifecycle" @@ -1073,6 +1078,44 @@ export class TaskHistoryStore { }) } + /** + * Disk-authoritative transition used only after the user accepts a resume + * prompt. Generic writes intentionally reject interrupted → active so stale + * task snapshots cannot revive cancelled work. + */ + public async resumeInterruptedTask(taskId: string): Promise { + return this.withLock(async () => { + const cached = this.cache.get(taskId) + if (!cached) { + throw new Error(`[TaskHistoryStore] resumeInterruptedTask: task ${taskId} not found in cache`) + } + + const filePath = await this.getTaskFilePath(taskId) + let authoritative: HistoryItem = cached + await safeWriteJson(filePath, cached, { + merge: (existing) => { + const parsed = historyItemSchema.safeParse(existing) + if (!parsed.success || parsed.data.id !== taskId) { + this.cache.delete(taskId) + this.taskFileMtimes.delete(taskId) + throw new Error( + `[TaskHistoryStore] resumeInterruptedTask: task ${taskId} has no valid disk record`, + ) + } + + authoritative = resumeInterruptedTaskRecord(existing as HistoryItem) + return authoritative + }, + }) + + this.cache.set(taskId, authoritative) + if (this.onWrite) { + await this.onWrite(this.getAll()) + } + return authoritative + }) + } + /** * Disk-authoritative compare-and-clear for a rejected `create_subtask` * pending action (#1714). The comparison runs inside the per-file diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index d35dfc49cf..e6de78b4b8 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -91,6 +91,54 @@ function createAction(actionId: string, message: string) { } describe("TaskHistoryStore real cross-host locking", () => { + it("resumes from the authoritative interrupted record even when the caller cache is stale", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-history-resume-lock-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + + try { + await storeA.initialize() + await storeA.upsert(item("shared-task")) + await storeB.initialize() + await storeA.upsert({ ...item("shared-task"), status: "interrupted" }) + + expect(storeB.get("shared-task")?.status).toBe("active") + await expect(storeB.resumeInterruptedTask("shared-task")).resolves.toMatchObject({ + id: "shared-task", + status: "active", + }) + await storeA.invalidate("shared-task") + expect(storeA.get("shared-task")?.status).toBe("active") + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("rejects resume when the authoritative record is no longer interrupted", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-history-resume-rejected-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + + try { + await storeA.initialize() + await storeA.upsert(item("shared-task")) + await storeB.initialize() + await storeA.upsert({ ...item("shared-task"), status: "completed" }) + + await expect(storeB.resumeInterruptedTask("shared-task")).rejects.toThrow( + "Cannot resume task shared-task with status completed", + ) + await storeB.invalidate("shared-task") + expect(storeB.get("shared-task")?.status).toBe("completed") + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + it("preserves independent stale-cache deltas through the real per-file lock", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-history-real-lock-")) const storeA = new TaskHistoryStore(storagePath) diff --git a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts index 5d3129e023..f70245cb35 100644 --- a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts +++ b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts @@ -6,6 +6,7 @@ import { delegateTaskToChild, interruptDelegatedChild, LifecycleTransitionError, + resumeInterruptedTask, settleRejectedCreateSubtaskAction, } from "../taskLifecycle" @@ -59,6 +60,24 @@ describe("task lifecycle transitions", () => { }) }) + it("allows an explicitly resumed interrupted child to delegate without weakening generic transitions", () => { + const interruptedChild = item("child", { status: "interrupted", parentTaskId: "parent" }) + + expect(() => delegateTaskToChild(interruptedChild, "grandchild")).toThrow( + "Invalid task status transition: interrupted → delegated", + ) + expect(delegateTaskToChild(resumeInterruptedTask(interruptedChild), "grandchild")).toMatchObject({ + status: "delegated", + parentTaskId: "parent", + awaitingChildId: "grandchild", + delegatedToId: "grandchild", + }) + }) + + it("rejects explicit resume for a task that is no longer interrupted", () => { + expect(() => resumeInterruptedTask(item("child"))).toThrow(LifecycleTransitionError) + }) + it("interrupts a child without clearing the parent's ownership", () => { const parent = item("parent", { status: "delegated", awaitingChildId: "child", delegatedToId: "child" }) const child = item("child", { parentTaskId: "parent" }) diff --git a/src/core/task-persistence/taskLifecycle.ts b/src/core/task-persistence/taskLifecycle.ts index b8a5e571f0..c39736f36b 100644 --- a/src/core/task-persistence/taskLifecycle.ts +++ b/src/core/task-persistence/taskLifecycle.ts @@ -24,6 +24,18 @@ export function assertValidTransition(from: HistoryItemStatus | undefined, to: H } } +/** + * Reactivates a task only through the explicit user-resume path. This is kept + * outside VALID_TASK_STATUS_TRANSITIONS so generic stale writes cannot revive + * an interrupted task. + */ +export function resumeInterruptedTask(task: HistoryItem): HistoryItem { + if (task.status !== "interrupted") { + throw new LifecycleTransitionError(`Cannot resume task ${task.id} with status ${task.status ?? "active"}`) + } + return { ...task, status: "active" } +} + /** * Settles the pending create_subtask action whose delegation the authoritative * parent record rejected (#1714). Only the exact matching action ID is cleared; diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index b30ddc49b7..f2a8c7724a 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3234,6 +3234,16 @@ export class Task extends EventEmitter implements TaskLike { this.isInitialized = true const { response, text, images } = await this.ask(askType) // Calls `postStateToWebview`. + if (this.abort || this.abandoned) { + return + } + if (this.initialStatus === "interrupted") { + const provider = this.providerRef.deref() + if (!provider) { + throw new Error(`[Task#resumeTaskFromHistory] Provider unavailable for task ${this.taskId}`) + } + await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + } let responseText: string | undefined let responseImages: string[] | undefined diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index 109eee8dc0..00a10f3974 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -126,6 +126,7 @@ vi.mock("../../task-persistence", async (importOriginal) => { delete: vi.fn().mockResolvedValue(undefined), deleteMany: vi.fn().mockResolvedValue(undefined), reconcile: vi.fn().mockResolvedValue(undefined), + resumeInterruptedTask: vi.fn().mockResolvedValue({ id: "test-id", status: "active" }), initialized: Promise.resolve(), } }), @@ -299,6 +300,7 @@ describe("Task persistence", () => { mockProvider.postStateToWebview = vi.fn().mockResolvedValue(undefined) mockProvider.postStateToWebviewWithoutTaskHistory = vi.fn().mockResolvedValue(undefined) mockProvider.updateTaskHistory = vi.fn().mockResolvedValue(undefined) + mockProvider.resumeInterruptedTask = vi.fn().mockResolvedValue(undefined) mockProvider.log = vi.fn() }) @@ -1203,6 +1205,40 @@ describe("Task persistence", () => { // ── resumeTaskFromHistory — interrupted tool calls must be recorded as errors ── describe("resumeTaskFromHistory interrupted tool calls", () => { + it("durably activates an interrupted task after the user accepts resume", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) + mockProvider.taskHistoryStore.get = vi.fn().mockReturnValue({ + id: "interrupted-child-resume", + status: "interrupted", + }) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + historyItem: { + id: "interrupted-child-resume", + number: 1, + ts: 1, + task: "Interrupted child", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "interrupted", + parentTaskId: "parent", + }, + startTask: false, + }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) + const initiateTaskLoop = vi + .spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") + .mockResolvedValue(undefined) + + await getTaskPersistenceAccess(task).resumeTaskFromHistory() + + expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledWith("interrupted-child-resume", "parent") + expect(initiateTaskLoop).toHaveBeenCalledOnce() + }) + const interruptedToolResultContent = "Task was interrupted before this tool call could be completed." it("marks synthetic tool_results from an interrupted assistant turn as errors", async () => { diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index cdd0975c5d..52a45df254 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -765,6 +765,32 @@ export class ClineProvider return cleared } + public async resumeInterruptedTask(taskId: string, parentTaskId?: string): Promise { + const resume = async () => { + if (parentTaskId) { + await this.taskHistoryStore.invalidate(parentTaskId) + const parent = this.taskHistoryStore.get(parentTaskId) + if (parent?.status !== "delegated" || parent.awaitingChildId !== taskId) { + throw new LifecycleTransitionError( + `Cannot resume task ${taskId}: parent ${parentTaskId} no longer awaits it`, + ) + } + } + + const resumed = await this.taskHistoryStore.resumeInterruptedTask(taskId) + this.recentTasksCache = undefined + if (this.isViewLaunched) { + await this.postMessageToWebview({ type: "taskHistoryItemUpdated", taskHistoryItem: resumed }) + } + } + + if (parentTaskId) { + await this.runDelegationTransition(parentTaskId, resume) + } else { + await resume() + } + } + // Pending Edit Operations Management /** From 646c431273574365cbfd4ac7580850feefc00995 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Mon, 5 Oct 2026 18:27:34 +0200 Subject: [PATCH 02/14] Make Task.create share its startup promise with run Amp-Thread-ID: https://ampcode.com/threads/T-01a10cc3-a401-7090-be6f-1ae696680827 Co-authored-by: Amp --- src/core/task/Task.ts | 12 ++------- .../task/__tests__/Task.persistence.spec.ts | 27 ++++++++++++------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index f2a8c7724a..79a0477994 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1377,19 +1377,11 @@ export class Task extends EventEmitter implements TaskLike { static create(options: TaskOptions): [Task, Promise] { const instance = new Task({ ...options, startTask: false }) const { images, task, historyItem } = options - let promise - - instance.startIdleTelemetryCheck() - - if (images || task) { - promise = instance.startTask(task, images) - } else if (historyItem) { - promise = instance.resumeTaskFromHistory() - } else { + if (!images && !task && !historyItem) { throw new Error("Either historyItem or task/images must be provided") } - return [instance, promise] + return [instance, instance.run()] } // API Messages diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index 00a10f3974..0fadd0dec3 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -1205,14 +1205,16 @@ describe("Task persistence", () => { // ── resumeTaskFromHistory — interrupted tool calls must be recorded as errors ── describe("resumeTaskFromHistory interrupted tool calls", () => { - it("durably activates an interrupted task after the user accepts resume", async () => { + it("resumes a Task.create history task only once when run is also called", async () => { mockReadTaskMessages.mockResolvedValue([]) mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) mockProvider.taskHistoryStore.get = vi.fn().mockReturnValue({ id: "interrupted-child-resume", status: "interrupted", }) - const task = new Task({ + const acceptance = createDeferred<{ response: "yesButtonClicked" }>() + const initiateTaskLoop = vi.fn().mockResolvedValue(undefined) + const [task, createdRun] = Task.create({ provider: mockProvider, apiConfiguration: mockApiConfig, historyItem: { @@ -1226,15 +1228,22 @@ describe("Task persistence", () => { status: "interrupted", parentTaskId: "parent", }, - startTask: false, + onCreated: (instance) => { + vi.spyOn(instance, "ask").mockReturnValue(acceptance.promise) + vi.spyOn(getTaskPersistenceAccess(instance), "initiateTaskLoop").mockImplementation( + initiateTaskLoop, + ) + }, }) - vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) - const initiateTaskLoop = vi - .spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") - .mockResolvedValue(undefined) - - await getTaskPersistenceAccess(task).resumeTaskFromHistory() + const scheduledRun = task.run() + expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() + acceptance.resolve({ response: "yesButtonClicked" }) + await Promise.all([createdRun, scheduledRun]) + expect(scheduledRun).toBe(createdRun) + expect(task.run()).toBe(createdRun) + expect(task.ask).toHaveBeenCalledOnce() + expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledOnce() expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledWith("interrupted-child-resume", "parent") expect(initiateTaskLoop).toHaveBeenCalledOnce() }) From 07e6f522fe8d6621cc61a2f541b16b9ff079cc9f Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Mon, 5 Oct 2026 19:12:46 +0200 Subject: [PATCH 03/14] Reject stale or denied interrupted task resumes --- docs/architecture/task-lifecycle-model.md | 2 + scripts/check-task-lifecycle.ts | 8 ++- .../ClineProvider.delegation.spec.ts | 60 +++++++++++++++- .../removeClineFromStack-delegation.spec.ts | 58 ++++++++-------- src/core/task-persistence/TaskHistoryStore.ts | 7 +- .../TaskHistoryStore.realConcurrency.spec.ts | 45 ++++++++++++ .../__tests__/taskLifecycle.spec.ts | 2 +- src/core/task-persistence/taskLifecycle.ts | 5 +- src/core/task/Task.ts | 8 ++- .../task/__tests__/Task.persistence.spec.ts | 69 ++++++++++++++++++- src/core/webview/ClineProvider.ts | 4 +- 11 files changed, 225 insertions(+), 43 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 50ada81c0c..43fa6ef1bb 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -60,6 +60,8 @@ Each task slot can also hold one of two pending `create_subtask` actions. A `sta Production completion also accepts a recovery-compatible `active` parent that still awaits the returning child, then clears the stale pointers. Normal model transitions never create that intermediate state, so it is covered by a focused reducer test rather than admitted as a generally valid reachable state. +Explicit resume compares the caller's expected parent with the authoritative child backlink under the child-file lock. Abandonment writes child detachment before parent release, so a stale approval is rejected even in that partial-write window. Representative reducer scenarios cover stale linked and standalone approvals; a two-store provider test covers abandonment after the parent precheck. Rehydrated interrupted tasks omit lineage from message metadata saves, preventing their stale construction snapshots from restoring severed links after rejection. This closes abandonment-before-resume, not every cross-host ordering: parent-only redelegation, reverse-order stale abandonment, and other live-task writers remain within the ownership/generation gaps below. + ## Shared-store concurrency model The same `pnpm lifecycle:model-check` command also runs a second bounded explorer over two `TaskHistoryStore` hosts. It imports the production `computeHistoryDelta` and `mergeHistoryDelta` functions, so its semantics match the store rather than assuming coherent caches or transactional pair writes: diff --git a/scripts/check-task-lifecycle.ts b/scripts/check-task-lifecycle.ts index 411bb8175a..ecbdc9da0e 100644 --- a/scripts/check-task-lifecycle.ts +++ b/scripts/check-task-lifecycle.ts @@ -195,7 +195,7 @@ function transitions(state: ModelState): Transition[] { if (resumeValid) { result.push({ name: `resume(${parentId})`, - next: replace(state, { ...resumeInterruptedTask(parent), modelWasResumed: true }), + next: replace(state, { ...resumeInterruptedTask(parent, parent.parentTaskId), modelWasResumed: true }), }) } } @@ -438,7 +438,7 @@ function runRepresentativeScenarios(): void { assert.throws(() => delegateTaskToChild(delegated, "child-b", "active"), /not interrupted/) const interruptedA = interruptDelegatedChild(delegated, childA) - const resumedA = resumeInterruptedTask(interruptedA) + const resumedA = resumeInterruptedTask(interruptedA, parent.id) const resumedNested = delegateTaskToChild(resumedA, "child-b") assert.equal(resumedNested.status, "delegated") assert.equal(resumedNested.awaitingChildId, "child-b") @@ -447,6 +447,10 @@ function runRepresentativeScenarios(): void { const abandoned = abandonDelegatedChild(delegated, interruptedA) assert.throws(() => completeDelegatedChild(abandoned.parent, abandoned.child, "late"), /not delegated to child/) + // A resume approved against the old linkage cannot revive an abandoned child. + assert.throws(() => resumeInterruptedTask(abandoned.child, parent.id), /parent linkage changed/) + assert.throws(() => resumeInterruptedTask(interruptedA), /parent linkage changed/) + assert.equal(resumeInterruptedTask(abandoned.child).status, "active") const childB = task("child-b", "child-a") const nestedParent = delegateTaskToChild(childA, childB.id) diff --git a/src/__tests__/ClineProvider.delegation.spec.ts b/src/__tests__/ClineProvider.delegation.spec.ts index 02b41a4d72..e5fd3b3ee0 100644 --- a/src/__tests__/ClineProvider.delegation.spec.ts +++ b/src/__tests__/ClineProvider.delegation.spec.ts @@ -9,7 +9,8 @@ import type { HistoryItem } from "@roo-code/types" import { providerIdentifiers, RooCodeEventName } from "@roo-code/types" import { ClineProvider } from "../core/webview/ClineProvider" import { TaskScheduler } from "../core/task/TaskScheduler" -import { LifecycleTransitionError } from "../core/task-persistence" +import { LifecycleTransitionError, TaskHistoryStore } from "../core/task-persistence" +import { abandonDelegatedChild } from "../core/task-persistence/taskLifecycle" import { makeProviderStub } from "./helpers/provider-stub" const parentHistoryItem: HistoryItem = { @@ -65,7 +66,62 @@ describe("ClineProvider.resumeInterruptedTask()", () => { await ClineProvider.prototype.resumeInterruptedTask.call(provider, "child-1", "parent-1") expect(taskHistoryStore.invalidate).toHaveBeenCalledWith("parent-1") - expect(taskHistoryStore.resumeInterruptedTask).toHaveBeenCalledWith("child-1") + expect(taskHistoryStore.resumeInterruptedTask).toHaveBeenCalledWith("child-1", "parent-1") + }) + + it("rejects abandonment committed by another host after the parent precheck", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-abandon-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + const parent: HistoryItem = { + id: "parent", + number: 1, + ts: 1, + task: "Parent", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child: HistoryItem = { + ...parent, + id: "child", + status: "interrupted", + parentTaskId: "parent", + rootTaskId: "parent", + awaitingChildId: undefined, + delegatedToId: undefined, + childIds: [], + } + try { + await storeA.initialize() + await storeA.upsert(parent) + await storeA.upsert(child) + await storeB.initialize() + const resume = storeA.resumeInterruptedTask.bind(storeA) + vi.spyOn(storeA, "resumeInterruptedTask").mockImplementationOnce(async (id, expectedParent) => { + // This hook runs after the provider's parent check. Commit only the + // first half of abandonment: the parent still appears to await this child. + await storeB.atomicReadAndUpdate("child", (current) => abandonDelegatedChild(parent, current).child) + return resume(id, expectedParent) + }) + const provider = makeProviderStub({ taskHistoryStore: storeA, isViewLaunched: false }) + await expect( + ClineProvider.prototype.resumeInterruptedTask.call(provider, "child", "parent"), + ).rejects.toThrow("parent linkage changed") + await storeB.invalidate("child") + expect(storeA.get("child")).toEqual(storeB.get("child")) + expect(storeB.get("child")).toMatchObject({ status: "interrupted" }) + expect(storeB.get("child")?.parentTaskId).toBeUndefined() + expect(storeB.get("child")?.rootTaskId).toBeUndefined() + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } }) it("rejects a stale interrupted child after its parent delegates elsewhere", async () => { diff --git a/src/__tests__/removeClineFromStack-delegation.spec.ts b/src/__tests__/removeClineFromStack-delegation.spec.ts index 0db228662e..0e1da0743c 100644 --- a/src/__tests__/removeClineFromStack-delegation.spec.ts +++ b/src/__tests__/removeClineFromStack-delegation.spec.ts @@ -4,6 +4,7 @@ import { describe, it, expect, vi } from "vitest" import { ClineProvider } from "../core/webview/ClineProvider" import { TaskRegistry } from "../core/task/TaskRegistry" import { PendingActionSettlementError, type Task } from "../core/task/Task" +import { LifecycleTransitionError } from "../core/task-persistence/taskLifecycle" import { makeProviderStub } from "./helpers/provider-stub" type PrivateClineProviderMethods = { @@ -172,35 +173,34 @@ describe("ClineProvider.removeClineFromStack() — pure lifecycle, no delegation }) describe("ClineProvider failed history restoration cleanup", () => { - it("removes the failed task, its listeners, and its resources without saving stale history", async () => { - const cleanupListener = vi.fn() - const task = { - taskId: "failed-history-task", - instanceId: "inst-1", - emit: vi.fn(), - dispose: vi.fn().mockResolvedValue(undefined), - } as unknown as Task - const taskRegistry = new TaskRegistry() - taskRegistry.push(task) - const taskEventListeners = new Map([[task, [cleanupListener]]]) - const provider = { - taskRegistry, - taskEventListeners, - log: vi.fn(), - } as unknown as ClineProvider - - await privateClineProvider.cleanupFailedHistoryTask.call( - provider, - task, - new PendingActionSettlementError("settlement failed"), - ) - - expect(taskRegistry.getById(task.taskId)).toBeUndefined() - expect(taskRegistry.current).toBeUndefined() - expect(cleanupListener).toHaveBeenCalledOnce() - expect(taskEventListeners.has(task)).toBe(false) - expect(task.dispose).toHaveBeenCalledOnce() - }) + it.each([new PendingActionSettlementError("settlement failed"), new LifecycleTransitionError("resume rejected")])( + "removes the failed task without saving stale history after %s", + async (error) => { + const cleanupListener = vi.fn() + const task = { + taskId: "failed-history-task", + instanceId: "inst-1", + emit: vi.fn(), + dispose: vi.fn().mockResolvedValue(undefined), + } as unknown as Task + const taskRegistry = new TaskRegistry() + taskRegistry.push(task) + const taskEventListeners = new Map([[task, [cleanupListener]]]) + const provider = { + taskRegistry, + taskEventListeners, + log: vi.fn(), + } as unknown as ClineProvider + + await privateClineProvider.cleanupFailedHistoryTask.call(provider, task, error) + + expect(taskRegistry.getById(task.taskId)).toBeUndefined() + expect(taskRegistry.current).toBeUndefined() + expect(cleanupListener).toHaveBeenCalledOnce() + expect(taskEventListeners.has(task)).toBe(false) + expect(task.dispose).toHaveBeenCalledOnce() + }, + ) it("keeps the task active after an unrelated history resume failure", async () => { const cleanupListener = vi.fn() diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index c6d811ad64..2e56f27b80 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -1083,7 +1083,7 @@ export class TaskHistoryStore { * prompt. Generic writes intentionally reject interrupted → active so stale * task snapshots cannot revive cancelled work. */ - public async resumeInterruptedTask(taskId: string): Promise { + public async resumeInterruptedTask(taskId: string, expectedParentTaskId?: string): Promise { return this.withLock(async () => { const cached = this.cache.get(taskId) if (!cached) { @@ -1103,7 +1103,10 @@ export class TaskHistoryStore { ) } - authoritative = resumeInterruptedTaskRecord(existing as HistoryItem) + // Abandonment commits child detachment before releasing the parent. + // Compare the caller's linkage under this same child-file lock. + this.cache.set(taskId, existing as HistoryItem) + authoritative = resumeInterruptedTaskRecord(existing as HistoryItem, expectedParentTaskId) return authoritative }, }) diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index e6de78b4b8..92af62ec6a 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -91,6 +91,51 @@ function createAction(actionId: string, message: string) { } describe("TaskHistoryStore real cross-host locking", () => { + it("rejects resume for a task absent from cache without creating a record", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-cache-miss-")) + const store = new TaskHistoryStore(storagePath) + try { + await store.initialize() + await expect(store.resumeInterruptedTask("missing")).rejects.toThrow("task missing not found in cache") + expect(store.get("missing")).toBeUndefined() + await expect( + fs.access(path.join(storagePath, "tasks", "missing", "history_item.json")), + ).rejects.toMatchObject({ code: "ENOENT" }) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it.each(["missing", "malformed", "schema", "identity"])( + "rejects %s disk records during resume without overwriting them", + async (scenario) => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-invalid-")) + const store = new TaskHistoryStore(storagePath) + const filePath = path.join(storagePath, "tasks", "child", "history_item.json") + try { + await store.initialize() + await store.upsert({ ...item("child"), status: "interrupted" }) + const contents = + scenario === "malformed" + ? "{broken" + : JSON.stringify(scenario === "schema" ? { id: "child" } : item("other")) + if (scenario === "missing") await fs.unlink(filePath) + else await fs.writeFile(filePath, contents) + await expect(store.resumeInterruptedTask("child")).rejects.toThrow( + "task child has no valid disk record", + ) + expect(store.get("child")).toBeUndefined() + expect(store["taskFileMtimes"].has("child")).toBe(false) + if (scenario === "missing") await expect(fs.access(filePath)).rejects.toMatchObject({ code: "ENOENT" }) + else expect(await fs.readFile(filePath, "utf8")).toBe(contents) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }, + ) + it("resumes from the authoritative interrupted record even when the caller cache is stale", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-history-resume-lock-")) const storeA = new TaskHistoryStore(storagePath) diff --git a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts index f70245cb35..8387f30e74 100644 --- a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts +++ b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts @@ -66,7 +66,7 @@ describe("task lifecycle transitions", () => { expect(() => delegateTaskToChild(interruptedChild, "grandchild")).toThrow( "Invalid task status transition: interrupted → delegated", ) - expect(delegateTaskToChild(resumeInterruptedTask(interruptedChild), "grandchild")).toMatchObject({ + expect(delegateTaskToChild(resumeInterruptedTask(interruptedChild, "parent"), "grandchild")).toMatchObject({ status: "delegated", parentTaskId: "parent", awaitingChildId: "grandchild", diff --git a/src/core/task-persistence/taskLifecycle.ts b/src/core/task-persistence/taskLifecycle.ts index c39736f36b..25e0beed9e 100644 --- a/src/core/task-persistence/taskLifecycle.ts +++ b/src/core/task-persistence/taskLifecycle.ts @@ -29,10 +29,13 @@ export function assertValidTransition(from: HistoryItemStatus | undefined, to: H * outside VALID_TASK_STATUS_TRANSITIONS so generic stale writes cannot revive * an interrupted task. */ -export function resumeInterruptedTask(task: HistoryItem): HistoryItem { +export function resumeInterruptedTask(task: HistoryItem, expectedParentTaskId?: string): HistoryItem { if (task.status !== "interrupted") { throw new LifecycleTransitionError(`Cannot resume task ${task.id} with status ${task.status ?? "active"}`) } + if (task.parentTaskId !== expectedParentTaskId) { + throw new LifecycleTransitionError(`Cannot resume task ${task.id}: parent linkage changed`) + } return { ...task, status: "active" } } diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 79a0477994..40a4794c62 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1799,6 +1799,12 @@ export class Task extends EventEmitter implements TaskLike { const provider = this.providerRef.deref() const existingStatus = provider?.taskHistoryStore.get(this.taskId)?.status + if (this.initialStatus === "interrupted") { + // A rehydrated task's links may have been severed by another host. + // Only lifecycle operations own lineage after interruption, not message saves. + delete historyItem.parentTaskId + delete historyItem.rootTaskId + } await provider?.updateTaskHistory(existingStatus ? { ...historyItem, status: existingStatus } : historyItem) } catch (error) { // The message array was persisted above; a metadata or task-history failure must @@ -3226,7 +3232,7 @@ export class Task extends EventEmitter implements TaskLike { this.isInitialized = true const { response, text, images } = await this.ask(askType) // Calls `postStateToWebview`. - if (this.abort || this.abandoned) { + if (this.abort || this.abandoned || (response !== "yesButtonClicked" && response !== "messageResponse")) { return } if (this.initialStatus === "interrupted") { diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index 0fadd0dec3..30b0ae097a 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -21,6 +21,7 @@ import { ContextProxy } from "../../config/ContextProxy" import { providerIdentifiers } from "@roo-code/types/provider-identifiers" import { attemptCompletionTool, type AttemptCompletionCallbacks } from "../../tools/AttemptCompletionTool" import type { AttemptCompletionToolUse } from "../../../shared/tools" +import { LifecycleTransitionError } from "../../task-persistence/taskLifecycle" type TaskPersistenceAccess = { addToApiConversationHistory: (message: Anthropic.MessageParam) => Promise @@ -1165,7 +1166,7 @@ describe("Task persistence", () => { }, startTask: false, }) - vi.spyOn(task, "ask").mockResolvedValue({ response: "noButtonClicked" }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) mockReadApiMessages.mockResolvedValue([ { @@ -1205,6 +1206,68 @@ describe("Task persistence", () => { // ── resumeTaskFromHistory — interrupted tool calls must be recorded as errors ── describe("resumeTaskFromHistory interrupted tool calls", () => { + it.each(["noButtonClicked", "unexpected"])( + "does not activate or run after resume response %s", + async (response) => { + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + startTask: false, + initialStatus: "interrupted", + }) + const answer: Awaited> = { response: "noButtonClicked" } + // Model an unvalidated response arriving across the webview boundary. + Reflect.set(answer, "response", response) + mockProvider.taskHistoryStore.get = vi.fn().mockReturnValue({ id: task.taskId, status: "interrupted" }) + vi.spyOn(task, "ask").mockResolvedValue(answer) + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + await task["resumeTaskFromHistory"]() + expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + }, + ) + + it("does not restore stale lineage through repeated message saves after rejected resume", async () => { + const history = { + id: "detached", + number: 1, + ts: 1, + task: "Detached", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "interrupted" as const, + parentTaskId: "parent", + rootTaskId: "root", + } + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + startTask: false, + historyItem: history, + }) + mockProvider.taskHistoryStore.get = vi + .fn() + .mockReturnValue({ ...history, parentTaskId: undefined, rootTaskId: undefined }) + mockProvider.resumeInterruptedTask = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + mockTaskMetadata.mockResolvedValueOnce({ historyItem: { ...history }, tokenUsage: {} }) + mockTaskMetadata.mockResolvedValueOnce({ historyItem: { ...history }, tokenUsage: {} }) + await task["saveClineMessages"]() + await task["saveClineMessages"]() + expect(loop).not.toHaveBeenCalled() + expect(mockProvider.updateTaskHistory).toHaveBeenCalled() + for (const [saved] of vi.mocked(mockProvider.updateTaskHistory).mock.calls) { + expect(saved).not.toHaveProperty("parentTaskId") + expect(saved).not.toHaveProperty("rootTaskId") + expect(saved.status).toBe("interrupted") + } + }) + it("resumes a Task.create history task only once when run is also called", async () => { mockReadTaskMessages.mockResolvedValue([]) mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) @@ -1271,7 +1334,7 @@ describe("Task persistence", () => { const initiateTaskLoopSpy = vi .spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") .mockResolvedValue(undefined) - vi.spyOn(task, "ask").mockResolvedValue({ response: "noButtonClicked" }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) // The persisted history ends with an assistant turn whose tool calls // (attempt_completion) were never answered because the task was @@ -1334,7 +1397,7 @@ describe("Task persistence", () => { const initiateTaskLoopSpy = vi .spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") .mockResolvedValue(undefined) - vi.spyOn(task, "ask").mockResolvedValue({ response: "noButtonClicked" }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "messageResponse", text: "Continue" }) // The persisted history ends with a user turn that only answered the // first of two parallel tool calls. diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 52a45df254..9c86034016 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -609,7 +609,7 @@ export class ClineProvider } private async cleanupFailedHistoryTask(task: Task, error: unknown): Promise { - if (!(error instanceof PendingActionSettlementError)) { + if (!(error instanceof PendingActionSettlementError || error instanceof LifecycleTransitionError)) { return } @@ -777,7 +777,7 @@ export class ClineProvider } } - const resumed = await this.taskHistoryStore.resumeInterruptedTask(taskId) + const resumed = await this.taskHistoryStore.resumeInterruptedTask(taskId, parentTaskId) this.recentTasksCache = undefined if (this.isViewLaunched) { await this.postMessageToWebview({ type: "taskHistoryItemUpdated", taskHistoryItem: resumed }) From ada30b465c9c0c54b907a2d5bfa860641e2f6df0 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 12 Oct 2026 16:12:02 +0000 Subject: [PATCH 04/14] fix: guard interrupted-child resume and preserve history backups --- docs/architecture/task-lifecycle-model.md | 33 +- scripts/check-task-lifecycle.ts | 32 +- .../ClineProvider.delegation.spec.ts | 367 +++++++++++++----- ...Provider.history-resume-delegation.spec.ts | 22 +- .../__tests__/repro-init-failure.spec.ts | 50 +++ src/core/task-persistence/TaskHistoryStore.ts | 145 ++++++- .../TaskHistoryStore.realConcurrency.spec.ts | 196 +++++++++- .../__tests__/taskLifecycle.spec.ts | 33 ++ src/core/task-persistence/index.ts | 1 + src/core/task-persistence/taskLifecycle.ts | 13 + .../task/__tests__/Task.persistence.spec.ts | 48 +++ src/core/webview/ClineProvider.ts | 54 ++- src/utils/__tests__/safeWriteJson.test.ts | 18 +- src/utils/safeWriteJson.ts | 27 +- 14 files changed, 870 insertions(+), 169 deletions(-) create mode 100644 src/core/checkpoints/__tests__/repro-init-failure.spec.ts diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 43fa6ef1bb..b093638246 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -42,25 +42,29 @@ TLA+/PlusCal or Quint with TLC becomes a better fit when the lifecycle needs tem ## Production mapping -| Model concept | Production concept | -| ------------------------- | -------------------------------------------------------------------------------------------------------------------- | -| Task record and status | `HistoryItem` persisted by `TaskHistoryStore` | -| `delegate(parent, child)` | `ClineProvider.delegateParentAndOpenChild` | -| `interrupt(child)` | cancellation or eviction through `markDelegatedChildInterrupted` | -| `resume(child)` | accepted `resume_task` prompt through `TaskHistoryStore.resumeInterruptedTask` | -| `complete(child)` | `ClineProvider.reopenParentFromDelegation` | -| `abandon(child)` | `ClineProvider.abandonSubtask` | -| Pending-action settlement | `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear in the rejected-delegation settlement path (#1714) | -| Atomic event step | `atomicReadAndUpdate`, `atomicUpdatePair`, and per-parent delegation transition lock | -| Event interleaving | Competing completion, cancellation, abandonment, and new delegation calls | - -The model has three fixed task slots, enough to cover competing siblings and a nested parent-child-grandchild chain. It explores every reachable interleaving through depth 13, deduplicating canonical states. Representative checks also exercise rejected operations that do not create a new state: a second concurrent delegation while the first child is active, stale completion after re-delegation, late completion after abandonment, completion after interruption, and nested completion. Named semantic landmarks require the graph to retain interrupted-child re-delegation, nested delegation, and an explicitly resumed interrupted child delegating to a grandchild even when the raw state total changes. The resume action calls the production reducer and carries a model-only provenance bit so the explorer can distinguish that path from a child that was never interrupted; generic persisted transitions still reject `interrupted → active`. +| Model concept | Production concept | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Task record and status | `HistoryItem` persisted by `TaskHistoryStore` | +| `delegate(parent, child)` | `ClineProvider.delegateParentAndOpenChild` | +| `interrupt(child)` | cancellation or eviction through `markDelegatedChildInterrupted` | +| `resume(child)` | accepted `resume_task` prompt through `TaskHistoryStore.resumeInterruptedTask` | +| `complete(child)` | `ClineProvider.reopenParentFromDelegation` | +| `abandon(child)` | `ClineProvider.abandonSubtask` | +| Pending-action settlement | `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear in the rejected-delegation settlement path (#1714) | +| Atomic event step | `atomicReadAndUpdate`, `atomicUpdatePair`, `atomicUpdateParentDelegation`, the parent-file-locked resume, and the per-parent delegation transition lock | +| Event interleaving | Competing completion, cancellation, abandonment, resume, and new delegation calls | + +The model has three fixed task slots, enough to cover competing siblings and a nested parent-child-grandchild chain. It explores every reachable interleaving through depth 13, deduplicating canonical states. Representative checks also exercise rejected operations that do not create a new state: a second concurrent delegation while the first child is active, stale completion after re-delegation, late completion after abandonment, completion after interruption, and nested completion. Named semantic landmarks require the graph to retain interrupted-child re-delegation, nested delegation, and an explicitly resumed interrupted child delegating to a grandchild even when the raw state total changes. The resume action calls the production `resumeInterruptedChild` composition, which validates the parent's delegation pointer before applying the child reducer, and carries a model-only provenance bit so the explorer can distinguish that path from a child that was never interrupted. Generic persisted transitions still reject `interrupted → active`. A `resume-stale` rejected action covers the competing-host class: after the parent re-delegates elsewhere, the parent-locked ownership check rejects the resume and every record stays unchanged. Each task slot can also hold one of two pending `create_subtask` actions. A `stage` action mirrors `setPendingTaskAction` overwrite semantics, delegation clears the action its request carried, completion clears the child's action only when its event carries the matching action ID, and a `settle-rejected` action models the settlement that follows an authoritative delegation rejection (#1714). Production settles through the typed `LifecycleTransitionError` from the shared guards: the provider calls the disk-authoritative `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear under the per-file lock, then propagates the original rejection. Six named witnesses must remain reachable: settlement from an interrupted record after rejection, settlement through a successful active delegation, unrelated-action preservation during completion, stale-action protection where a settlement targeting one action ID leaves a replacement action intact, matching-ID completion clearing, and replacement-ID completion preservation. A mismatched pending-action request keeps its production behavior: the atomic update throws before any transition, and no settlement runs. Production completion also accepts a recovery-compatible `active` parent that still awaits the returning child, then clears the stale pointers. Normal model transitions never create that intermediate state, so it is covered by a focused reducer test rather than admitted as a generally valid reachable state. -Explicit resume compares the caller's expected parent with the authoritative child backlink under the child-file lock. Abandonment writes child detachment before parent release, so a stale approval is rejected even in that partial-write window. Representative reducer scenarios cover stale linked and standalone approvals; a two-store provider test covers abandonment after the parent precheck. Rehydrated interrupted tasks omit lineage from message metadata saves, preventing their stale construction snapshots from restoring severed links after rejection. This closes abandonment-before-resume, not every cross-host ordering: parent-only redelegation, reverse-order stale abandonment, and other live-task writers remain within the ownership/generation gaps below. +Explicit resume validates both records under file locks. A linked child resume holds the parent file's advisory lock across the parent ownership check and the child activation, so a competing host cannot re-delegate the parent in between. The re-delegation write (`atomicUpdateParentDelegation`) re-reads the awaited child's on-disk record inside the parent-file lock and rejects the delegation unless that child is still interrupted. The two guards share one lock order, parent file before child file, so they close the competing resume and re-delegation interleavings in both directions. The child-file merge still compares the caller's expected parent with the authoritative child backlink. The resume rejects an abandonment that committed before it, even in its partial-write window. + +Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted non-interrupted resume still runs without a store transition. + +Rehydrated interrupted tasks omit lineage from message metadata saves, so their stale construction snapshots cannot restore severed links after rejection. Reverse-order stale abandonment and other live-task writers remain within the ownership/generation gaps below. ## Shared-store concurrency model @@ -180,6 +184,7 @@ The task delegation checker currently enforces: 7. Active-child re-delegation, stale completion after ownership moves to another child, duplicate/late completion, and abandonment of a live child are rejected by the shared production guards. 8. A rejected delegation settles only the exact matching pending `create_subtask` action. Settlement preserves status, lineage, and accounting, never clears a replacement or different-kind action, and never mutates a completed record. The settlement compare-and-clear reads the persisted record under the per-file lock, so it never clears from a stale host cache. 9. A completion clears the completing child's pending action only when the completion event carries the exact matching action ID. A completion with no action ID or a different ID preserves the pending action. +10. A linked interrupted child resumes only while the on-disk parent still awaits it, checked under the parent-file lock, and a re-delegation commits only while the awaited child's on-disk record is still interrupted, checked inside the parent-file lock. The rejected `resume-stale` action after a competing re-delegation leaves every record unchanged. The completion persistence checker additionally enforces: diff --git a/scripts/check-task-lifecycle.ts b/scripts/check-task-lifecycle.ts index ecbdc9da0e..dcd1e5fc22 100644 --- a/scripts/check-task-lifecycle.ts +++ b/scripts/check-task-lifecycle.ts @@ -7,6 +7,8 @@ import { completeDelegatedChild, delegateTaskToChild, interruptDelegatedChild, + LifecycleTransitionError, + resumeInterruptedChild, resumeInterruptedTask, settleRejectedCreateSubtaskAction, } from "../src/core/task-persistence/taskLifecycle" @@ -40,7 +42,16 @@ interface WitnessContext { const MAX_DEPTH = 13 const MAX_STATES = 10_000 const actionIds = ["action-1", "action-2"] as const -const expectedActions = ["delegate", "interrupt", "resume", "complete", "abandon", "stage", "settle-rejected"] as const +const expectedActions = [ + "delegate", + "interrupt", + "resume", + "resume-stale", + "complete", + "abandon", + "stage", + "settle-rejected", +] as const const semanticLandmarks = { "interrupted-child-redelegation": (state: ModelState) => state.parent?.status === "delegated" && @@ -193,9 +204,22 @@ function transitions(state: ModelState): Transition[] { (!parent.parentTaskId || (lineageParent?.status === "delegated" && lineageParent.awaitingChildId === parent.id)) if (resumeValid) { + const resumed = + lineageParent !== undefined + ? resumeInterruptedChild(lineageParent, parent) + : resumeInterruptedTask(parent, undefined) result.push({ name: `resume(${parentId})`, - next: replace(state, { ...resumeInterruptedTask(parent, parent.parentTaskId), modelWasResumed: true }), + next: replace(state, { ...resumed, modelWasResumed: true }), + }) + } else if (parent.status === "interrupted" && parent.parentTaskId && lineageParent !== undefined) { + // Representative rejected event: the parent re-delegated elsewhere, + // so the store's parent-locked ownership check must reject the + // resume and leave every record unchanged. + assert.throws(() => resumeInterruptedChild(lineageParent, parent), LifecycleTransitionError) + result.push({ + name: `resume-stale(${parentId})`, + next: state, }) } } @@ -444,6 +468,10 @@ function runRepresentativeScenarios(): void { assert.equal(resumedNested.awaitingChildId, "child-b") const redelegated = delegateTaskToChild(delegated, "child-b", interruptedA.status) assert.throws(() => completeDelegatedChild(redelegated, interruptedA, "stale"), /not delegated to child/) + // The parent-locked resume rejects a child whose parent re-delegated, and + // accepts the same child while the parent still awaits it. + assert.throws(() => resumeInterruptedChild(redelegated, interruptedA), /no longer awaits/) + assert.equal(resumeInterruptedChild(delegated, interruptedA).status, "active") const abandoned = abandonDelegatedChild(delegated, interruptedA) assert.throws(() => completeDelegatedChild(abandoned.parent, abandoned.child, "late"), /not delegated to child/) diff --git a/src/__tests__/ClineProvider.delegation.spec.ts b/src/__tests__/ClineProvider.delegation.spec.ts index e5fd3b3ee0..1e3eacf7e6 100644 --- a/src/__tests__/ClineProvider.delegation.spec.ts +++ b/src/__tests__/ClineProvider.delegation.spec.ts @@ -9,7 +9,7 @@ import type { HistoryItem } from "@roo-code/types" import { providerIdentifiers, RooCodeEventName } from "@roo-code/types" import { ClineProvider } from "../core/webview/ClineProvider" import { TaskScheduler } from "../core/task/TaskScheduler" -import { LifecycleTransitionError, TaskHistoryStore } from "../core/task-persistence" +import { LifecycleTransitionError, TaskHistoryStore, type HistoryItemStatus } from "../core/task-persistence" import { abandonDelegatedChild } from "../core/task-persistence/taskLifecycle" import { makeProviderStub } from "./helpers/provider-stub" @@ -22,16 +22,21 @@ const parentHistoryItem: HistoryItem = { childIds: [], } as unknown as HistoryItem -/** Minimal taskHistoryStore stub whose atomicReadAndUpdate calls the updater with the parent item. */ +/** Minimal taskHistoryStore stub whose atomicUpdateParentDelegation builds from the parent item. */ function makeStoreStub( - overrides: Partial<{ atomicReadAndUpdate: ReturnType; get: ReturnType }> = {}, + overrides: Partial<{ + atomicUpdateParentDelegation: ReturnType + get: ReturnType + }> = {}, ) { return { invalidate: vi.fn().mockResolvedValue(undefined), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (h: HistoryItem) => HistoryItem) => { - updater(parentHistoryItem) - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: (diskParent: HistoryItem, awaitedChildDiskStatus: HistoryItemStatus | undefined) => HistoryItem, + ) => build(parentHistoryItem, undefined), + ), get: vi.fn().mockReturnValue(undefined), ...overrides, } @@ -137,6 +142,104 @@ describe("ClineProvider.resumeInterruptedTask()", () => { ).rejects.toThrow("parent parent-1 no longer awaits it") expect(taskHistoryStore.resumeInterruptedTask).not.toHaveBeenCalled() }) + + it("delegates a resumed interrupted child to a grandchild through the real provider flow", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-then-grandchild-")) + const store = new TaskHistoryStore(storagePath) + const parent: HistoryItem = { + id: "parent", + number: 1, + ts: 1, + task: "Parent", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child: HistoryItem = { + ...parent, + id: "child", + task: "Child", + status: "interrupted", + parentTaskId: "parent", + rootTaskId: "parent", + awaitingChildId: undefined, + delegatedToId: undefined, + childIds: [], + } + + try { + await store.initialize() + await store.upsert(parent) + await store.upsert(child) + + // Accepted resume: the child record activates while the parent stays delegated. + const resumeProvider = makeProviderStub({ taskHistoryStore: store, isViewLaunched: false }) + await ClineProvider.prototype.resumeInterruptedTask.call(resumeProvider, "child", "parent") + expect(store.get("child")?.status).toBe("active") + expect(store.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child" }) + + // The resumed child runs the real delegation flow and spawns a grandchild. + const resumedChildTask = { + taskId: "child", + apiConfiguration: { apiProvider: providerIdentifiers.anthropic, anthropicApiKey: "task-local-key" }, + getTaskMode: vi.fn().mockResolvedValue("code"), + getTaskApiConfigName: vi.fn().mockResolvedValue("task-local-profile"), + emit: vi.fn(), + flushPendingToolResultsToHistory: vi.fn().mockResolvedValue(true), + retrySaveApiConversationHistory: vi.fn(), + } + const grandchildRun = vi.fn().mockResolvedValue(undefined) + const grandchild = { taskId: "grandchild", run: grandchildRun } + const createTask = vi.fn().mockResolvedValue(grandchild) + const provider = makeProviderStub({ + taskHistoryStore: store, + isViewLaunched: false, + tasks: [], + }) + const providerFields = provider as unknown as Record + providerFields.getCurrentTask = vi.fn(() => resumedChildTask) + providerFields.removeClineFromStack = vi.fn().mockResolvedValue(undefined) + providerFields.createTask = createTask + providerFields.taskScheduler = new TaskScheduler() + providerFields.emit = vi.fn() + providerFields.log = vi.fn() + providerFields.recentTasksCache = undefined + + const openedChild = await ClineProvider.prototype.delegateParentAndOpenChild.call(provider, { + parentTaskId: "child", + message: "Grandchild work", + initialTodos: [], + mode: "code", + }) + await Promise.resolve() // drain scheduler microtask so grandchild.run() is invoked + + expect(openedChild.taskId).toBe("grandchild") + expect(createTask).toHaveBeenCalledWith( + "Grandchild work", + undefined, + resumedChildTask, + expect.objectContaining({ initialStatus: "active", startTask: false }), + ) + await store.invalidate("child") + await store.invalidate("parent") + expect(store.get("child")).toMatchObject({ + status: "delegated", + awaitingChildId: "grandchild", + delegatedToId: "grandchild", + }) + expect(store.get("child")?.childIds).toContain("grandchild") + // The top parent record is untouched by the nested delegation. + expect(store.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child" }) + expect(grandchildRun).toHaveBeenCalledTimes(1) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) }) describe("ClineProvider.delegateParentAndOpenChild()", () => { @@ -180,7 +283,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { expect(removeClineFromStack).not.toHaveBeenCalled() expect(handleModeSwitch).not.toHaveBeenCalled() expect(createTask).not.toHaveBeenCalled() - expect(taskHistoryStore.atomicReadAndUpdate).not.toHaveBeenCalled() + expect(taskHistoryStore.atomicUpdateParentDelegation).not.toHaveBeenCalled() }) it("clears a matching pending action when delegation commits", async () => { @@ -196,10 +299,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { const taskHistoryStore = { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => current), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - current = updater(current) - return [current] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + current = build(current, undefined) + return current + }, + ), } const parentTask = makeParentTask() const child = { taskId: "child-1", run: vi.fn().mockResolvedValue(undefined) } @@ -246,10 +357,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { const replacementAction = { ...pendingAction, actionId: "replacement-action" } const taskHistoryStore = makeStoreStub({ get: vi.fn().mockReturnValue({ ...parentHistoryItem, status: "active", pendingAction }), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - updater({ ...parentHistoryItem, status: "active", pendingAction: replacementAction }) - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + build({ ...parentHistoryItem, status: "active", pendingAction: replacementAction }, undefined) + return { ...parentHistoryItem, status: "active", pendingAction: replacementAction } + }, + ), }) const deleteTaskWithId = vi.fn().mockResolvedValue(undefined) const createTaskWithHistoryItem = vi.fn().mockResolvedValue(undefined) @@ -283,7 +402,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { expect(createTaskWithHistoryItem).toHaveBeenCalledWith(parentHistoryItem) }) - it("persists parent delegation metadata via atomicReadAndUpdate and emits TaskDelegated", async () => { + it("persists parent delegation metadata via atomicUpdateParentDelegation and emits TaskDelegated", async () => { const providerEmit = vi.fn() const parentTask = makeParentTask() @@ -334,13 +453,13 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { startTask: false, }) - // Delegation metadata written via atomicReadAndUpdate with correct taskId - expect(taskHistoryStore.atomicReadAndUpdate).toHaveBeenCalledTimes(1) - const [calledTaskId, updater] = taskHistoryStore.atomicReadAndUpdate.mock.calls[0] + // Delegation metadata written via atomicUpdateParentDelegation with correct taskId + expect(taskHistoryStore.atomicUpdateParentDelegation).toHaveBeenCalledTimes(1) + const [calledTaskId, build] = taskHistoryStore.atomicUpdateParentDelegation.mock.calls[0] expect(calledTaskId).toBe("parent-1") - // The updater must produce the correct delegation fields - const result = updater(parentHistoryItem) + // The builder must produce the correct delegation fields + const result = build(parentHistoryItem, undefined) expect(result).toMatchObject({ id: "parent-1", status: "delegated", @@ -487,7 +606,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { const postMessageToWebview = vi.fn().mockResolvedValue(undefined) const parentTask = makeParentTask() const taskHistoryStore = makeStoreStub({ - get: vi.fn().mockReturnValueOnce(parentHistoryItem).mockReturnValue(updatedParent), + atomicUpdateParentDelegation: vi.fn().mockResolvedValue(updatedParent), }) const provider = { @@ -517,11 +636,11 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { }) }) - it("skips postMessageToWebview when isViewLaunched is true but store returns undefined", async () => { + it("skips postMessageToWebview when isViewLaunched is true but the store returns undefined", async () => { const postMessageToWebview = vi.fn().mockResolvedValue(undefined) const parentTask = makeParentTask() const taskHistoryStore = makeStoreStub({ - get: vi.fn().mockReturnValue(undefined), + atomicUpdateParentDelegation: vi.fn().mockResolvedValue(undefined), }) const provider = { @@ -548,7 +667,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { expect(postMessageToWebview).not.toHaveBeenCalled() }) - it("calls child.run() only after atomicReadAndUpdate completes (no race condition)", async () => { + it("calls child.run() only after atomicUpdateParentDelegation completes (no race condition)", async () => { const callOrder: string[] = [] const parentTask = makeParentTask() @@ -560,10 +679,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { }) const handleModeSwitch = vi.fn().mockResolvedValue(undefined) const taskHistoryStore = makeStoreStub({ - atomicReadAndUpdate: vi.fn(async (_taskId: string, _updater: (h: HistoryItem) => HistoryItem) => { - callOrder.push("atomicReadAndUpdate") - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + _build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + callOrder.push("atomicUpdateParentDelegation") + return parentHistoryItem + }, + ), }) const provider = { @@ -587,8 +714,8 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { }) await Promise.resolve() // drain scheduler microtask so child.run() is invoked - // createTask → atomicReadAndUpdate → child.run: scheduler admits child only after metadata is persisted - expect(callOrder).toEqual(["createTask", "atomicReadAndUpdate", "child.run"]) + // createTask → atomicUpdateParentDelegation → child.run: scheduler admits child only after metadata is persisted + expect(callOrder).toEqual(["createTask", "atomicUpdateParentDelegation", "child.run"]) }) it("implicitly severs interrupted awaited child and re-delegates when parent is already delegated", async () => { @@ -607,10 +734,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { get: vi.fn((id: string) => id === "parent-1" ? alreadyDelegatedParent : id === oldChildId ? oldChild : undefined, ), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (h: HistoryItem) => HistoryItem) => { - updater(alreadyDelegatedParent) - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + build(alreadyDelegatedParent, "interrupted") + return alreadyDelegatedParent + }, + ), }) const provider = { @@ -633,9 +768,9 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { mode: "code", }) - // The updater must sever the old link and apply the new delegation - const [, updater] = taskHistoryStore.atomicReadAndUpdate.mock.calls[0] - const result = updater(alreadyDelegatedParent) + // The builder must sever the old link and apply the new delegation + const [, build] = taskHistoryStore.atomicUpdateParentDelegation.mock.calls[0] + const result = build(alreadyDelegatedParent, "interrupted") expect(result).toMatchObject({ status: "delegated", awaitingChildId: "child-2", @@ -667,11 +802,19 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { get: vi.fn((id: string) => id === "parent-1" ? alreadyDelegatedParent : id === oldChildId ? activeChild : undefined, ), - // Real atomicReadAndUpdate behaviour: call the updater and propagate any throw - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (h: HistoryItem) => HistoryItem) => { - updater(alreadyDelegatedParent) - return [] - }), + // Real atomicUpdateParentDelegation behaviour: call the builder and propagate any throw + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + build(alreadyDelegatedParent, "active") + return alreadyDelegatedParent + }, + ), }) const provider = { @@ -746,12 +889,20 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { const store = { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn((id: string) => (id === parent.taskId ? durableParent : undefined)), - atomicReadAndUpdate: vi.fn(async (_id: string, updater: (item: HistoryItem) => HistoryItem) => { - markCommitStarted() - await commitMayFinish - durableParent = updater(durableParent) as typeof durableParent - return [durableParent] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _id: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + markCommitStarted() + await commitMayFinish + durableParent = build(durableParent, undefined) as typeof durableParent + return durableParent + }, + ), } const provider = { taskScheduler: new TaskScheduler(), @@ -793,7 +944,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { expect(durableParent.awaitingChildId).toBe("child-1") }) - it("rolls back the paused child and restores the parent when atomicReadAndUpdate fails", async () => { + it("rolls back the paused child and restores the parent when atomicUpdateParentDelegation fails", async () => { const persistError = new Error("parent metadata persist failed") const parentTask = makeParentTask() const childRun = vi.fn().mockResolvedValue(undefined) @@ -803,7 +954,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { const getTaskWithId = vi.fn().mockResolvedValue({ historyItem: parentHistoryItem }) const taskHistoryStore = makeStoreStub({ - atomicReadAndUpdate: vi.fn().mockRejectedValue(persistError), + atomicUpdateParentDelegation: vi.fn().mockRejectedValue(persistError), }) const child = { taskId: "child-1", start: vi.fn(), run: childRun } @@ -871,10 +1022,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { const taskHistoryStore = { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => current), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - current = updater(current) - return [current] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + current = build(current, undefined) + return current + }, + ), clearPendingActionIfMatching: vi.fn(async (_taskId: string, actionId: string) => { if (current.pendingAction?.kind === "create_subtask" && current.pendingAction.actionId === actionId) { current = { ...current, pendingAction: undefined } @@ -933,15 +1092,20 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { return child }) let transitionError: LifecycleTransitionError | undefined - const atomicReadAndUpdate = vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - try { - updater(interruptedParent) - } catch (error) { - if (error instanceof LifecycleTransitionError) transitionError = error - throw error - } - return [] - }) + const atomicUpdateParentDelegation = vi.fn( + async ( + _taskId: string, + build: (diskParent: HistoryItem, awaitedChildDiskStatus: HistoryItemStatus | undefined) => HistoryItem, + ) => { + try { + build(interruptedParent, undefined) + } catch (error) { + if (error instanceof LifecycleTransitionError) transitionError = error + throw error + } + return interruptedParent + }, + ) const clearPendingActionIfMatching = vi.fn() const deleteTaskWithId = vi.fn().mockResolvedValue(undefined) const createTaskWithHistoryItem = vi.fn().mockResolvedValue(undefined) @@ -960,7 +1124,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { taskHistoryStore: { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => interruptedParent), - atomicReadAndUpdate, + atomicUpdateParentDelegation, clearPendingActionIfMatching, }, } as unknown as ClineProvider @@ -1006,7 +1170,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { getCurrentTask.mockReturnValue(child) return child }) - const atomicReadAndUpdate = vi.fn().mockRejectedValue(persistenceError) + const atomicUpdateParentDelegation = vi.fn().mockRejectedValue(persistenceError) const clearPendingActionIfMatching = vi.fn().mockResolvedValue(interruptedParent) const createTaskWithHistoryItem = vi.fn().mockResolvedValue(undefined) const provider = { @@ -1024,7 +1188,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { taskHistoryStore: { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => interruptedParent), - atomicReadAndUpdate, + atomicUpdateParentDelegation, clearPendingActionIfMatching, }, } as unknown as ClineProvider @@ -1039,7 +1203,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { }), ).rejects.toThrow(persistenceError) - expect(atomicReadAndUpdate).toHaveBeenCalledTimes(1) + expect(atomicUpdateParentDelegation).toHaveBeenCalledTimes(1) expect(clearPendingActionIfMatching).not.toHaveBeenCalled() expect(interruptedParent.pendingAction).toBe(pendingAction) expect(provider.deleteTaskWithId).toHaveBeenCalledWith("child-1", false) @@ -1068,10 +1232,15 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { getCurrentTask.mockReturnValue(child) return child }) - const atomicReadAndUpdate = vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - updater(interruptedParent) - return [] - }) + const atomicUpdateParentDelegation = vi.fn( + async ( + _taskId: string, + build: (diskParent: HistoryItem, awaitedChildDiskStatus: HistoryItemStatus | undefined) => HistoryItem, + ) => { + build(interruptedParent, undefined) + return interruptedParent + }, + ) const clearPendingActionIfMatching = vi.fn().mockRejectedValue(settlementError) const createTaskWithHistoryItem = vi.fn().mockResolvedValue(undefined) const provider = { @@ -1089,7 +1258,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { taskHistoryStore: { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => interruptedParent), - atomicReadAndUpdate, + atomicUpdateParentDelegation, clearPendingActionIfMatching, }, } as unknown as ClineProvider @@ -1104,7 +1273,7 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { }), ).rejects.toThrow("Invalid task status transition: interrupted → delegated") - expect(atomicReadAndUpdate).toHaveBeenCalledTimes(1) + expect(atomicUpdateParentDelegation).toHaveBeenCalledTimes(1) expect(clearPendingActionIfMatching).toHaveBeenCalledTimes(1) expect(provider.deleteTaskWithId).toHaveBeenCalledWith("child-1", false) expect(createTaskWithHistoryItem).not.toHaveBeenCalled() @@ -1153,10 +1322,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { taskHistoryStore: { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => interruptedParent), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - updater(interruptedParent) - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + build(interruptedParent, undefined) + return interruptedParent + }, + ), clearPendingActionIfMatching, }, } as unknown as ClineProvider @@ -1220,10 +1397,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { taskHistoryStore: { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => cachedParent), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - updater(cachedParent) - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + build(cachedParent, undefined) + return cachedParent + }, + ), clearPendingActionIfMatching, }, } as unknown as ClineProvider @@ -1303,10 +1488,18 @@ describe("ClineProvider.delegateParentAndOpenChild()", () => { taskHistoryStore: { invalidate: vi.fn().mockResolvedValue(undefined), get: vi.fn(() => interruptedParent), - atomicReadAndUpdate: vi.fn(async (_taskId: string, updater: (item: HistoryItem) => HistoryItem) => { - updater(interruptedParent) - return [] - }), + atomicUpdateParentDelegation: vi.fn( + async ( + _taskId: string, + build: ( + diskParent: HistoryItem, + awaitedChildDiskStatus: HistoryItemStatus | undefined, + ) => HistoryItem, + ) => { + build(interruptedParent, undefined) + return interruptedParent + }, + ), clearPendingActionIfMatching, deleteMany: vi.fn().mockResolvedValue(undefined), }, diff --git a/src/__tests__/ClineProvider.history-resume-delegation.spec.ts b/src/__tests__/ClineProvider.history-resume-delegation.spec.ts index ee87830da0..c6c1c5a3d0 100644 --- a/src/__tests__/ClineProvider.history-resume-delegation.spec.ts +++ b/src/__tests__/ClineProvider.history-resume-delegation.spec.ts @@ -5,7 +5,7 @@ import * as vscode from "vscode" import { providerIdentifiers, RooCodeEventName } from "@roo-code/types" import type { ClineMessage, HistoryItem } from "@roo-code/types" -import type { ApiMessage } from "../core/task-persistence" +import type { ApiMessage, HistoryItemStatus } from "../core/task-persistence" import type { Task } from "../core/task/Task" /* vscode mock for Task/Provider imports */ @@ -120,6 +120,22 @@ function makeStatefulTaskHistoryStore(...items: HistoryItem[]) { itemMap.set(id, updated) return [updated] }), + atomicUpdateParentDelegation: vi.fn( + async ( + id: string, + build: (diskParent: HistoryItem, awaitedChildDiskStatus: HistoryItemStatus | undefined) => HistoryItem, + ) => { + const item = itemMap.get(id) + if (!item) throw new Error(`Missing history item: ${id}`) + const awaitedChildDiskStatus = + item.status === "delegated" && item.awaitingChildId + ? (itemMap.get(item.awaitingChildId)?.status ?? undefined) + : undefined + const updated = build(item, awaitedChildDiskStatus) + itemMap.set(id, updated) + return updated + }, + ), } } @@ -1265,14 +1281,14 @@ describe("History resume delegation - parent metadata transitions", () => { await Promise.resolve() expect(removeParentB).not.toHaveBeenCalled() expect(createChildC2).not.toHaveBeenCalled() - expect(taskHistoryStore.atomicReadAndUpdate).not.toHaveBeenCalled() + expect(taskHistoryStore.atomicUpdateParentDelegation).not.toHaveBeenCalled() const continuationRun = runScheduledContinuation() await vi.waitFor(() => expect(parentInstanceA.resumeAfterDelegation).toHaveBeenCalledTimes(1)) await expect(providerBTransition).resolves.toBe(childC2) expect(removeParentB).toHaveBeenCalledTimes(1) expect(createChildC2).toHaveBeenCalledTimes(1) - expect(taskHistoryStore.atomicReadAndUpdate).toHaveBeenCalledTimes(1) + expect(taskHistoryStore.atomicUpdateParentDelegation).toHaveBeenCalledTimes(1) expect(taskHistoryStore.get(parentItem.id)).toMatchObject({ status: "delegated", awaitingChildId: "child-c2", diff --git a/src/core/checkpoints/__tests__/repro-init-failure.spec.ts b/src/core/checkpoints/__tests__/repro-init-failure.spec.ts new file mode 100644 index 0000000000..ffc2535a3c --- /dev/null +++ b/src/core/checkpoints/__tests__/repro-init-failure.spec.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi } from "vitest" +import { getCheckpointService } from "../index" + +vi.mock("vscode", () => ({ + window: { showWarningMessage: vi.fn(), createTextEditorDecorationType: vi.fn(() => ({})) }, + env: {}, + Uri: {}, +})) +vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: {} } })) +vi.mock("../../../utils/path", () => ({ getWorkspacePath: vi.fn(() => "/ws") })) +vi.mock("../../../utils/git", () => ({ checkGitInstalled: vi.fn().mockResolvedValue(true) })) +vi.mock("../../../i18n", () => ({ t: (k: string) => k })) +vi.mock("../../../services/checkpoints") + +describe("repro: init failure disables checkpoints for the whole task", () => { + it("does not retry after one initShadowGit failure", async () => { + const { RepoPerTaskCheckpointService } = await import("../../../services/checkpoints") + const initShadowGit = vi.fn().mockRejectedValue(new Error("simulated git failure")) + vi.mocked(RepoPerTaskCheckpointService.create).mockReturnValue({ on: vi.fn(), initShadowGit } as never) + + const task = { + taskId: "t", + enableCheckpoints: true, + checkpointTimeout: 60, + checkpointService: undefined, + checkpointServiceInitializing: false, + providerRef: { + deref: () => ({ + context: { globalStorageUri: { fsPath: "/s" } }, + log: vi.fn(), + postMessageToWebview: vi.fn(), + }), + }, + } + + await getCheckpointService(task as never) + console.log("after 1st:", { + enable: task.enableCheckpoints, + svcSet: !!task.checkpointService, + initializing: task.checkpointServiceInitializing, + }) + const second = await getCheckpointService(task as never) + const third = await getCheckpointService(task as never) + + expect(initShadowGit).toHaveBeenCalledTimes(1) + expect(task.enableCheckpoints).toBe(false) + expect(second).toBeUndefined() + expect(third).toBeUndefined() + }) +}) diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index 2e56f27b80..5c4bb34556 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -12,6 +12,8 @@ import { safeWriteJson } from "../../utils/safeWriteJson" import { getStorageBasePath } from "../../utils/storage" import { assertValidTransition, + LifecycleTransitionError, + resumeInterruptedChild, resumeInterruptedTask as resumeInterruptedTaskRecord, settleRejectedCreateSubtaskAction, type HistoryItemStatus, @@ -901,6 +903,28 @@ export class TaskHistoryStore { } } + /** + * Read the canonical on-disk record for one task, failing closed with a + * typed lifecycle error when the record is missing, malformed, or + * persisted under a different task ID. Locked authoritative transitions + * call this before they write, so a cached record never replaces unknown + * disk state. + */ + private async readValidatedTaskFile(taskId: string): Promise { + const filePath = await this.getTaskFilePath(taskId) + let parsed: unknown + try { + parsed = JSON.parse(await fs.readFile(filePath, "utf8")) + } catch { + throw new LifecycleTransitionError(`[TaskHistoryStore] task ${taskId} has no valid disk record`) + } + const result = historyItemSchema.safeParse(parsed) + if (!result.success || result.data.id !== taskId) { + throw new LifecycleTransitionError(`[TaskHistoryStore] task ${taskId} has no valid disk record`) + } + return result.data as HistoryItem + } + // ────────────────────────────── Private: fs.watch ────────────────────────────── /** @@ -1081,35 +1105,56 @@ export class TaskHistoryStore { /** * Disk-authoritative transition used only after the user accepts a resume * prompt. Generic writes intentionally reject interrupted → active so stale - * task snapshots cannot revive cancelled work. + * task snapshots cannot revive cancelled work. Failures for a missing + * cache entry or an invalid disk record throw `LifecycleTransitionError` + * so callers can settle the failed task without deleting its history. */ public async resumeInterruptedTask(taskId: string, expectedParentTaskId?: string): Promise { return this.withLock(async () => { const cached = this.cache.get(taskId) if (!cached) { - throw new Error(`[TaskHistoryStore] resumeInterruptedTask: task ${taskId} not found in cache`) + throw new LifecycleTransitionError( + `[TaskHistoryStore] resumeInterruptedTask: task ${taskId} not found in cache`, + ) } const filePath = await this.getTaskFilePath(taskId) let authoritative: HistoryItem = cached - await safeWriteJson(filePath, cached, { - merge: (existing) => { - const parsed = historyItemSchema.safeParse(existing) - if (!parsed.success || parsed.data.id !== taskId) { - this.cache.delete(taskId) - this.taskFileMtimes.delete(taskId) - throw new Error( - `[TaskHistoryStore] resumeInterruptedTask: task ${taskId} has no valid disk record`, - ) - } + const activate = async (parent?: HistoryItem): Promise => { + await safeWriteJson(filePath, cached, { + merge: (existing) => { + const parsed = historyItemSchema.safeParse(existing) + if (!parsed.success || parsed.data.id !== taskId) { + this.cache.delete(taskId) + this.taskFileMtimes.delete(taskId) + throw new LifecycleTransitionError( + `[TaskHistoryStore] resumeInterruptedTask: task ${taskId} has no valid disk record`, + ) + } - // Abandonment commits child detachment before releasing the parent. - // Compare the caller's linkage under this same child-file lock. - this.cache.set(taskId, existing as HistoryItem) - authoritative = resumeInterruptedTaskRecord(existing as HistoryItem, expectedParentTaskId) - return authoritative - }, - }) + // Abandonment commits child detachment before releasing + // the parent. Compare the caller's linkage under this + // same child-file lock. The caller still holds the + // parent-file lock when a parent record is provided. + this.cache.set(taskId, existing as HistoryItem) + authoritative = + parent !== undefined + ? resumeInterruptedChild(parent, existing as HistoryItem) + : resumeInterruptedTaskRecord(existing as HistoryItem, undefined) + return authoritative + }, + }) + } + + if (expectedParentTaskId === undefined) { + await activate() + } else { + const parentFilePath = await this.getTaskFilePath(expectedParentTaskId) + await withFileLock(parentFilePath, async () => { + const parent = await this.readValidatedTaskFile(expectedParentTaskId) + await activate(parent) + }) + } this.cache.set(taskId, authoritative) if (this.onWrite) { @@ -1119,6 +1164,68 @@ export class TaskHistoryStore { }) } + /** + * Disk-authoritative delegation write for the provider's delegation flow. + * The builder runs against the on-disk parent record inside the parent + * file's advisory lock, so a competing host cannot change the parent or + * its awaited child between validation and commit. When the disk parent is + * delegated, its awaited child's on-disk record must still be interrupted. + * A host with a stale cache cannot re-delegate over a child another host + * has already resumed. First delegation from an active parent commits + * through the same lock-held window. + */ + public async atomicUpdateParentDelegation( + parentTaskId: string, + build: (diskParent: HistoryItem, awaitedChildDiskStatus: HistoryItemStatus | undefined) => HistoryItem, + ): Promise { + return this.withLock(async () => { + const cached = this.cache.get(parentTaskId) + if (!cached) { + throw new LifecycleTransitionError( + `[TaskHistoryStore] atomicUpdateParentDelegation: task ${parentTaskId} not found in cache`, + ) + } + + const parentFilePath = await this.getTaskFilePath(parentTaskId) + return withFileLock(parentFilePath, async () => { + const diskParent = await this.readValidatedTaskFile(parentTaskId) + let awaitedChildDiskStatus: HistoryItemStatus | undefined + if (diskParent.status === "delegated") { + const awaitedChildId = diskParent.awaitingChildId + if (!awaitedChildId) { + throw new LifecycleTransitionError( + `[TaskHistoryStore] atomicUpdateParentDelegation: delegated task ${parentTaskId} has no awaited child`, + ) + } + const awaitedChild = await this.readValidatedTaskFile(awaitedChildId) + awaitedChildDiskStatus = awaitedChild.status + if (awaitedChildDiskStatus !== "interrupted") { + throw new LifecycleTransitionError( + `[TaskHistoryStore] atomicUpdateParentDelegation: awaited child ${awaitedChildId} of task ${parentTaskId} is ${awaitedChildDiskStatus ?? "active"}, not interrupted`, + ) + } + } + + const next = build(diskParent, awaitedChildDiskStatus) + if (next.id !== parentTaskId) { + throw new Error( + `[TaskHistoryStore] atomicUpdateParentDelegation: builder changed task id from ${parentTaskId} to ${next.id}`, + ) + } + + const delta = this.buildDelta(parentTaskId, cached, next) + const merged = mergeHistoryDelta(diskParent, next, delta) + await safeWriteJson(parentFilePath, merged, { assumeLockHeld: true }) + + this.cache.set(parentTaskId, merged) + if (this.onWrite) { + await this.onWrite(this.getAll()) + } + return merged + }) + }) + } + /** * Disk-authoritative compare-and-clear for a rejected `create_subtask` * pending action (#1714). The comparison runs inside the per-file diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index 92af62ec6a..ad53465151 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -5,12 +5,13 @@ import * as path from "path" import type { HistoryItem } from "@roo-code/types" import { TaskHistoryStore } from "../TaskHistoryStore" +import { delegateTaskToChild, LifecycleTransitionError } from "../taskLifecycle" // Wrap the real fs/promises so a test can pause inside the per-file lock // while every call still runs against the real filesystem. vi.mock("fs/promises", async () => { const actual = await vi.importActual("fs/promises") - return { ...actual, readFile: vi.fn(actual.readFile) } + return { ...actual, readFile: vi.fn(actual.readFile), rename: vi.fn(actual.rename) } }) type WriteTaskFile = (item: HistoryItem, delta?: Partial) => Promise @@ -96,6 +97,7 @@ describe("TaskHistoryStore real cross-host locking", () => { const store = new TaskHistoryStore(storagePath) try { await store.initialize() + await expect(store.resumeInterruptedTask("missing")).rejects.toThrow(LifecycleTransitionError) await expect(store.resumeInterruptedTask("missing")).rejects.toThrow("task missing not found in cache") expect(store.get("missing")).toBeUndefined() await expect( @@ -122,9 +124,9 @@ describe("TaskHistoryStore real cross-host locking", () => { : JSON.stringify(scenario === "schema" ? { id: "child" } : item("other")) if (scenario === "missing") await fs.unlink(filePath) else await fs.writeFile(filePath, contents) - await expect(store.resumeInterruptedTask("child")).rejects.toThrow( - "task child has no valid disk record", - ) + const rejection = store.resumeInterruptedTask("child") + await expect(rejection).rejects.toThrow(LifecycleTransitionError) + await expect(rejection).rejects.toThrow("task child has no valid disk record") expect(store.get("child")).toBeUndefined() expect(store["taskFileMtimes"].has("child")).toBe(false) if (scenario === "missing") await expect(fs.access(filePath)).rejects.toMatchObject({ code: "ENOENT" }) @@ -184,6 +186,192 @@ describe("TaskHistoryStore real cross-host locking", () => { } }) + it("rejects resume after a competing host re-delegates the parent", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-redelegated-parent-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await storeA.initialize() + await storeA.upsert(parent) + await storeA.upsert(child) + await storeB.initialize() + + await storeB.atomicUpdateParentDelegation("parent", (diskParent, awaitedChildDiskStatus) => + delegateTaskToChild(diskParent, "child-b", awaitedChildDiskStatus), + ) + + await expect(storeA.resumeInterruptedTask("child", "parent")).rejects.toThrow( + "parent parent no longer awaits it", + ) + await storeA.invalidate("child") + await storeA.invalidate("parent") + expect(storeA.get("child")?.status).toBe("interrupted") + expect(storeA.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child-b" }) + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("rejects re-delegation after a competing host resumed the awaited child", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "redelegation-resumed-child-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await storeA.initialize() + await storeA.upsert(parent) + await storeA.upsert(child) + await storeB.initialize() + + await storeA.resumeInterruptedTask("child", "parent") + expect(storeA.get("child")?.status).toBe("active") + + await expect( + storeB.atomicUpdateParentDelegation("parent", (diskParent, awaitedChildDiskStatus) => + delegateTaskToChild(diskParent, "child-b", awaitedChildDiskStatus), + ), + ).rejects.toThrow("not interrupted") + + await storeB.invalidate("parent") + await storeB.invalidate("child") + expect(storeB.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child" }) + expect(storeB.get("child")?.status).toBe("active") + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("reads the awaited child's disk record at lock time even when the cache is stale", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "redelegation-stale-cache-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await storeA.initialize() + await storeA.upsert(parent) + await storeA.upsert(child) + // storeB's cache captures the pre-resume state and never refreshes. + await storeB.initialize() + expect(storeB.get("child")?.status).toBe("interrupted") + + await storeA.resumeInterruptedTask("child", "parent") + + await expect( + storeB.atomicUpdateParentDelegation("parent", (diskParent, awaitedChildDiskStatus) => + delegateTaskToChild(diskParent, "child-b", awaitedChildDiskStatus), + ), + ).rejects.toThrow("not interrupted") + + await storeB.invalidate("parent") + expect(storeB.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child" }) + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("commits a first delegation through the same lock-held parent write", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "delegation-first-lock-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + + try { + await storeA.initialize() + await storeA.upsert(item("parent")) + await storeB.initialize() + + await storeB.atomicUpdateParentDelegation("parent", (diskParent) => + delegateTaskToChild(diskParent, "child-b", undefined), + ) + + await storeA.invalidate("parent") + expect(storeA.get("parent")).toMatchObject({ + status: "delegated", + awaitingChildId: "child-b", + delegatedToId: "child-b", + childIds: ["child-b"], + }) + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("keeps the prior task record recoverable when resume persistence fails both renames", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-partial-failure-")) + const store = new TaskHistoryStore(storagePath) + const taskDir = path.join(storagePath, "tasks", "child") + const renameMock = vi.mocked(fs.rename) + const originalRename = renameMock.getMockImplementation() + + try { + await store.initialize() + await store.upsert({ ...item("child"), status: "interrupted" }) + + let renameCall = 0 + renameMock.mockImplementation(async (oldPath, newPath) => { + renameCall++ + if (renameCall === 2) throw new Error("Primary rename failed") + if (renameCall === 3) throw new Error("Rollback rename failed") + return originalRename + ? originalRename(oldPath, newPath) + : (fs.rename as unknown as typeof import("fs/promises").rename)(oldPath, newPath) + }) + + await expect(store.resumeInterruptedTask("child")).rejects.toThrow("Primary rename failed") + + // The commit rename removed the target and the rollback rename + // failed. The surviving backup holds the only copy of the prior + // interrupted record. The store kept that record in its cache. + const files = await fs.readdir(taskDir) + const backupName = files.find((name) => name.includes(".bak_")) + expect(backupName).toBeDefined() + const backupRecord = JSON.parse(await fs.readFile(path.join(taskDir, backupName!), "utf8")) + expect(backupRecord).toMatchObject({ id: "child", status: "interrupted" }) + expect(store.get("child")?.status).toBe("interrupted") + + // Recovery: restore the backup and the record is readable again. + await fs.rename(path.join(taskDir, backupName!), path.join(taskDir, "history_item.json")) + await store.invalidate("child") + expect(store.get("child")).toMatchObject({ id: "child", status: "interrupted" }) + } finally { + if (originalRename) renameMock.mockImplementation(originalRename) + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + it("preserves independent stale-cache deltas through the real per-file lock", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-history-real-lock-")) const storeA = new TaskHistoryStore(storagePath) diff --git a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts index 8387f30e74..ebb71b4f7c 100644 --- a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts +++ b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts @@ -6,6 +6,7 @@ import { delegateTaskToChild, interruptDelegatedChild, LifecycleTransitionError, + resumeInterruptedChild, resumeInterruptedTask, settleRejectedCreateSubtaskAction, } from "../taskLifecycle" @@ -78,6 +79,38 @@ describe("task lifecycle transitions", () => { expect(() => resumeInterruptedTask(item("child"))).toThrow(LifecycleTransitionError) }) + it("resumes an interrupted child only while its parent still awaits it", () => { + const parent = item("parent", { + status: "delegated", + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + }) + const child = item("child", { status: "interrupted", parentTaskId: "parent" }) + + expect(resumeInterruptedChild(parent, child)).toMatchObject({ status: "active", parentTaskId: "parent" }) + }) + + it("rejects the parent-locked resume after the parent re-delegates elsewhere", () => { + const redelegatedParent = item("parent", { + status: "delegated", + awaitingChildId: "new-child", + delegatedToId: "new-child", + childIds: ["child", "new-child"], + }) + const child = item("child", { status: "interrupted", parentTaskId: "parent" }) + + expect(() => resumeInterruptedChild(redelegatedParent, child)).toThrow(LifecycleTransitionError) + expect(() => resumeInterruptedChild(redelegatedParent, child)).toThrow(/no longer awaits/) + }) + + it("rejects the parent-locked resume when the parent released the delegation", () => { + const activeParent = item("parent") + const child = item("child", { status: "interrupted", parentTaskId: "parent" }) + + expect(() => resumeInterruptedChild(activeParent, child)).toThrow(/no longer awaits/) + }) + it("interrupts a child without clearing the parent's ownership", () => { const parent = item("parent", { status: "delegated", awaitingChildId: "child", delegatedToId: "child" }) const child = item("child", { parentTaskId: "parent" }) diff --git a/src/core/task-persistence/index.ts b/src/core/task-persistence/index.ts index 0baf9ce57e..6afd1e7d9b 100644 --- a/src/core/task-persistence/index.ts +++ b/src/core/task-persistence/index.ts @@ -21,6 +21,7 @@ export { delegateTaskToChild, interruptDelegatedChild, LifecycleTransitionError, + resumeInterruptedChild, settleRejectedCreateSubtaskAction, type HistoryItemStatus, VALID_TASK_STATUS_TRANSITIONS, diff --git a/src/core/task-persistence/taskLifecycle.ts b/src/core/task-persistence/taskLifecycle.ts index 25e0beed9e..cfb57bd68d 100644 --- a/src/core/task-persistence/taskLifecycle.ts +++ b/src/core/task-persistence/taskLifecycle.ts @@ -39,6 +39,19 @@ export function resumeInterruptedTask(task: HistoryItem, expectedParentTaskId?: return { ...task, status: "active" } } +/** + * Resumes an interrupted child through its parent's authoritative delegation + * pointer. The parent check must run under the parent record's lock so a + * competing host cannot re-delegate the parent between the check and the + * child activation. + */ +export function resumeInterruptedChild(parent: HistoryItem, child: HistoryItem): HistoryItem { + if (parent.status !== "delegated" || parent.awaitingChildId !== child.id) { + throw new LifecycleTransitionError(`Cannot resume task ${child.id}: parent ${parent.id} no longer awaits it`) + } + return resumeInterruptedTask(child, parent.id) +} + /** * Settles the pending create_subtask action whose delegation the authoritative * parent record rejected (#1714). Only the exact matching action ID is cleared; diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index 30b0ae097a..3682321ef2 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -1227,6 +1227,54 @@ describe("Task persistence", () => { }, ) + it.each(["noButtonClicked", "unexpected"])( + "does not resume a completed task after a declined %s resume_completed_task response", + async (response) => { + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + startTask: false, + initialStatus: "completed", + }) + const answer: Awaited> = { response: "noButtonClicked" } + // Model an unvalidated response arriving across the webview boundary. + Reflect.set(answer, "response", response) + vi.spyOn(task, "ask").mockResolvedValue(answer) + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + await task["resumeTaskFromHistory"]() + + expect(task.ask).toHaveBeenCalledWith("resume_completed_task") + expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() + expect(mockProvider.updateTaskHistory).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + }, + ) + + it("resumes an accepted completed task through the task loop without a store transition", async () => { + mockReadTaskMessages.mockResolvedValue([ + { ts: 1, type: "say" as const, say: "text" as const, text: "Original task" }, + ]) + mockReadApiMessages.mockResolvedValue([ + { role: "user", content: [{ type: "text", text: "Original task" }] }, + ]) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + startTask: false, + initialStatus: "completed", + }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + await task["resumeTaskFromHistory"]() + + expect(task.ask).toHaveBeenCalledWith("resume_completed_task") + expect(loop).toHaveBeenCalledTimes(1) + // Non-interrupted semantics: an accepted completed-task resume never + // moves the store record or calls the interrupted-resume path. + expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() + expect(mockProvider.updateTaskHistory).not.toHaveBeenCalled() + }) + it("does not restore stale lineage through repeated message saves after rejected resume", async () => { const history = { id: "detached", diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 9c86034016..2dadf4bf68 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -3938,41 +3938,39 @@ export class ClineProvider }) // 5) Persist parent delegation metadata BEFORE the child starts writing. - // atomicReadAndUpdate reads from the in-memory cache and writes back within a - // single lock acquisition — no concurrent writer can slip between the read and - // write, and the pure updater cannot re-enter the lock (no deadlock). - // Broadcast and cache invalidation happen outside the lock after it releases. + // atomicUpdateParentDelegation reads the on-disk parent record and + // writes back within a single parent-file lock acquisition. The + // builder runs inside that lock, so no host can change the parent or + // its awaited child between validation and commit. Broadcast and + // cache invalidation happen outside the lock after it releases. // // If the parent is already "delegated" to a previous interrupted child (the user // navigated back to the parent and continued working), we implicitly sever the old // link here (delegated → active → delegated) so no explicit Abandon step is needed. - // The old awaited child's status is re-read INSIDE the updater (which runs - // synchronously under the store lock) so a concurrent abandon or completion cannot - // slip between the status snapshot and the write. An active child must never be + // The store re-reads the old awaited child's status from disk INSIDE the + // lock, so a concurrent resume, abandon, or completion cannot slip between + // the status snapshot and the write. An active child must never be // silently detached. try { - await this.taskHistoryStore.atomicReadAndUpdate(parentTaskId, (historyItem) => { - if (pendingActionId && historyItem.pendingAction?.actionId !== pendingActionId) { - throw new Error( - `[delegateParentAndOpenChild] Pending action mismatch for parent ${parentTaskId}: expected ${pendingActionId}, found ${historyItem.pendingAction?.actionId}`, - ) - } - const awaitedChildStatus = historyItem.awaitingChildId - ? this.taskHistoryStore.get(historyItem.awaitingChildId)?.status - : undefined - const delegated = delegateTaskToChild(historyItem, child.taskId, awaitedChildStatus) - return { - ...delegated, - pendingAction: - delegated.pendingAction?.actionId === pendingActionId ? undefined : delegated.pendingAction, - } - }) + const updatedItem = await this.taskHistoryStore.atomicUpdateParentDelegation( + parentTaskId, + (diskParent, awaitedChildDiskStatus) => { + if (pendingActionId && diskParent.pendingAction?.actionId !== pendingActionId) { + throw new Error( + `[delegateParentAndOpenChild] Pending action mismatch for parent ${parentTaskId}: expected ${pendingActionId}, found ${diskParent.pendingAction?.actionId}`, + ) + } + const delegated = delegateTaskToChild(diskParent, child.taskId, awaitedChildDiskStatus) + return { + ...delegated, + pendingAction: + delegated.pendingAction?.actionId === pendingActionId ? undefined : delegated.pendingAction, + } + }, + ) this.recentTasksCache = undefined - if (this.isViewLaunched) { - const updatedItem = this.taskHistoryStore.get(parentTaskId) - if (updatedItem) { - await this.postMessageToWebview({ type: "taskHistoryItemUpdated", taskHistoryItem: updatedItem }) - } + if (this.isViewLaunched && updatedItem) { + await this.postMessageToWebview({ type: "taskHistoryItemUpdated", taskHistoryItem: updatedItem }) } } catch (err) { this.log( diff --git a/src/utils/__tests__/safeWriteJson.test.ts b/src/utils/__tests__/safeWriteJson.test.ts index 0ebdea281d..d2f6ff75d5 100644 --- a/src/utils/__tests__/safeWriteJson.test.ts +++ b/src/utils/__tests__/safeWriteJson.test.ts @@ -435,8 +435,8 @@ describe("safeWriteJson", () => { }) // Test for rollback failure scenario - test("should log error and re-throw original if rollback fails", async () => { - const initialData = { message: "Initial, should be lost if rollback fails" } + test("should preserve the backup and report partial failure when both renames fail", async () => { + const initialData = { message: "Initial, must stay recoverable" } const newData = { message: "New content" } await fsPromisesActuals.writeFile!(currentTestFilePath, JSON.stringify(initialData)) @@ -466,6 +466,20 @@ describe("safeWriteJson", () => { expect.objectContaining({ message: "Rollback rename failed" }), ) + // The backup survives: with the target gone and the rollback failed, + // it holds the only copy of the previous record. The partial failure + // names the recovery path instead of erasing the record. + const leftoverFiles = (await fs.readdir(tempDir)).filter((name) => name.includes(".bak_")) + expect(leftoverFiles).toHaveLength(1) + const backupContent = JSON.parse(await fs.readFile(path.join(tempDir, leftoverFiles[0]!), "utf-8")) + expect(backupContent).toEqual(initialData) + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Partial write failure")) + // The partial-failure report names the surviving backup file. + const partialFailureCall = consoleErrorSpy.mock.calls + .map((call) => String(call[0])) + .find((text) => text.includes("Partial write failure")) + expect(partialFailureCall).toContain(leftoverFiles[0]!) + consoleErrorSpy.mockRestore() }) diff --git a/src/utils/safeWriteJson.ts b/src/utils/safeWriteJson.ts index 7da68b2a7a..d36c9b0b87 100644 --- a/src/utils/safeWriteJson.ts +++ b/src/utils/safeWriteJson.ts @@ -17,6 +17,15 @@ export interface SafeWriteJsonOptions { */ prettyPrint?: boolean + /** + * Skip advisory-lock acquisition because the caller already holds the + * shared advisory lock for this exact path through `acquireFileLock` or + * `withFileLock`. `proper-lockfile` cannot re-enter a lock it already + * holds, so only callers that acquired that lock for this path may set + * this option. + */ + assumeLockHeld?: boolean + /** * When provided, the current file is read under the advisory lock * and passed to this function along with the incoming data. The @@ -69,7 +78,7 @@ async function safeWriteJson(filePath: string, data: any, options?: SafeWriteJso // remains a no-op, so the finally block in the main file operations // try-catch-finally won't try to release an unacquired lock if this // path is taken. - releaseLock = await acquireFileLock(absoluteFilePath) + releaseLock = options?.assumeLockHeld ? async () => {} : await acquireFileLock(absoluteFilePath) // Variables to hold the actual paths of temp files if they are created. let actualTempNewFilePath: string | null = null @@ -177,16 +186,14 @@ async function safeWriteJson(filePath: string, data: any, options?: SafeWriteJso } } - // Cleanup the .bak file if it still needs to be (i.e., wasn't successfully restored) + // A surviving backup holds the only copy of the previous record: the + // commit rename removed the target and the rollback rename failed. + // Never delete it. Report the recovery path so the record stays + // recoverable and the failure is not mistaken for a clean rollback. if (actualTempBackupFilePath) { - try { - await fs.unlink(actualTempBackupFilePath) - } catch (cleanupError) { - console.error( - `[Catch] Failed to clean up temporary backup file ${actualTempBackupFilePath}:`, - cleanupError, - ) - } + console.error( + `[Catch] Partial write failure for ${absoluteFilePath}: the previous record is preserved at ${actualTempBackupFilePath} and must be recovered manually.`, + ) } throw originalError // This MUST be the error that rejects the promise. } finally { From 266db7006f2b6f0ef4047cfe4e7e8b960148f27e Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 13 Oct 2026 00:47:42 +0000 Subject: [PATCH 05/14] fix: use locked disk state for delegation writes --- .../__tests__/repro-init-failure.spec.ts | 50 --------------- src/core/task-persistence/TaskHistoryStore.ts | 6 +- .../TaskHistoryStore.realConcurrency.spec.ts | 61 ++++++++++++++++++- .../__tests__/taskLifecycle.spec.ts | 10 +++ 4 files changed, 75 insertions(+), 52 deletions(-) delete mode 100644 src/core/checkpoints/__tests__/repro-init-failure.spec.ts diff --git a/src/core/checkpoints/__tests__/repro-init-failure.spec.ts b/src/core/checkpoints/__tests__/repro-init-failure.spec.ts deleted file mode 100644 index ffc2535a3c..0000000000 --- a/src/core/checkpoints/__tests__/repro-init-failure.spec.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { describe, it, expect, vi } from "vitest" -import { getCheckpointService } from "../index" - -vi.mock("vscode", () => ({ - window: { showWarningMessage: vi.fn(), createTextEditorDecorationType: vi.fn(() => ({})) }, - env: {}, - Uri: {}, -})) -vi.mock("@roo-code/telemetry", () => ({ TelemetryService: { instance: {} } })) -vi.mock("../../../utils/path", () => ({ getWorkspacePath: vi.fn(() => "/ws") })) -vi.mock("../../../utils/git", () => ({ checkGitInstalled: vi.fn().mockResolvedValue(true) })) -vi.mock("../../../i18n", () => ({ t: (k: string) => k })) -vi.mock("../../../services/checkpoints") - -describe("repro: init failure disables checkpoints for the whole task", () => { - it("does not retry after one initShadowGit failure", async () => { - const { RepoPerTaskCheckpointService } = await import("../../../services/checkpoints") - const initShadowGit = vi.fn().mockRejectedValue(new Error("simulated git failure")) - vi.mocked(RepoPerTaskCheckpointService.create).mockReturnValue({ on: vi.fn(), initShadowGit } as never) - - const task = { - taskId: "t", - enableCheckpoints: true, - checkpointTimeout: 60, - checkpointService: undefined, - checkpointServiceInitializing: false, - providerRef: { - deref: () => ({ - context: { globalStorageUri: { fsPath: "/s" } }, - log: vi.fn(), - postMessageToWebview: vi.fn(), - }), - }, - } - - await getCheckpointService(task as never) - console.log("after 1st:", { - enable: task.enableCheckpoints, - svcSet: !!task.checkpointService, - initializing: task.checkpointServiceInitializing, - }) - const second = await getCheckpointService(task as never) - const third = await getCheckpointService(task as never) - - expect(initShadowGit).toHaveBeenCalledTimes(1) - expect(task.enableCheckpoints).toBe(false) - expect(second).toBeUndefined() - expect(third).toBeUndefined() - }) -}) diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index 5c4bb34556..4276d6e5d7 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -1213,7 +1213,11 @@ export class TaskHistoryStore { ) } - const delta = this.buildDelta(parentTaskId, cached, next) + // Diff against the locked disk record, not the stale cache: a + // field whose new value equals the stale cached value would + // otherwise be dropped from the delta, and mergeHistoryDelta + // would keep the disk value for it. + const delta = this.buildDelta(parentTaskId, diskParent, next) const merged = mergeHistoryDelta(diskParent, next, delta) await safeWriteJson(parentFilePath, merged, { assumeLockHeld: true }) diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index ad53465151..5ad91cb52a 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -5,7 +5,7 @@ import * as path from "path" import type { HistoryItem } from "@roo-code/types" import { TaskHistoryStore } from "../TaskHistoryStore" -import { delegateTaskToChild, LifecycleTransitionError } from "../taskLifecycle" +import { abandonDelegatedChild, delegateTaskToChild, LifecycleTransitionError } from "../taskLifecycle" // Wrap the real fs/promises so a test can pause inside the per-file lock // while every call still runs against the real filesystem. @@ -328,6 +328,65 @@ describe("TaskHistoryStore real cross-host locking", () => { } }) + it("keeps the persisted delegation when a stale cached parent already shows the new status", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "delegation-stale-status-delta-")) + const storeA = new TaskHistoryStore(storagePath) + const storeB = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await storeA.initialize() + await storeA.upsert(parent) + await storeA.upsert(child) + // storeB caches the delegated parent and never refreshes it. + await storeB.initialize() + expect(storeB.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child" }) + + // Another host abandons the awaited child, so the disk parent + // becomes active with no delegation pointers. + const abandonedParent = storeA.get("parent") + const abandonedChild = storeA.get("child") + if (!abandonedParent || !abandonedChild) { + throw new Error("abandon setup records are missing from store A") + } + await storeA.atomicUpdatePair( + "child", + "parent", + (childRecord) => abandonDelegatedChild(abandonedParent, childRecord).child, + (parentRecord) => abandonDelegatedChild(parentRecord, abandonedChild).parent, + ) + await storeA.invalidate("parent") + const parentAfterAbandon = storeA.get("parent") + expect(parentAfterAbandon?.status).toBe("active") + expect(parentAfterAbandon?.awaitingChildId).toBeUndefined() + + // storeB re-delegates from its stale cache. The builder runs on the + // disk parent (active), but the cached parent already shows the new + // status, so the delta must be computed against the disk record. + await storeB.atomicUpdateParentDelegation("parent", (diskParent, awaitedChildDiskStatus) => + delegateTaskToChild(diskParent, "child-b", awaitedChildDiskStatus), + ) + + await storeB.invalidate("parent") + expect(storeB.get("parent")).toMatchObject({ + status: "delegated", + awaitingChildId: "child-b", + delegatedToId: "child-b", + }) + } finally { + storeA.dispose() + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + it("keeps the prior task record recoverable when resume persistence fails both renames", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-partial-failure-")) const store = new TaskHistoryStore(storagePath) diff --git a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts index ebb71b4f7c..2c717a7d42 100644 --- a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts +++ b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts @@ -111,6 +111,16 @@ describe("task lifecycle transitions", () => { expect(() => resumeInterruptedChild(activeParent, child)).toThrow(/no longer awaits/) }) + it("rejects the parent-locked resume when an active parent retains a stale awaited pointer", () => { + // Completion accepts an active parent with a surviving awaited pointer. + // The status check must reject the resume on its own, without help from + // the awaited-pointer check. + const activeParent = item("parent", { awaitingChildId: "child", delegatedToId: "child" }) + const child = item("child", { status: "interrupted", parentTaskId: "parent" }) + + expect(() => resumeInterruptedChild(activeParent, child)).toThrow(/no longer awaits/) + }) + it("interrupts a child without clearing the parent's ownership", () => { const parent = item("parent", { status: "delegated", awaitingChildId: "child", delegatedToId: "child" }) const child = item("child", { parentTaskId: "parent" }) From 6c0242041ca56e0b41b71a8e218a9ef96870f357 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 13 Oct 2026 03:49:48 +0000 Subject: [PATCH 06/14] fix: preserve persisted lineage on interrupted task saves --- docs/architecture/task-lifecycle-model.md | 26 +- ...ask-lifecycle-persisted-ownership-model.md | 52 ++-- scripts/check-task-store-concurrency.ts | 46 +++ .../removeClineFromStack-delegation.spec.ts | 69 ++++- src/core/task-persistence/TaskHistoryStore.ts | 58 +++- .../TaskHistoryStore.realConcurrency.spec.ts | 120 ++++++++ src/core/task/Task.ts | 12 +- .../task/__tests__/Task.persistence.spec.ts | 11 +- .../Task.saveClineMessages.lineage.spec.ts | 284 ++++++++++++++++++ src/core/webview/ClineProvider.ts | 24 +- 10 files changed, 625 insertions(+), 77 deletions(-) create mode 100644 src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index b093638246..23d1fda7bb 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -225,19 +225,19 @@ The [Task lifecycle verification GAP report](./task-lifecycle-gap-report.md) is - **Planned-only:** specifies behavior not enabled in production. - **Type/static convention:** centralized typing or guidance without repository-wide enforcement. -| Gap | Invariant | Current evidence | Production/model boundary and runtime impact | Missing work and objective closure criteria | -| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Cross-host completion ownership | A completion may mutate a parent only while that parent authoritatively awaits the same child. | **Known-unsafe witness** plus a production-backed reducer guard. | The reducer rejects stale authoritative input, but disk revalidation checks status legality rather than exact-child ownership. A stale host can orphan the newer child. | Revalidate exact ownership under the disk lock; add deterministic two-store regression coverage; replace the expected witness with a universal bounded invariant. | -| Monotonic detachment | Message persistence must never restore lineage after abandonment clears it. | **Known-unsafe witness** plus a production-backed detach reducer. | A live task rebuilds lineage from stale fields; a later delta can restore parent/root IDs while preserving the interrupted status. | Give lifecycle fields a disk-authoritative write owner or tombstone/generation; prove stale metadata saves cannot alter them; promote the witness to an invariant. | -| Task-local mode consumers | Every mode-sensitive child operation must use the child task's immutable mode, not shared provider/view state. | Write side: **production-backed bounded matrix**. Three confirmed readers: **focused production tests**. Reader checker: **proxy/partial**. | The merged fix covers environment rendering, built-in validation, and custom-tool execution. The pure checker does not execute them, and other provider-mode readers are not exhaustively classified. | Complete the reader inventory, add focused divergent-mode evidence for each required task-local consumer, and enforce a task-local reader boundary before claiming universal isolation. | -| Tool-originated child initialization | Every required child field originating in `new_task` must be task-scoped, durably owned, and equivalent after rehydration. | Argument forwarding has focused tests; durable initial-state refinement is **unmodeled**. | Normal creation uses explicit model-supplied child todos rather than copying the parent, but initial todos demonstrate that some child state is process-local and can disappear. | Inventory all child initialization fields; define durable ownership, deep-copy, precedence, publication, and rehydration contracts with focused and E2E checks (`LIFE-GAP-035`). | -| Tool approval ownership | An interactive edit or settlement may affect only the matching task, action, and tool call. | Single-approval behavior has focused tests; cross-task correlation is **unmodeled and known unsafe by inspection**. | `update_todo_list` uses process-global edit state without task/action identity, allowing delayed or concurrent approval contamination. | Correlate proposal/edit/approval/cancellation by task and action/tool-call ID; reject stale edits and test interleavings (`LIFE-GAP-036`). | -| Tool partial-state isolation | Partial presentation state must be owned by one task and tool call and cleared on every terminal path. | Tool-local tests only; cross-call/task interleavings are **unmodeled**. | Singleton handlers share `lastSeenPartialPath`, so another call can create false or missed path stabilization. | Key state by `(taskId, toolCallId)` or instantiate handlers per call; test interleavings and cleanup (`LIFE-GAP-037`). | -| Tool identity correspondence | Parsed call, durable history, approval, execution, result, pending action, and replay must have one collision-resistant identity. | Duplicate-ID helper tests are **proxy/partial**; end-to-end correspondence is **unmodeled**. | Non-injective sanitization can collapse distinct raw IDs in history while execution still treats them as separate calls. | Reject/disambiguate collisions and prove a one-to-one identity mapping across native/MCP calls and restart (`LIFE-GAP-038`). | -| Serial delegation baseline | Current production permits one awaited active child per parent and resumes the parent only after child release. | Singular ownership is **production-backed bounded**; scheduler/provider ordering is mixed production/abstract. | Reducers and the normal provider path enforce singular ownership, but stale cross-host persistence can still violate the relationship; scheduler capacity is defaulted, not statically fixed. | Close current serial persistence/ordering gaps and ratchet the provider's one-permit baseline. Treat fan-out as separate optional scope. | -| Shared lifecycle status vocabulary | All consumers should derive task status from one exported owner. | **Type/static convention**; the visible interrupted-status symptom is repaired. | Persistence derives its alias from `HistoryItem`, but CLI history adapters still copy the union, allowing future drift. | Export one shared status type, consume it at CLI/persistence boundaries, and rely on typechecking or a static rule to reject copied incompatible vocabulary. | -| Cross-model refinement | No independent checker result may be combined into a stronger end-to-end claim without an executable boundary mapping. | **Explicit limitation** only. | Persistence, provider publication, parser scope, cleanup, and completion durability run as separate state spaces; omitted consumers can violate a premise after another checker passes. | Add production-grounded boundary events and a bounded joint strategy for each genuinely cross-model invariant, or keep claims explicitly local. | -| Completion event consumers | Public completion must not be treated as universally durable beyond the modeled readiness contract. | **Abstract bounded** model plus focused tests and one fresh-host E2E path. | The model abstracts durability and parent reopen; provider status metadata and downstream `TaskCompleted` consumers are outside its state. | Inventory downstream consumers and add refinement checks only for guarantees they require; retain power-loss, filesystem, retry-count, and liveness exclusions. | +| Gap | Invariant | Current evidence | Production/model boundary and runtime impact | Missing work and objective closure criteria | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cross-host completion ownership | A completion may mutate a parent only while that parent authoritatively awaits the same child. | **Known-unsafe witness** plus a production-backed reducer guard. | The reducer rejects stale authoritative input, but disk revalidation checks status legality rather than exact-child ownership. A stale host can orphan the newer child. | Revalidate exact ownership under the disk lock; add deterministic two-store regression coverage; replace the expected witness with a universal bounded invariant. | +| Monotonic detachment | Message persistence must never restore lineage after abandonment clears it. | Rehydrated-interrupted saves: **production-backed write owner with focused integration tests**. Live-task saves: **known-unsafe witness**. | A rehydrated interrupted save now preserves the persisted lineage under the disk lock (`upsertPreservingLineage`), so it cannot restore or resurrect links. A live task still rebuilds lineage from stale fields; a later delta can restore parent/root IDs while preserving the interrupted status. | Keep the rehydrated-class owner; give live-task saves a disk-authoritative write owner or tombstone/generation; promote the witness to an invariant. | +| Task-local mode consumers | Every mode-sensitive child operation must use the child task's immutable mode, not shared provider/view state. | Write side: **production-backed bounded matrix**. Three confirmed readers: **focused production tests**. Reader checker: **proxy/partial**. | The merged fix covers environment rendering, built-in validation, and custom-tool execution. The pure checker does not execute them, and other provider-mode readers are not exhaustively classified. | Complete the reader inventory, add focused divergent-mode evidence for each required task-local consumer, and enforce a task-local reader boundary before claiming universal isolation. | +| Tool-originated child initialization | Every required child field originating in `new_task` must be task-scoped, durably owned, and equivalent after rehydration. | Argument forwarding has focused tests; durable initial-state refinement is **unmodeled**. | Normal creation uses explicit model-supplied child todos rather than copying the parent, but initial todos demonstrate that some child state is process-local and can disappear. | Inventory all child initialization fields; define durable ownership, deep-copy, precedence, publication, and rehydration contracts with focused and E2E checks (`LIFE-GAP-035`). | +| Tool approval ownership | An interactive edit or settlement may affect only the matching task, action, and tool call. | Single-approval behavior has focused tests; cross-task correlation is **unmodeled and known unsafe by inspection**. | `update_todo_list` uses process-global edit state without task/action identity, allowing delayed or concurrent approval contamination. | Correlate proposal/edit/approval/cancellation by task and action/tool-call ID; reject stale edits and test interleavings (`LIFE-GAP-036`). | +| Tool partial-state isolation | Partial presentation state must be owned by one task and tool call and cleared on every terminal path. | Tool-local tests only; cross-call/task interleavings are **unmodeled**. | Singleton handlers share `lastSeenPartialPath`, so another call can create false or missed path stabilization. | Key state by `(taskId, toolCallId)` or instantiate handlers per call; test interleavings and cleanup (`LIFE-GAP-037`). | +| Tool identity correspondence | Parsed call, durable history, approval, execution, result, pending action, and replay must have one collision-resistant identity. | Duplicate-ID helper tests are **proxy/partial**; end-to-end correspondence is **unmodeled**. | Non-injective sanitization can collapse distinct raw IDs in history while execution still treats them as separate calls. | Reject/disambiguate collisions and prove a one-to-one identity mapping across native/MCP calls and restart (`LIFE-GAP-038`). | +| Serial delegation baseline | Current production permits one awaited active child per parent and resumes the parent only after child release. | Singular ownership is **production-backed bounded**; scheduler/provider ordering is mixed production/abstract. | Reducers and the normal provider path enforce singular ownership, but stale cross-host persistence can still violate the relationship; scheduler capacity is defaulted, not statically fixed. | Close current serial persistence/ordering gaps and ratchet the provider's one-permit baseline. Treat fan-out as separate optional scope. | +| Shared lifecycle status vocabulary | All consumers should derive task status from one exported owner. | **Type/static convention**; the visible interrupted-status symptom is repaired. | Persistence derives its alias from `HistoryItem`, but CLI history adapters still copy the union, allowing future drift. | Export one shared status type, consume it at CLI/persistence boundaries, and rely on typechecking or a static rule to reject copied incompatible vocabulary. | +| Cross-model refinement | No independent checker result may be combined into a stronger end-to-end claim without an executable boundary mapping. | **Explicit limitation** only. | Persistence, provider publication, parser scope, cleanup, and completion durability run as separate state spaces; omitted consumers can violate a premise after another checker passes. | Add production-grounded boundary events and a bounded joint strategy for each genuinely cross-model invariant, or keep claims explicitly local. | +| Completion event consumers | Public completion must not be treated as universally durable beyond the modeled readiness contract. | **Abstract bounded** model plus focused tests and one fresh-host E2E path. | The model abstracts durability and parent reopen; provider status metadata and downstream `TaskCompleted` consumers are outside its state. | Inventory downstream consumers and add refinement checks only for guarantees they require; retain power-loss, filesystem, retry-count, and liveness exclusions. | ## Historical provenance diff --git a/docs/architecture/task-lifecycle-persisted-ownership-model.md b/docs/architecture/task-lifecycle-persisted-ownership-model.md index 36c04b2447..a72a9c4098 100644 --- a/docs/architecture/task-lifecycle-persisted-ownership-model.md +++ b/docs/architecture/task-lifecycle-persisted-ownership-model.md @@ -60,34 +60,36 @@ Inside each write, `writeTaskFile` runs `mergeHistoryDelta` under that file's ad ## LIFE-BLK-P1-002 — lifecycle-owned fields versus metadata writes (GAP-002) -**Bounded property (monotonic detachment).** Once a detach commit lands — `abandonDelegatedChild` clears the child's `parentTaskId`/`rootTaskId` — no later write whose lineage fields derive from a pre-detach snapshot may reintroduce them. This property is **currently violated in production**; this block makes the invariant and its ownership contract explicit and claims no present safety. +**Bounded property (monotonic detachment).** Once a detach commit lands — `abandonDelegatedChild` clears the child's `parentTaskId`/`rootTaskId` — no later write whose lineage fields derive from a pre-detach snapshot may reintroduce them. This property is **still violated for live-task saves**: a live task keeps its pre-detach lineage fields and rewrites them on every message save. Message saves from rehydrated interrupted tasks now route through a disk-authoritative write owner (`TaskHistoryStore.upsertPreservingLineage`), so that subclass satisfies the property; the production tests in `Task.saveClineMessages.lineage.spec.ts` pin it. This block makes the invariant and its ownership contract explicit. **Field ownership table.** Authoritative-writer assignment for every `historyItemSchema` field (`packages/types/src/history.ts`): -| Field | Writers today | Metadata writers may write? | -| ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| `status` | Lifecycle writes. Delegation, completion, and abandonment use the atomic updaters (`ClineProvider.ts:3992, 4273, 4468`). Standalone completion, interrupt, and cancel use `updateTaskHistory`, which calls `upsert` (`:400, 718, 3612`). The fail-closed cancel write at `:3638` also writes `interrupted`, and it clears lineage.. A child's first status comes from `initialStatus: "active"` at creation (`:3973`). | Preserve the current value only. `Task.saveClineMessages` does this (`Task.ts:1411-1412`). | -| `rootTaskId`, `parentTaskId` | Set on the child at creation, from the live `Task` through `taskMetadata` (`taskMetadata.ts:98-114`). Cleared by `abandonDelegatedChild`, and by the fail-closed cancel detach, which writes through `updateTaskHistory` (`ClineProvider.ts:3626-3638`). `delegateTaskToChild` writes parent fields only (`taskLifecycle.ts:48-54`). | **No.** Today every message save rewrites them from live `Task` state. | -| `delegatedToId`, `awaitingChildId` | `delegateTaskToChild`, `completeDelegatedChild`, `abandonDelegatedChild`, and the repair paths | No | -| `childIds` | The same reducers. The merge unions concurrent additions. | No | -| `completedByChildId`, `completionResultSummary` | `completeDelegatedChild` and reconciliation repair. Standalone completion writes `status` only. | No | -| `pendingAction` | `setPendingTaskAction` (`ClineProvider.ts:740`), the compare-and-clear settlement (`:752-757`), and the delegation and completion paths | No | -| `id`, `number`, `task` | Creation | Creation only | -| `ts` | Metadata. `taskMetadata` sets it from the last relevant message on every save (`taskMetadata.ts:69-73`). | Yes | -| `tokensIn`, `tokensOut`, `cacheWrites`, `cacheReads`, `totalCost`, `size` | Metadata (`taskMetadata` accounting) | Yes | -| `workspace`, `mode`, `apiConfigName` | Metadata. Mode and profile updates use `updateTaskHistory`. | Yes | +| Field | Writers today | Metadata writers may write? | +| ------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `status` | Lifecycle writes. Delegation, completion, and abandonment use the atomic updaters (`ClineProvider.ts:3992, 4273, 4468`). Standalone completion, interrupt, and cancel use `updateTaskHistory`, which calls `upsert` (`:400, 718, 3612`). The fail-closed cancel write at `:3638` also writes `interrupted`, and it clears lineage.. A child's first status comes from `initialStatus: "active"` at creation (`:3973`). | Preserve the current value only. `Task.saveClineMessages` does this (`Task.ts:1411-1412`). | +| `rootTaskId`, `parentTaskId` | Set on the child at creation, from the live `Task` through `taskMetadata` (`taskMetadata.ts:98-114`). Cleared by `abandonDelegatedChild`, and by the fail-closed cancel detach, which writes through `updateTaskHistory` (`ClineProvider.ts:3626-3638`). `delegateTaskToChild` writes parent fields only (`taskLifecycle.ts:48-54`). Rehydrated interrupted saves preserve the persisted links instead (`upsertPreservingLineage`). | **No.** Live-task message saves still rewrite them from live `Task` state. | +| `delegatedToId`, `awaitingChildId` | `delegateTaskToChild`, `completeDelegatedChild`, `abandonDelegatedChild`, and the repair paths | No | +| `childIds` | The same reducers. The merge unions concurrent additions. | No | +| `completedByChildId`, `completionResultSummary` | `completeDelegatedChild` and reconciliation repair. Standalone completion writes `status` only. | No | +| `pendingAction` | `setPendingTaskAction` (`ClineProvider.ts:740`), the compare-and-clear settlement (`:752-757`), and the delegation and completion paths | No | +| `id`, `number`, `task` | Creation | Creation only | +| `ts` | Metadata. `taskMetadata` sets it from the last relevant message on every save (`taskMetadata.ts:69-73`). | Yes | +| `tokensIn`, `tokensOut`, `cacheWrites`, `cacheReads`, `totalCost`, `size` | Metadata (`taskMetadata` accounting) | Yes | +| `workspace`, `mode`, `apiConfigName` | Metadata. Mode and profile updates use `updateTaskHistory`. | Yes | Administrative repair paths (`reconcileDelegationStateCore`, `repairActiveDelegation`, migration) write lifecycle fields with `skipTransitionCheck: true`; they are lifecycle-owned writes, not metadata writes. -**Current deviation and witness.** `Task.saveClineMessages` rebuilds `rootTaskId`/`parentTaskId` from the live `Task` via `taskMetadata`, preserving only `status` from the store (`Task.ts`: the `existingStatus` spread). `computeHistoryDelta` therefore includes lineage fields whenever the live snapshot diverges from cache, and `mergeHistoryDelta` applies them last-writer-wins. The shared-store checker's `#1021` known-unsafe witness (`stale save detachment`, the exact 11-action shortest trace through `A.refresh` into `stale-save-a.commit(child-a)`) shows a stale save re-attaching abandoned lineage while preserving the newer `interrupted` status. The witness stays ratcheted until the closure criteria below are met. +**Current deviation and witness.** A live task's message save still rebuilds `rootTaskId`/`parentTaskId` from the live `Task` via `taskMetadata` and rewrites them through `upsert` whenever they diverge from cache. A rehydrated interrupted task's save instead calls `updateTaskHistory` with `preserveLineage: true`, which strips the links from the delta and from the written payload (`TaskHistoryStore.upsertCore`), so the locked disk state stays authoritative and a deleted record is recreated without links. The shared-store checker retains the `#1021` known-unsafe witness (`stale save detachment`, the exact 11-action shortest trace through `A.refresh` into `stale-save-a.commit(child-a)`) for the live-task class, and a new `preserve save keeps detachment` scenario pins the monotonic-detachment invariant for the preserve-lineage write owner. The witness stays ratcheted until the closure criteria below are met. -**Model/checker change.** None in this PR: the field ownership table and monotonic-detachment invariant are the deliverable, and the checker already retains the exact witness. The later implementation PR must not strip lifecycle fields at the `updateTaskHistory` boundary, because the interrupt, cancel, and standalone-completion writes pass through it (see the table). It must do one of these, then promote the witness to a universal invariant: +**Implemented write owner (rehydrated interrupted saves).** `Task.saveClineMessages` passes `preserveLineage: true` to `updateTaskHistory` only when `initialStatus === "interrupted"`, so creation-time lineage still flows through the normal first save of a live child. The store strips the links inside the write lock (`upsertPreservingLineage`), never at the `updateTaskHistory` boundary for other callers: interrupt, cancel, and standalone-completion writes keep full authority (see the table). -- Add a new history write inside `createTask` that persists the child's lineage and its first status. `createTask` writes no history today. `initialStatus: "active"` is only an argument (`ClineProvider.ts:3971-3976`), and the first message save persists both that status and the lineage. Then strip lineage fields from every message save (`Task.saveClineMessages` into `taskMetadata`). Stripping without the new creation write would leave new children unlinked. +**Model/checker change.** The shared-store checker adds the `preserve-save-a` action and the `preserve save keeps detachment` scenario with the per-commit monotonic-detachment invariant. The `#1021` witness keeps ratcheting the live-task class. Closing it needs one of these, then promotion of the witness to a universal invariant: + +- Add a new history write inside `createTask` that persists the child's lineage and its first status. Then strip lineage fields from every message save (`Task.saveClineMessages` into `taskMetadata`). Stripping without the new creation write would leave new children unlinked. - Route every lifecycle write through one lifecycle write API first, and then strip lifecycle fields at the metadata boundary. - Make lock-time revalidation reject stale lineage deltas (tombstone or generation). It depends on the LIFE-BLK-P1-001 vocabulary above, which this document defines. -**Test evidence required for closure (deterministic fault injection).** Interrupt a save at each persistence step of `Task.saveClineMessages` (message write, metadata build, `updateTaskHistory` upsert) with an interleaved abandonment commit, and assert that after every interruption point the detached lineage is never restored and any partial write rolls back or is superseded by the authoritative record. +**Test evidence for the rehydrated class (landed).** `Task.saveClineMessages.lineage.spec.ts` drives the real store, real metadata builder, and real per-task files through the actual save: a record deleted by another host is recreated without links, a record detached on disk stays detached beside a linked local cache, and a valid linked record keeps its links. `TaskHistoryStore.realConcurrency.spec.ts` pins the same three outcomes at the store layer. **Still required for full closure (deterministic fault injection).** Interrupt a save at each persistence step of `Task.saveClineMessages` (message write, metadata build, `updateTaskHistory` upsert) with an interleaved abandonment commit, and assert that after every interruption point the detached lineage is never restored and any partial write rolls back or is superseded by the authoritative record. ## LIFE-BLK-P1-012 — attempt-generation state (GAP-012) @@ -215,13 +217,13 @@ A `completed → active` transition breaks two existing guarantees, so that opti ## Traceability -| Block | GAP | Delivered here | Closure evidence deferred to implementation PR | -| --------------- | --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| LIFE-BLK-P1-001 | 001 | Lock-time ownership check for completion, abandonment, re-delegation, and first delegation, write order and the roll-forward recovery rule (ADR-0005), witness, bounds, and promotion tests | Lock-time ownership check, two-store regression tests, witness promotion | -| LIFE-BLK-P1-002 | 002 | Field ownership table, monotonic-detachment invariant, stale-save witness linked | Owner enforcement or tombstone/generation, fault-injection save tests | -| LIFE-BLK-P1-012 | 012 | Lock-time generation check, generation semantics, missing-token rule, replay binding, increment at resume, settlement on actionId and generation (ADR-0003), pending-action replay and settlement, landmarks, bounded checker shape | Persisted generation field, reducer, store, and API rejection, restart E2E | -| LIFE-BLK-P1-017 | 017 | Read and first-insert aliasing, caller classification, immutable-read recommendation, test criteria | Clone-and-freeze implementation plus mutation-regression test | -| LIFE-BLK-P1-020 | 020 | Named stale-cache histories, including watcher scope and pending actions, bounded properties, test criteria | Missed-watch, same-mtime, and fault-injection convergence tests | -| LIFE-BLK-P1-039 | 039 | Gap and open decision recorded | Continuation decision, specification, and tests | +| Block | GAP | Delivered here | Closure evidence deferred to implementation PR | +| --------------- | --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| LIFE-BLK-P1-001 | 001 | Lock-time ownership check for completion, abandonment, re-delegation, and first delegation, write order and the roll-forward recovery rule (ADR-0005), witness, bounds, and promotion tests | Lock-time ownership check, two-store regression tests, witness promotion | +| LIFE-BLK-P1-002 | 002 | Field ownership table, monotonic-detachment invariant, stale-save witness linked, preserve-lineage write owner for rehydrated interrupted saves, checker scenario and invariant | Live-task owner enforcement or tombstone/generation, fault-injection save tests | +| LIFE-BLK-P1-012 | 012 | Lock-time generation check, generation semantics, missing-token rule, replay binding, increment at resume, settlement on actionId and generation (ADR-0003), pending-action replay and settlement, landmarks, bounded checker shape | Persisted generation field, reducer, store, and API rejection, restart E2E | +| LIFE-BLK-P1-017 | 017 | Read and first-insert aliasing, caller classification, immutable-read recommendation, test criteria | Clone-and-freeze implementation plus mutation-regression test | +| LIFE-BLK-P1-020 | 020 | Named stale-cache histories, including watcher scope and pending actions, bounded properties, test criteria | Missed-watch, same-mtime, and fault-injection convergence tests | +| LIFE-BLK-P1-039 | 039 | Gap and open decision recorded | Continuation decision, specification, and tests | No block in this document claims its parent GAP is closed or that current production behavior is safe against the named witnesses. diff --git a/scripts/check-task-store-concurrency.ts b/scripts/check-task-store-concurrency.ts index cf6f5f7d64..68dd1becb1 100644 --- a/scripts/check-task-store-concurrency.ts +++ b/scripts/check-task-store-concurrency.ts @@ -23,6 +23,7 @@ type OperationId = | "complete-a" | "redelegate-b" | "stale-save-a" + | "preserve-save-a" | "abandon-b" | "reject-a" type RecordMap = Partial> @@ -233,6 +234,23 @@ const operationSpecs: Record = { return [{ ...stale, tokensOut: stale.tokensOut + 1 }] }, }, + "preserve-save-a": { + id: "preserve-save-a", + host: "A", + externalSnapshot: true, + allowRefreshAfterRead: true, + buildWrites: (snapshot) => { + const stale = getRequired(snapshot, "child-a") + // A rehydrated interrupted task's message save declares its + // in-memory lineage non-authoritative. The preserve-lineage + // write owner strips parent and root links from the payload + // and delta before anything is written. + const stripped: HistoryItem = { ...stale, tokensOut: stale.tokensOut + 1 } + delete stripped.parentTaskId + delete stripped.rootTaskId + return [stripped] + }, + }, "abandon-b": { id: "abandon-b", host: "B", @@ -681,6 +699,34 @@ const scenarios: Scenario[] = [ }, check: () => [], }, + { + name: "preserve save keeps detachment", + operations: [operationSpecs["preserve-save-a"], operationSpecs["abandon-b"]], + check: (state) => { + // Monotonic detachment for the preserve-lineage write owner: + // once the detach commit lands, no preserve-save commit may + // reintroduce the child's parent or root links. + for (const commit of state.commits) { + if (commit.operationId !== "preserve-save-a" || commit.taskId !== "child-a" || !commit.previous) { + continue + } + if (commit.previous.parentTaskId === undefined && commit.next.parentTaskId !== undefined) { + return ["preserve-lineage save restored a detached parent link"] + } + if (commit.previous.rootTaskId === undefined && commit.next.rootTaskId !== undefined) { + return ["preserve-lineage save restored a detached root link"] + } + } + if (state.operations["abandon-b"]?.phase === "done") { + const disk = state.disk["child-a"] + if (!disk) return ["preserve save dropped the detached child record"] + if (disk.parentTaskId !== undefined || disk.rootTaskId !== undefined) { + return ["preserve-lineage save left reattached lineage after abandonment"] + } + } + return [] + }, + }, ] let totalStates = 0 diff --git a/src/__tests__/removeClineFromStack-delegation.spec.ts b/src/__tests__/removeClineFromStack-delegation.spec.ts index 0e1da0743c..d5fb5da886 100644 --- a/src/__tests__/removeClineFromStack-delegation.spec.ts +++ b/src/__tests__/removeClineFromStack-delegation.spec.ts @@ -173,24 +173,30 @@ describe("ClineProvider.removeClineFromStack() — pure lifecycle, no delegation }) describe("ClineProvider failed history restoration cleanup", () => { + function buildCleanupProvider() { + const cleanupListener = vi.fn() + const task = { + taskId: "failed-history-task", + instanceId: "inst-1", + emit: vi.fn(), + dispose: vi.fn().mockResolvedValue(undefined), + } as unknown as Task + const taskRegistry = new TaskRegistry() + taskRegistry.push(task) + const taskEventListeners = new Map([[task, [cleanupListener]]]) + const provider = { + taskRegistry, + taskEventListeners, + log: vi.fn(), + postStateToWebview: vi.fn().mockResolvedValue(undefined), + } as unknown as ClineProvider + return { provider, task, taskRegistry, taskEventListeners, cleanupListener } + } + it.each([new PendingActionSettlementError("settlement failed"), new LifecycleTransitionError("resume rejected")])( "removes the failed task without saving stale history after %s", async (error) => { - const cleanupListener = vi.fn() - const task = { - taskId: "failed-history-task", - instanceId: "inst-1", - emit: vi.fn(), - dispose: vi.fn().mockResolvedValue(undefined), - } as unknown as Task - const taskRegistry = new TaskRegistry() - taskRegistry.push(task) - const taskEventListeners = new Map([[task, [cleanupListener]]]) - const provider = { - taskRegistry, - taskEventListeners, - log: vi.fn(), - } as unknown as ClineProvider + const { provider, task, taskRegistry, taskEventListeners, cleanupListener } = buildCleanupProvider() await privateClineProvider.cleanupFailedHistoryTask.call(provider, task, error) @@ -199,9 +205,40 @@ describe("ClineProvider failed history restoration cleanup", () => { expect(cleanupListener).toHaveBeenCalledOnce() expect(taskEventListeners.has(task)).toBe(false) expect(task.dispose).toHaveBeenCalledOnce() + // The webview must drop the removed task instead of keeping a + // stale chat view for an unregistered instance. + expect(provider.postStateToWebview).toHaveBeenCalledOnce() }, ) + it("shows the rejection reason for a lifecycle resume failure", async () => { + const showErrorMessage = vi.spyOn(vscode.window, "showErrorMessage").mockResolvedValue(undefined) + const { provider, task } = buildCleanupProvider() + + await privateClineProvider.cleanupFailedHistoryTask.call( + provider, + task, + new LifecycleTransitionError("resume rejected"), + ) + + expect(showErrorMessage).toHaveBeenCalledWith("resume rejected") + showErrorMessage.mockRestore() + }) + + it("does not show a message for a pending action settlement failure", async () => { + const showErrorMessage = vi.spyOn(vscode.window, "showErrorMessage").mockResolvedValue(undefined) + const { provider, task } = buildCleanupProvider() + + await privateClineProvider.cleanupFailedHistoryTask.call( + provider, + task, + new PendingActionSettlementError("settlement failed"), + ) + + expect(showErrorMessage).not.toHaveBeenCalled() + showErrorMessage.mockRestore() + }) + it("keeps the task active after an unrelated history resume failure", async () => { const cleanupListener = vi.fn() const task = { @@ -217,6 +254,7 @@ describe("ClineProvider failed history restoration cleanup", () => { taskRegistry, taskEventListeners, log: vi.fn(), + postStateToWebview: vi.fn().mockResolvedValue(undefined), } as unknown as ClineProvider await privateClineProvider.cleanupFailedHistoryTask.call(provider, task, new Error("history read failed")) @@ -226,6 +264,7 @@ describe("ClineProvider failed history restoration cleanup", () => { expect(cleanupListener).not.toHaveBeenCalled() expect(taskEventListeners.has(task)).toBe(true) expect(task.dispose).not.toHaveBeenCalled() + expect(provider.postStateToWebview).not.toHaveBeenCalled() }) }) diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index 4276d6e5d7..cc2903e685 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -31,6 +31,18 @@ function mergeWithDisk(delta: Partial): (existing: unknown, incomin return (existing, incoming) => mergeHistoryDelta(existing, incoming as HistoryItem, delta) } +/** + * Copy a history item without its parent and root links. A preserve-lineage + * write persists this copy, so the write carries no lineage in its delta or + * payload and cannot overwrite or resurrect the authoritative disk links. + */ +function withoutLineage(item: HistoryItem): HistoryItem { + const copy: HistoryItem = { ...item } + delete copy.parentTaskId + delete copy.rootTaskId + return copy +} + /** * Durable intent for the one repair that spans an active delegated child and * its parent. Task files remain authoritative; this file only records the @@ -203,6 +215,20 @@ export class TaskHistoryStore { return this.withLock(() => this.upsertCore(item)) } + /** + * Insert or update a history item whose parent and root links are not + * authoritative. Message saves from rehydrated interrupted tasks use + * this: the in-memory lineage may be stale after another host detached + * or deleted the record. The write never carries lineage into the delta + * or the written payload, so the locked disk state stays authoritative. + * A linked record keeps its links, a detached record stays detached, and + * a deleted record is recreated without links. Lifecycle operations that + * legitimately set lineage must use `upsert` instead. + */ + async upsertPreservingLineage(item: HistoryItem): Promise { + return this.withLock(() => this.upsertCore(item, { preserveLineage: true })) + } + /** * Core upsert logic — must only be called from within `withLock`. * @@ -212,44 +238,52 @@ export class TaskHistoryStore { */ private async upsertCore( item: HistoryItem, - options: { skipTransitionCheck?: boolean } = {}, + options: { skipTransitionCheck?: boolean; preserveLineage?: boolean } = {}, ): Promise { - const existing = this.cache.get(item.id) + // A preserve-lineage write owns no lineage. Strip the links before the + // cache merge and the delta are built, so omission can neither retain + // stale cached links in the written payload nor resurrect them when + // the disk record is missing or unparseable. + const incoming = options.preserveLineage ? withoutLineage(item) : item + const existing = this.cache.get(incoming.id) // Enforce transition validity at the write boundary so that any caller // (including fire-and-forget saves) cannot silently stomp a terminal status. // Skip when there is no existing record — first insert has no prior state to transition from. // Normalize existing.status (undefined = legacy "active") before comparing so that writing // status: "active" onto a legacy item without a status field is not treated as a transition. - if (!options.skipTransitionCheck && existing && item.status !== undefined) { + if (!options.skipTransitionCheck && existing && incoming.status !== undefined) { const normalizedExisting: HistoryItemStatus = existing.status ?? "active" - if (item.status !== normalizedExisting) { + if (incoming.status !== normalizedExisting) { try { - assertValidTransition(existing.status, item.status) + assertValidTransition(existing.status, incoming.status) } catch (cacheError) { // Cache may be stale from a peer write. Re-read disk // under the store lock before rejecting the transition. - const diskItem = await this.readTaskFile(item.id) + const diskItem = await this.readTaskFile(incoming.id) if (!diskItem) { throw cacheError } - assertValidTransition(diskItem.status, item.status) + assertValidTransition(diskItem.status, incoming.status) } } } - // Merge: preserve existing metadata unless explicitly overwritten - const merged = existing ? { ...existing, ...item } : item + // Merge: preserve existing metadata unless explicitly overwritten. + // A preserve-lineage write also owns no cached lineage: the merged + // payload keeps no links, so a missing or unparseable disk record is + // written without them instead of resurrecting stale cached links. + const merged = existing && !options.preserveLineage ? { ...existing, ...incoming } : incoming - const delta = existing ? this.buildDelta(item.id, existing, item) : { ...item } + const delta = existing ? this.buildDelta(incoming.id, existing, incoming) : { ...incoming } let written: HistoryItem try { written = await this.writeTaskFile(merged, delta) } catch (error) { if (error instanceof DeltaRejectedError) { - const diskItem = await this.readTaskFile(item.id) + const diskItem = await this.readTaskFile(incoming.id) if (diskItem) { - this.cache.set(item.id, diskItem) + this.cache.set(incoming.id, diskItem) } throw error } diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index 5ad91cb52a..6d338e0124 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -743,4 +743,124 @@ describe("TaskHistoryStore real cross-host locking", () => { await fs.rm(storagePath, { recursive: true, force: true }) } }) + + describe("upsertPreservingLineage", () => { + const detachedItem = (id: string): HistoryItem => { + const record = { ...item(id), status: "interrupted" as const } + delete (record as Partial).parentTaskId + delete (record as Partial).rootTaskId + return record + } + + it("keeps a linked record linked through a stale message save", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "preserve-lineage-linked-")) + const store = new TaskHistoryStore(storagePath) + const filePath = path.join(storagePath, "tasks", "child", "history_item.json") + + try { + await store.initialize() + await store.upsert({ + ...item("child"), + status: "interrupted", + parentTaskId: "parent", + rootTaskId: "root", + }) + + // A rehydrated task saves messages without holding lineage + // authority, and another host has refreshed nothing here. + const staleSave: HistoryItem = { + ...item("child"), + status: "interrupted", + task: "Updated", + tokensOut: 7, + } + await expect(store.upsertPreservingLineage(staleSave)).resolves.toBeDefined() + + const persisted = JSON.parse(await fs.readFile(filePath, "utf8")) + expect(persisted.parentTaskId).toBe("parent") + expect(persisted.rootTaskId).toBe("root") + expect(persisted.task).toBe("Updated") + expect(store.get("child")).toMatchObject({ parentTaskId: "parent", rootTaskId: "root" }) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("keeps a detached record detached when the local cache still holds stale links", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "preserve-lineage-detached-")) + const store = new TaskHistoryStore(storagePath) + const filePath = path.join(storagePath, "tasks", "child", "history_item.json") + + try { + await store.initialize() + await store.upsert({ + ...item("child"), + status: "interrupted", + parentTaskId: "parent", + rootTaskId: "root", + }) + + // Another host detaches the record directly on disk; this + // host's cache still shows the removed links. + await fs.writeFile(filePath, JSON.stringify(detachedItem("child")), "utf8") + + const staleSave: HistoryItem = { + ...item("child"), + status: "interrupted", + task: "Updated", + parentTaskId: "parent", + rootTaskId: "root", + } + await store.upsertPreservingLineage(staleSave) + + const persisted = JSON.parse(await fs.readFile(filePath, "utf8")) + expect(persisted).not.toHaveProperty("parentTaskId") + expect(persisted).not.toHaveProperty("rootTaskId") + expect(persisted.task).toBe("Updated") + expect(store.get("child")).not.toHaveProperty("parentTaskId") + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("recreates a deleted record without lineage when a stale save writes it", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "preserve-lineage-deleted-")) + const store = new TaskHistoryStore(storagePath) + const filePath = path.join(storagePath, "tasks", "child", "history_item.json") + + try { + await store.initialize() + await store.upsert({ + ...item("child"), + status: "interrupted", + parentTaskId: "parent", + rootTaskId: "root", + }) + + // Another host removed the whole task record; the local + // cache still holds the linked item. + await fs.rm(path.dirname(filePath), { recursive: true, force: true }) + + const staleSave: HistoryItem = { + ...item("child"), + status: "interrupted", + task: "Updated", + parentTaskId: "parent", + rootTaskId: "root", + } + await store.upsertPreservingLineage(staleSave) + + const persisted = JSON.parse(await fs.readFile(filePath, "utf8")) + expect(persisted).not.toHaveProperty("parentTaskId") + expect(persisted).not.toHaveProperty("rootTaskId") + expect(persisted.status).toBe("interrupted") + expect(store.get("child")).not.toHaveProperty("parentTaskId") + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + }) }) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 40a4794c62..e45186dcd6 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1799,13 +1799,17 @@ export class Task extends EventEmitter implements TaskLike { const provider = this.providerRef.deref() const existingStatus = provider?.taskHistoryStore.get(this.taskId)?.status + const persistedHistoryItem = existingStatus ? { ...historyItem, status: existingStatus } : historyItem if (this.initialStatus === "interrupted") { // A rehydrated task's links may have been severed by another host. - // Only lifecycle operations own lineage after interruption, not message saves. - delete historyItem.parentTaskId - delete historyItem.rootTaskId + // Only lifecycle operations own lineage after interruption, not + // message saves. The in-memory links stay out of the write: the + // store preserves the on-disk lineage and never resurrects the + // stale links, even when the disk record was deleted. + await provider?.updateTaskHistory(persistedHistoryItem, { preserveLineage: true }) + } else { + await provider?.updateTaskHistory(persistedHistoryItem) } - await provider?.updateTaskHistory(existingStatus ? { ...historyItem, status: existingStatus } : historyItem) } catch (error) { // The message array was persisted above; a metadata or task-history failure must // not mask that write (see the method docs). The next saveClineMessages() call diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index 3682321ef2..cfa98439ff 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -1103,6 +1103,8 @@ describe("Task persistence", () => { status: "completed", tokensIn: 10, }), + // The interrupted rehydrated task declares its lineage non-authoritative. + { preserveLineage: true }, ) }) }) @@ -1308,10 +1310,11 @@ describe("Task persistence", () => { await task["saveClineMessages"]() await task["saveClineMessages"]() expect(loop).not.toHaveBeenCalled() - expect(mockProvider.updateTaskHistory).toHaveBeenCalled() - for (const [saved] of vi.mocked(mockProvider.updateTaskHistory).mock.calls) { - expect(saved).not.toHaveProperty("parentTaskId") - expect(saved).not.toHaveProperty("rootTaskId") + expect(mockProvider.updateTaskHistory).toHaveBeenCalledTimes(2) + for (const [saved, options] of vi.mocked(mockProvider.updateTaskHistory).mock.calls) { + // The stale rehydrated links must be declared non-authoritative so + // the store keeps the persisted lineage and never restores these. + expect(options).toEqual({ preserveLineage: true }) expect(saved.status).toBe("interrupted") } }) diff --git a/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts b/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts new file mode 100644 index 0000000000..0cbcc8bbfb --- /dev/null +++ b/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts @@ -0,0 +1,284 @@ +// cd src && npx vitest run core/task/__tests__/Task.saveClineMessages.lineage.spec.ts + +import * as fs from "fs/promises" +import * as os from "os" +import * as path from "path" + +import * as vscode from "vscode" + +import type { ClineMessage, HistoryItem } from "@roo-code/types" +import { TelemetryService } from "@roo-code/telemetry" + +import { Task } from "../Task" +import { ClineProvider } from "../../webview/ClineProvider" +import { ContextProxy } from "../../config/ContextProxy" +import { WebviewFocusTracker } from "../../webview/WebviewFocusTracker" +import { providerIdentifiers } from "@roo-code/types/provider-identifiers" + +// Only the message persistence helpers are mocked. The store, the metadata +// builder, the provider routing, and the per-task file writes all run for real. +vi.mock("../../task-persistence", async (importOriginal) => { + const mod = await importOriginal() + return { + ...mod, + saveApiMessages: vi.fn().mockResolvedValue(undefined), + saveTaskMessages: vi.fn().mockResolvedValue(undefined), + readApiMessages: vi.fn().mockResolvedValue([]), + readTaskMessages: vi.fn().mockResolvedValue([]), + } +}) + +vi.mock("delay", () => ({ + __esModule: true, + default: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock("execa", () => ({ + execa: vi.fn(), +})) + +vi.mock("p-wait-for", () => ({ + default: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock("vscode", () => { + const mockDisposable = { dispose: vi.fn() } + const mockEventEmitter = { event: vi.fn(), fire: vi.fn() } + const mockTextDocument = { uri: { fsPath: "/mock/workspace/path/file.ts" } } + const mockTextEditor = { document: mockTextDocument } + const mockTab = { input: { uri: { fsPath: "/mock/workspace/path/file.ts" } } } + const mockTabGroup = { tabs: [mockTab] } + + return { + TabInputTextDiff: vi.fn(), + CodeActionKind: { + QuickFix: { value: "quickfix" }, + RefactorRewrite: { value: "refactor.rewrite" }, + }, + window: { + createTextEditorDecorationType: vi.fn().mockReturnValue({ dispose: vi.fn() }), + visibleTextEditors: [mockTextEditor], + tabGroups: { + all: [mockTabGroup], + close: vi.fn(), + onDidChangeTabs: vi.fn(() => ({ dispose: vi.fn() })), + }, + showErrorMessage: vi.fn(), + }, + workspace: { + workspaceFolders: [ + { + uri: { fsPath: "/mock/workspace/path" }, + name: "mock-workspace", + index: 0, + }, + ], + createFileSystemWatcher: vi.fn(() => ({ + onDidCreate: vi.fn(() => mockDisposable), + onDidDelete: vi.fn(() => mockDisposable), + onDidChange: vi.fn(() => mockDisposable), + dispose: vi.fn(), + })), + fs: { + stat: vi.fn().mockResolvedValue({ type: 1 }), + }, + onDidSaveTextDocument: vi.fn(() => mockDisposable), + getConfiguration: vi.fn(() => ({ get: (_key: string, defaultValue: unknown) => defaultValue })), + }, + env: { + uriScheme: "vscode", + language: "en", + }, + EventEmitter: vi.fn().mockImplementation(function () { + return mockEventEmitter + }), + Disposable: { + from: vi.fn(), + }, + TabInputText: vi.fn(), + } +}) + +vi.mock("../../mentions", () => ({ + parseMentions: vi.fn().mockImplementation((text: string) => { + return Promise.resolve({ text, mode: undefined, contentBlocks: [] }) + }), + openMention: vi.fn(), + getLatestTerminalOutput: vi.fn(), +})) + +vi.mock("../../../integrations/misc/extract-text", () => ({ + extractTextFromFile: vi.fn().mockResolvedValue("Mock file content"), +})) + +vi.mock("../../environment/getEnvironmentDetails", () => ({ + getEnvironmentDetails: vi.fn().mockResolvedValue(""), +})) + +vi.mock("../../ignore/RooIgnoreController") + +vi.mock("../../condense", () => ({ + CondenseContext: vi.fn(), + summarizeTask: vi.fn().mockResolvedValue({ success: false }), +})) + +type SaveClineMessagesAccess = { saveClineMessages: () => Promise } + +function makeLinkedInterruptedHistory(id: string): HistoryItem { + return { + id, + number: 1, + ts: 1, + task: "Delegated child", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + workspace: "/mock/workspace/path", + status: "interrupted", + parentTaskId: "parent-1", + rootTaskId: "root-1", + } +} + +function makeMessages(): ClineMessage[] { + return [{ ts: 1, type: "say", say: "text", text: "Rehydrated child progress" }] +} + +async function readPersistedRecord(storagePath: string, taskId: string): Promise> { + const filePath = path.join(storagePath, "tasks", taskId, "history_item.json") + return JSON.parse(await fs.readFile(filePath, "utf8")) +} + +describe("Task.saveClineMessages lineage ownership for rehydrated interrupted tasks", () => { + let storagePath: string + let provider: ClineProvider & Record + + beforeEach(async () => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + + storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-save-lineage-")) + + const mockExtensionContext = { + globalState: { + get: vi.fn().mockReturnValue(undefined), + update: vi.fn().mockResolvedValue(undefined), + keys: vi.fn().mockReturnValue([]), + }, + globalStorageUri: { fsPath: storagePath }, + workspaceState: { + get: vi.fn().mockReturnValue(undefined), + update: vi.fn().mockResolvedValue(undefined), + keys: vi.fn().mockReturnValue([]), + }, + secrets: { + get: vi.fn().mockResolvedValue(undefined), + store: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + }, + extensionUri: { fsPath: "/mock/extension/path" }, + extension: { packageJSON: { version: "1.0.0" } }, + } as unknown as vscode.ExtensionContext + + const mockOutputChannel = { + appendLine: vi.fn(), + append: vi.fn(), + clear: vi.fn(), + show: vi.fn(), + hide: vi.fn(), + dispose: vi.fn(), + } as unknown as vscode.OutputChannel + + provider = new ClineProvider( + mockExtensionContext, + mockOutputChannel, + "sidebar", + new ContextProxy(mockExtensionContext), + new WebviewFocusTracker(), + ) as ClineProvider & Record + + provider.postMessageToWebview = vi.fn().mockResolvedValue(undefined) + provider.postStateToWebview = vi.fn().mockResolvedValue(undefined) + provider.log = vi.fn() + + await provider.taskHistoryStore.initialize() + }) + + afterEach(async () => { + provider.taskHistoryStore.dispose() + await fs.rm(storagePath, { recursive: true, force: true }).catch(() => {}) + }) + + async function createRehydratedTask(history: HistoryItem): Promise { + return new Task({ + provider, + apiConfiguration: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-3-5-sonnet-20241022", + apiKey: "test-api-key", + }, + historyItem: history, + startTask: false, + }) + } + + it("recreates a record deleted by another host without stale lineage", async () => { + const history = makeLinkedInterruptedHistory("detached-child") + await provider.taskHistoryStore.upsert(history) + + // The local cache stays linked. Another host removes the record. + await fs.rm(path.join(storagePath, "tasks", "detached-child"), { recursive: true, force: true }) + expect(provider.taskHistoryStore.get("detached-child")?.parentTaskId).toBe("parent-1") + + const task = await createRehydratedTask(history) + await (task as unknown as SaveClineMessagesAccess).saveClineMessages() + + const persisted = await readPersistedRecord(storagePath, "detached-child") + expect(persisted).not.toHaveProperty("parentTaskId") + expect(persisted).not.toHaveProperty("rootTaskId") + expect(persisted.status).toBe("interrupted") + expect(provider.taskHistoryStore.get("detached-child")).not.toHaveProperty("parentTaskId") + }) + + it("keeps a record detached by another host after a stale save", async () => { + const history = makeLinkedInterruptedHistory("detached-child") + await provider.taskHistoryStore.upsert(history) + + // Another host detaches the authoritative record on disk only. + const detached = { ...history } as Record + delete detached.parentTaskId + delete detached.rootTaskId + await fs.writeFile( + path.join(storagePath, "tasks", "detached-child", "history_item.json"), + JSON.stringify(detached, null, "\t"), + "utf8", + ) + expect(provider.taskHistoryStore.get("detached-child")?.parentTaskId).toBe("parent-1") + + const task = await createRehydratedTask(history) + await (task as unknown as SaveClineMessagesAccess).saveClineMessages() + + const persisted = await readPersistedRecord(storagePath, "detached-child") + expect(persisted).not.toHaveProperty("parentTaskId") + expect(persisted).not.toHaveProperty("rootTaskId") + expect(provider.taskHistoryStore.get("detached-child")).not.toHaveProperty("parentTaskId") + }) + + it("preserves valid authoritative lineage through a message save", async () => { + const history = makeLinkedInterruptedHistory("linked-child") + await provider.taskHistoryStore.upsert(history) + + const task = await createRehydratedTask(history) + await (task as unknown as SaveClineMessagesAccess).saveClineMessages() + + const persisted = await readPersistedRecord(storagePath, "linked-child") + expect(persisted.parentTaskId).toBe("parent-1") + expect(persisted.rootTaskId).toBe("root-1") + expect(persisted.status).toBe("interrupted") + expect(provider.taskHistoryStore.get("linked-child")).toMatchObject({ + parentTaskId: "parent-1", + rootTaskId: "root-1", + }) + }) +}) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 2dadf4bf68..52402edbc9 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -633,6 +633,14 @@ export class ClineProvider `[cleanupFailedHistoryTask] dispose() failed for ${task.taskId}.${task.instanceId}: ${error instanceof Error ? error.message : String(error)}`, ) } + + if (error instanceof LifecycleTransitionError) { + void vscode.window.showErrorMessage(error.message) + } + + // The registry already dropped the task. Post fresh state so the webview + // stops showing the removed task after a rejected resume or settlement. + await this.postStateToWebview() } /** @@ -3121,12 +3129,20 @@ export class ClineProvider * * @param item The history item to update or add * @param options.broadcast Whether to broadcast the updated history to the webview (default: true) + * @param options.preserveLineage When true, the item's parent and root links are not + * authoritative: the on-disk lineage is kept as is and stale in-memory links are never + * written or resurrected. Use only for message saves from rehydrated interrupted tasks. * @returns The updated task history array */ - async updateTaskHistory(item: HistoryItem, options: { broadcast?: boolean } = {}): Promise { - const { broadcast = true } = options - - const history = await this.taskHistoryStore.upsert(item) + async updateTaskHistory( + item: HistoryItem, + options: { broadcast?: boolean; preserveLineage?: boolean } = {}, + ): Promise { + const { broadcast = true, preserveLineage = false } = options + + const history = preserveLineage + ? await this.taskHistoryStore.upsertPreservingLineage(item) + : await this.taskHistoryStore.upsert(item) this.recentTasksCache = undefined // Broadcast the updated history to the webview if requested. From 6791f56299561bf6469ec2b46c5c251bc2f9167a Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 14 Oct 2026 00:59:33 +0000 Subject: [PATCH 07/14] fix: prepare interrupted resumes before activation --- docs/architecture/task-lifecycle-model.md | 2 +- scripts/check-task-store-concurrency.ts | 28 ++++++- .../ClineProvider.delegation.spec.ts | 56 ++++++++++--- .../removeClineFromStack-delegation.spec.ts | 1 + src/core/task/Task.ts | 32 +++++-- .../task/__tests__/Task.persistence.spec.ts | 84 +++++++++++++++++++ .../Task.saveClineMessages.lineage.spec.ts | 18 +++- src/core/webview/ClineProvider.ts | 11 ++- 8 files changed, 202 insertions(+), 30 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 23d1fda7bb..66ae466d61 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -62,7 +62,7 @@ Production completion also accepts a recovery-compatible `active` parent that st Explicit resume validates both records under file locks. A linked child resume holds the parent file's advisory lock across the parent ownership check and the child activation, so a competing host cannot re-delegate the parent in between. The re-delegation write (`atomicUpdateParentDelegation`) re-reads the awaited child's on-disk record inside the parent-file lock and rejects the delegation unless that child is still interrupted. The two guards share one lock order, parent file before child file, so they close the competing resume and re-delegation interleavings in both directions. The child-file merge still compares the caller's expected parent with the authoritative child backlink. The resume rejects an abandonment that committed before it, even in its partial-write window. -Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted non-interrupted resume still runs without a store transition. +Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A denied pending-action replay activates the interrupted record through the same locked path before its loop starts. An accepted non-interrupted resume still runs without a store transition. Rehydrated interrupted tasks omit lineage from message metadata saves, so their stale construction snapshots cannot restore severed links after rejection. Reverse-order stale abandonment and other live-task writers remain within the ownership/generation gaps below. diff --git a/scripts/check-task-store-concurrency.ts b/scripts/check-task-store-concurrency.ts index 68dd1becb1..1f9f8d2d0d 100644 --- a/scripts/check-task-store-concurrency.ts +++ b/scripts/check-task-store-concurrency.ts @@ -114,6 +114,22 @@ const semanticLandmarks = { state.commits.filter((entry) => entry.operationId === "complete-a").length === 1 && state.caches.A["child-a"]?.status === "completed" && state.caches.A.parent?.status === "delegated", + "preserve-save-commits-after-detachment": (state: ModelState) => { + const detachIndex = state.commits.findIndex( + (entry) => + entry.operationId === "abandon-b" && + entry.taskId === "child-a" && + entry.next.parentTaskId === undefined && + entry.next.rootTaskId === undefined, + ) + if (detachIndex < 0) return false + const preserveIndex = state.commits.findIndex( + (entry) => entry.operationId === "preserve-save-a" && entry.taskId === "child-a", + ) + if (preserveIndex <= detachIndex) return false + const disk = state.disk["child-a"] + return Boolean(disk && disk.parentTaskId === undefined && disk.rootTaskId === undefined) + }, } satisfies Record boolean> function item(id: TaskId, overrides: Partial = {}): HistoryItem { @@ -281,8 +297,11 @@ function prepareWrites(state: ModelState, spec: OperationSpec, operation: Operat const cached = state.caches[spec.host][taskId] // Task.saveClineMessages rebuilds lineage from the live Task but preserves the // store's current status before upsert, so stale lineage is not accompanied by - // a stale status transition. - const incoming = spec.id === "stale-save-a" && cached?.status ? { ...built, status: cached.status } : built + // a stale status transition. The preserve-lineage save follows the same rule, + // so a save that commits after a peer's detachment never models a stale + // interrupted → active delta that the real store would never send. + const preservesStoreStatus = spec.id === "stale-save-a" || spec.id === "preserve-save-a" + const incoming = preservesStoreStatus && cached?.status ? { ...built, status: cached.status } : built return { taskId, incoming, @@ -724,6 +743,11 @@ const scenarios: Scenario[] = [ return ["preserve-lineage save left reattached lineage after abandonment"] } } + // The production save carries the store's current status, so it + // never models a status delta and is never rejected. + if (state.operations["preserve-save-a"]?.phase === "rejected") { + return ["preserve-lineage save was rejected for a stale status delta"] + } return [] }, }, diff --git a/src/__tests__/ClineProvider.delegation.spec.ts b/src/__tests__/ClineProvider.delegation.spec.ts index 1e3eacf7e6..60d530104a 100644 --- a/src/__tests__/ClineProvider.delegation.spec.ts +++ b/src/__tests__/ClineProvider.delegation.spec.ts @@ -108,8 +108,9 @@ describe("ClineProvider.resumeInterruptedTask()", () => { await storeB.initialize() const resume = storeA.resumeInterruptedTask.bind(storeA) vi.spyOn(storeA, "resumeInterruptedTask").mockImplementationOnce(async (id, expectedParent) => { - // This hook runs after the provider's parent check. Commit only the - // first half of abandonment: the parent still appears to await this child. + // This hook runs after the provider's cache refresh but before the + // store's locked ownership check. Commit only the first half of + // abandonment: the parent still appears to await this child. await storeB.atomicReadAndUpdate("child", (current) => abandonDelegatedChild(parent, current).child) return resume(id, expectedParent) }) @@ -130,17 +131,50 @@ describe("ClineProvider.resumeInterruptedTask()", () => { }) it("rejects a stale interrupted child after its parent delegates elsewhere", async () => { - const taskHistoryStore = { - invalidate: vi.fn().mockResolvedValue(undefined), - get: vi.fn().mockReturnValue({ id: "parent-1", status: "delegated", awaitingChildId: "child-2" }), - resumeInterruptedTask: vi.fn(), + // The provider does not gate on an unlocked cache read. The store's + // parent-file-locked check rejects the stale child, so this flow uses + // a real store with a stale parent on disk. + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-stale-")) + const store = new TaskHistoryStore(storagePath) + const parent: HistoryItem = { + id: "parent", + number: 1, + ts: 1, + task: "Parent", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: "child-2", + delegatedToId: "child-2", + childIds: ["child", "child-2"], } - const provider = makeProviderStub({ taskHistoryStore, isViewLaunched: false }) + const child: HistoryItem = { + ...parent, + id: "child", + status: "interrupted", + parentTaskId: "parent", + rootTaskId: "parent", + awaitingChildId: undefined, + delegatedToId: undefined, + childIds: [], + } + try { + await store.initialize() + await store.upsert(parent) + await store.upsert(child) + const provider = makeProviderStub({ taskHistoryStore: store, isViewLaunched: false }) - await expect( - ClineProvider.prototype.resumeInterruptedTask.call(provider, "child-1", "parent-1"), - ).rejects.toThrow("parent parent-1 no longer awaits it") - expect(taskHistoryStore.resumeInterruptedTask).not.toHaveBeenCalled() + await expect( + ClineProvider.prototype.resumeInterruptedTask.call(provider, "child", "parent"), + ).rejects.toThrow("parent parent no longer awaits it") + + // The stale child record stays untouched for recovery or abandonment. + expect(store.get("child")).toMatchObject({ status: "interrupted", parentTaskId: "parent" }) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } }) it("delegates a resumed interrupted child to a grandchild through the real provider flow", async () => { diff --git a/src/__tests__/removeClineFromStack-delegation.spec.ts b/src/__tests__/removeClineFromStack-delegation.spec.ts index d5fb5da886..6b562a90fa 100644 --- a/src/__tests__/removeClineFromStack-delegation.spec.ts +++ b/src/__tests__/removeClineFromStack-delegation.spec.ts @@ -1,6 +1,7 @@ // npx vitest run __tests__/removeClineFromStack-delegation.spec.ts import { describe, it, expect, vi } from "vitest" +import * as vscode from "vscode" import { ClineProvider } from "../core/webview/ClineProvider" import { TaskRegistry } from "../core/task/TaskRegistry" import { PendingActionSettlementError, type Task } from "../core/task/Task" diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index e45186dcd6..ecef4016e7 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3239,13 +3239,6 @@ export class Task extends EventEmitter implements TaskLike { if (this.abort || this.abandoned || (response !== "yesButtonClicked" && response !== "messageResponse")) { return } - if (this.initialStatus === "interrupted") { - const provider = this.providerRef.deref() - if (!provider) { - throw new Error(`[Task#resumeTaskFromHistory] Provider unavailable for task ${this.taskId}`) - } - await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) - } let responseText: string | undefined let responseImages: string[] | undefined @@ -3257,7 +3250,9 @@ export class Task extends EventEmitter implements TaskLike { } // Make sure that the api conversation history can be resumed by the API, - // even if it goes out of sync with cline messages. + // even if it goes out of sync with cline messages. This read and the + // derivation below are fallible. They run before the activation + // commit, so a failure leaves the record interrupted and retryable. const existingApiConversationHistory: ApiMessage[] = await this.getSavedApiConversationHistory() // Tool blocks are always preserved; native tool calling only. @@ -3388,6 +3383,20 @@ export class Task extends EventEmitter implements TaskLike { await this.overwriteApiConversationHistory(modifiedApiConversationHistory) + // All fallible resume preparation is complete. Activate the + // interrupted record last, so a preparation failure never consumes + // the one interrupted → active resume. + if (this.abort || this.abandoned) { + return + } + if (this.initialStatus === "interrupted") { + const provider = this.providerRef.deref() + if (!provider) { + throw new Error(`[Task#resumeTaskFromHistory] Provider unavailable for task ${this.taskId}`) + } + await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + } + // Task resuming from history item. await this.initiateTaskLoop(newUserContent) } catch (error) { @@ -3452,6 +3461,13 @@ export class Task extends EventEmitter implements TaskLike { await this.say("user_feedback", text ?? "", images) } + // A denied pending action still starts the task loop. Activate the + // interrupted record first, so the persisted status matches the running + // task. Approval paths above return early and run their own transitions. + if (this.initialStatus === "interrupted") { + await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + } + const deniedContent = text ? formatResponse.toolDeniedWithFeedback(text) : formatResponse.toolDenied() await this.initiateTaskLoop([ { diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index cfa98439ff..ff9df8b976 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -1278,6 +1278,10 @@ describe("Task persistence", () => { }) it("does not restore stale lineage through repeated message saves after rejected resume", async () => { + // The accepted resume completes its fallible api history preparation + // before the activation commit, so the harness must serve a readable + // api history for the rejection to come from the activation path. + mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) const history = { id: "detached", number: 1, @@ -1319,6 +1323,31 @@ describe("Task persistence", () => { } }) + it("keeps the interrupted record retryable when the api history read fails after acceptance", async () => { + // The first read hydrates the task. The second read runs after the + // user accepts, and the activation commit must wait for it. A failed + // read then leaves the record interrupted, so the user can retry. + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockRejectedValueOnce(new Error("api history read failed")) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + startTask: false, + initialStatus: "interrupted", + }) + mockProvider.taskHistoryStore.get = vi.fn().mockReturnValue({ id: task.taskId, status: "interrupted" }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("api history read failed") + + expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() + expect(mockProvider.updateTaskHistory).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + }) + it("resumes a Task.create history task only once when run is also called", async () => { mockReadTaskMessages.mockResolvedValue([]) mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) @@ -1566,6 +1595,61 @@ describe("Task persistence", () => { expect(mockSaveTaskMessages).not.toHaveBeenCalled() }) + it("activates an interrupted child before a denied pending-action replay starts the loop", async () => { + // A denied finish action still feeds the task loop. The record must + // leave interrupted through the authoritative resume before that + // loop starts, exactly as a generic accepted resume does. + mockReadTaskMessages.mockResolvedValue([ + { ts: 1, type: "say", say: "text", text: "Child" }, + { ts: 2, type: "ask", ask: "tool", text: pendingAction.approvalText }, + ]) + mockReadApiMessages.mockResolvedValue([ + { + role: "assistant", + content: [{ type: "tool_use", id: "finish-action", name: "attempt_completion", input: {} }], + }, + ]) + // Restart settlement refreshes the record and re-reads the pending + // action from it, so the refreshed record must still carry the action. + mockProvider.taskHistoryStore.reconcile = vi.fn().mockResolvedValue(undefined) + mockProvider.taskHistoryStore.get = vi.fn().mockReturnValue({ + id: "child-1", + status: "interrupted", + pendingAction, + }) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + historyItem: { + id: "child-1", + number: 1, + ts: 1, + task: "Child", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "interrupted", + parentTaskId: "parent-1", + pendingAction, + }, + startTask: false, + initialStatus: "interrupted", + }) + const activate = vi.fn<(taskId: string, parentTaskId?: string) => Promise>() + activate.mockResolvedValue(undefined) + mockProvider.resumeInterruptedTask = activate + vi.spyOn(task, "ask").mockResolvedValue({ response: "noButtonClicked" }) + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + + await getTaskPersistenceAccess(task).resumeTaskFromHistory() + + expect(task.ask).toHaveBeenCalledWith("tool", pendingAction.approvalText, false) + expect(activate).toHaveBeenCalledOnce() + expect(activate).toHaveBeenCalledWith("child-1", "parent-1") + expect(loop).toHaveBeenCalledOnce() + expect(activate.mock.invocationCallOrder[0]).toBeLessThan(loop.mock.invocationCallOrder[0]) + }) + it("replays a create-subtask action for an active historical task without restart settlement", async () => { mockReadTaskMessages.mockResolvedValue([ { ts: 1, type: "ask", ask: "tool", text: createSubtaskAction.approvalText }, diff --git a/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts b/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts index 0cbcc8bbfb..9d85fc4de7 100644 --- a/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts +++ b/src/core/task/__tests__/Task.saveClineMessages.lineage.spec.ts @@ -141,7 +141,12 @@ function makeLinkedInterruptedHistory(id: string): HistoryItem { } function makeMessages(): ClineMessage[] { - return [{ ts: 1, type: "say", say: "text", text: "Rehydrated child progress" }] + // The first rehydrated message is the original task say. Its text feeds + // the metadata title, so it must match the persisted child title. + return [ + { ts: 1, type: "say", say: "text", text: "Delegated child" }, + { ts: 2, type: "say", say: "text", text: "Rehydrated child progress" }, + ] } async function readPersistedRecord(storagePath: string, taskId: string): Promise> { @@ -211,7 +216,7 @@ describe("Task.saveClineMessages lineage ownership for rehydrated interrupted ta }) async function createRehydratedTask(history: HistoryItem): Promise { - return new Task({ + const task = new Task({ provider, apiConfiguration: { apiProvider: providerIdentifiers.anthropic, @@ -221,6 +226,11 @@ describe("Task.saveClineMessages lineage ownership for rehydrated interrupted ta historyItem: history, startTask: false, }) + // Hydrate the rehydrated messages without persisting them. The tested + // save then matches the production resume flow, where cline messages + // are hydrated before the first metadata save. + task.clineMessages = makeMessages() + return task } it("recreates a record deleted by another host without stale lineage", async () => { @@ -238,6 +248,7 @@ describe("Task.saveClineMessages lineage ownership for rehydrated interrupted ta expect(persisted).not.toHaveProperty("parentTaskId") expect(persisted).not.toHaveProperty("rootTaskId") expect(persisted.status).toBe("interrupted") + expect(persisted.task).toBe("Delegated child") expect(provider.taskHistoryStore.get("detached-child")).not.toHaveProperty("parentTaskId") }) @@ -262,6 +273,8 @@ describe("Task.saveClineMessages lineage ownership for rehydrated interrupted ta const persisted = await readPersistedRecord(storagePath, "detached-child") expect(persisted).not.toHaveProperty("parentTaskId") expect(persisted).not.toHaveProperty("rootTaskId") + expect(persisted.status).toBe("interrupted") + expect(persisted.task).toBe("Delegated child") expect(provider.taskHistoryStore.get("detached-child")).not.toHaveProperty("parentTaskId") }) @@ -276,6 +289,7 @@ describe("Task.saveClineMessages lineage ownership for rehydrated interrupted ta expect(persisted.parentTaskId).toBe("parent-1") expect(persisted.rootTaskId).toBe("root-1") expect(persisted.status).toBe("interrupted") + expect(persisted.task).toBe("Delegated child") expect(provider.taskHistoryStore.get("linked-child")).toMatchObject({ parentTaskId: "parent-1", rootTaskId: "root-1", diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 52402edbc9..ae56292a1a 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -776,13 +776,12 @@ export class ClineProvider public async resumeInterruptedTask(taskId: string, parentTaskId?: string): Promise { const resume = async () => { if (parentTaskId) { + // A peer write holds the parent file during its rename. The + // unlocked cache read can then miss a valid parent and reject a + // resume the user already accepted. Do not gate here. The store + // re-reads the parent under the parent file lock, and that + // locked check alone decides the resume. await this.taskHistoryStore.invalidate(parentTaskId) - const parent = this.taskHistoryStore.get(parentTaskId) - if (parent?.status !== "delegated" || parent.awaitingChildId !== taskId) { - throw new LifecycleTransitionError( - `Cannot resume task ${taskId}: parent ${parentTaskId} no longer awaits it`, - ) - } } const resumed = await this.taskHistoryStore.resumeInterruptedTask(taskId, parentTaskId) From 2c97609fb4cce46c43ddd7c9a36bb918e696b65c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 16 Oct 2026 03:20:55 +0000 Subject: [PATCH 08/14] fix: preserve api history when interrupted resume fails --- docs/architecture/task-lifecycle-model.md | 2 +- src/core/task-persistence/TaskHistoryStore.ts | 24 + .../__tests__/TaskHistoryStore.spec.ts | 45 ++ .../__tests__/apiMessages.spec.ts | 230 +++++++- src/core/task-persistence/apiMessages.ts | 180 +++++- src/core/task-persistence/index.ts | 7 + src/core/task/Task.ts | 155 +++++- ...Task.ignored-disabled-tools-notice.spec.ts | 12 +- .../task/__tests__/Task.persistence.spec.ts | 244 ++++++++ .../Task.resumeApiHistoryPersistence.spec.ts | 521 ++++++++++++++++++ src/core/task/__tests__/Task.spec.ts | 10 +- 11 files changed, 1411 insertions(+), 19 deletions(-) create mode 100644 src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 66ae466d61..4868504c95 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -62,7 +62,7 @@ Production completion also accepts a recovery-compatible `active` parent that st Explicit resume validates both records under file locks. A linked child resume holds the parent file's advisory lock across the parent ownership check and the child activation, so a competing host cannot re-delegate the parent in between. The re-delegation write (`atomicUpdateParentDelegation`) re-reads the awaited child's on-disk record inside the parent-file lock and rejects the delegation unless that child is still interrupted. The two guards share one lock order, parent file before child file, so they close the competing resume and re-delegation interleavings in both directions. The child-file merge still compares the caller's expected parent with the authoritative child backlink. The resume rejects an abandonment that committed before it, even in its partial-write window. -Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A denied pending-action replay activates the interrupted record through the same locked path before its loop starts. An accepted non-interrupted resume still runs without a store transition. +Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A failed preparation write also stops the resume, and the Task restores the authoritative API conversation history in memory. Before its destructive preparation write the resume first stores a durable pre-resume recovery copy of the original history in the task directory under a UUID-distinct name, so a failed preparation or restore cannot lose the original across a restart. When the activation commit rejects, the Task restores the persisted history through a conditional write whose status gate, content compare, and commit all run under the API-history file's advisory lock; the gate reads the task record under the record file's lock inside that scope, and no existing writer takes the reverse lock order. The restore commits only while the persisted record still shows the status the resume started from and the disk still holds the prepared slice, so an active record and a newer foreign history always win. An unreadable record status or an unreadable history fails the restore closed, and the restore never runs on unknown state. A record that is already active on disk keeps the prepared history, and the skipped restore is logged as a partial failure with the now-obsolete recovery copy removed. A restore write that fails logs the same partial failure and names the recovery copy that still holds the original bytes. These guarantees bind writers that honor the shared advisory locks, which covers every in-repo writer of both files. A denied pending-action replay activates the interrupted record through the same locked path before its loop starts. An accepted non-interrupted resume still runs without a store transition. Rehydrated interrupted tasks omit lineage from message metadata saves, so their stale construction snapshots cannot restore severed links after rejection. Reverse-order stale abandonment and other live-task writers remain within the ownership/generation gaps below. diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index cc2903e685..b18a19816a 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -833,6 +833,30 @@ export class TaskHistoryStore { }) } + /** + * Read one task's lifecycle status from its validated on-disk record inside + * the record file's advisory lock. Throws `LifecycleTransitionError` when + * the record is missing, malformed, or persisted under a different task ID, + * so callers can fail closed instead of restoring over unknown disk state. + * The cache is not consulted and not refreshed. Callers must not already + * hold this record file's advisory lock. + */ + public async readValidatedDiskStatus(taskId: string): Promise { + const filePath = await this.getTaskFilePath(taskId) + // A missing file cannot be locked, so the existence check runs first + // and maps the missing record to the same typed failure as the + // validated read below. + try { + await fs.stat(filePath) + } catch { + throw new LifecycleTransitionError(`[TaskHistoryStore] task ${taskId} has no valid disk record`) + } + return withFileLock(filePath, async () => { + const item = await this.readValidatedTaskFile(taskId) + return item.status ?? "active" + }) + } + // ────────────────────────────── Migration ────────────────────────────── /** diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.spec.ts index 3e277ac867..8ba0686b72 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.spec.ts @@ -577,6 +577,51 @@ describe("TaskHistoryStore", () => { }) }) + describe("readValidatedDiskStatus()", () => { + it("returns the validated disk status instead of the cache value", async () => { + const item = makeHistoryItem({ id: "status-task", status: "interrupted" }) + await store.upsert(item) + + // A competing host moved the record on disk; the cache still holds + // the interrupted snapshot. + const filePath = path.join(tmpDir, "tasks", "status-task", GlobalFileNames.historyItem) + await fs.writeFile(filePath, JSON.stringify({ ...item, status: "active" })) + + await expect(store.readValidatedDiskStatus("status-task")).resolves.toBe("active") + }) + + it("normalizes a legacy record without a status to active", async () => { + const item = makeHistoryItem({ id: "status-legacy" }) + const { status: _status, ...legacyItem } = item + await store.upsert(legacyItem) + + await expect(store.readValidatedDiskStatus("status-legacy")).resolves.toBe("active") + }) + + it("throws for a missing record", async () => { + await expect(store.readValidatedDiskStatus("status-missing")).rejects.toThrow("no valid disk record") + }) + + it("throws for a malformed record", async () => { + const taskDir = path.join(tmpDir, "tasks", "status-malformed") + await fs.mkdir(taskDir, { recursive: true }) + await fs.writeFile(path.join(taskDir, GlobalFileNames.historyItem), "not json") + + await expect(store.readValidatedDiskStatus("status-malformed")).rejects.toThrow("no valid disk record") + }) + + it("throws for a record persisted under a different task id", async () => { + const taskDir = path.join(tmpDir, "tasks", "status-wrong-id") + await fs.mkdir(taskDir, { recursive: true }) + await fs.writeFile( + path.join(taskDir, GlobalFileNames.historyItem), + JSON.stringify(makeHistoryItem({ id: "status-other-id" })), + ) + + await expect(store.readValidatedDiskStatus("status-wrong-id")).rejects.toThrow("no valid disk record") + }) + }) + describe("atomicUpdatePair()", () => { it("updates both records and both files are written before lock releases", async () => { await store.initialize() diff --git a/src/core/task-persistence/__tests__/apiMessages.spec.ts b/src/core/task-persistence/__tests__/apiMessages.spec.ts index c74ef6b150..47f705cdc2 100644 --- a/src/core/task-persistence/__tests__/apiMessages.spec.ts +++ b/src/core/task-persistence/__tests__/apiMessages.spec.ts @@ -10,7 +10,14 @@ vi.mock("fs/promises", async (importOriginal) => ({ readFile: hoisted.readFileMock, })) -import { readApiMessages, saveApiMessages } from "../apiMessages" +import { + createResumeHistoryRecoveryCopy, + type ApiMessage, + readApiMessages, + removeResumeHistoryRecoveryCopy, + restoreApiMessagesIfUnchanged, + saveApiMessages, +} from "../apiMessages" let tmpBaseDir: string @@ -181,3 +188,224 @@ describe("apiMessages.saveApiMessages", () => { ]) }) }) + +describe("apiMessages.createResumeHistoryRecoveryCopy", () => { + it("writes the original history to a unique identifiable copy without mutating the caller", async () => { + const taskId = "task-recovery-copy" + const taskDir = path.join(tmpBaseDir, "tasks", taskId) + await fs.mkdir(taskDir, { recursive: true }) + const messages: ApiMessage[] = [{ role: "user", content: "original", ts: 1 }] + + const copyPath = await createResumeHistoryRecoveryCopy({ taskId, globalStoragePath: tmpBaseDir, messages }) + + expect(path.dirname(copyPath)).toBe(taskDir) + expect(path.basename(copyPath)).toMatch(/^api_conversation_history\.json\.pre-resume-/) + // The copy carries message identifiers while the caller's objects stay + // untouched, because the helper clones before it ensures identifiers. + expect(await fs.readFile(copyPath, "utf8")).toBe( + JSON.stringify([{ role: "user", content: "original", ts: 1, messageId: "legacy:1:0" }]), + ) + expect(messages[0].messageId).toBeUndefined() + }) + + it("never overwrites a peer recovery copy", async () => { + const taskId = "task-recovery-peer" + const taskDir = path.join(tmpBaseDir, "tasks", taskId) + await fs.mkdir(taskDir, { recursive: true }) + const messages: ApiMessage[] = [{ role: "user", content: "original", ts: 1, messageId: "m-1" }] + + const firstPath = await createResumeHistoryRecoveryCopy({ taskId, globalStoragePath: tmpBaseDir, messages }) + const secondPath = await createResumeHistoryRecoveryCopy({ taskId, globalStoragePath: tmpBaseDir, messages }) + + expect(firstPath).not.toBe(secondPath) + expect(await fs.readFile(firstPath, "utf8")).toBe(JSON.stringify(messages)) + expect(await fs.readFile(secondPath, "utf8")).toBe(JSON.stringify(messages)) + }) +}) + +describe("apiMessages.removeResumeHistoryRecoveryCopy", () => { + it("removes the copy and ignores an already missing copy", async () => { + const taskId = "task-recovery-remove" + const taskDir = path.join(tmpBaseDir, "tasks", taskId) + await fs.mkdir(taskDir, { recursive: true }) + const copyPath = await createResumeHistoryRecoveryCopy({ + taskId, + globalStoragePath: tmpBaseDir, + messages: [{ role: "user", content: "original", ts: 1, messageId: "m-1" }], + }) + + await removeResumeHistoryRecoveryCopy(copyPath) + await expect(fs.access(copyPath)).rejects.toMatchObject({ code: "ENOENT" }) + await expect(removeResumeHistoryRecoveryCopy(copyPath)).resolves.toBeUndefined() + }) +}) + +describe("apiMessages.restoreApiMessagesIfUnchanged", () => { + const prepared: ApiMessage[] = [ + { role: "user", content: "first", ts: 1, messageId: "m-1" }, + { role: "assistant", content: "reply", ts: 2, messageId: "m-2" }, + ] + const original: ApiMessage[] = [...prepared, { role: "user", content: "last", ts: 3, messageId: "m-3" }] + + async function seedDisk(taskId: string, content: ApiMessage[]): Promise { + const taskDir = path.join(tmpBaseDir, "tasks", taskId) + await fs.mkdir(taskDir, { recursive: true }) + const filePath = path.join(taskDir, "api_conversation_history.json") + await fs.writeFile(filePath, JSON.stringify(content), "utf8") + return filePath + } + + function proceedGate() { + return async () => "proceed" as const + } + + it("restores the replacement when the disk still holds the prepared content", async () => { + const taskId = "task-restore-ok" + const filePath = await seedDisk(taskId, prepared) + const gate = vi.fn(proceedGate()) + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: gate, + }) + + expect(result).toEqual({ outcome: "restored" }) + // The gate ran inside the lock scope, before the write. + expect(gate).toHaveBeenCalledTimes(1) + expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual(original) + }) + + it("keeps a foreign history that is written inside the lock window", async () => { + const taskId = "task-restore-foreign" + const filePath = await seedDisk(taskId, prepared) + const foreign: ApiMessage[] = [...prepared, { role: "assistant", content: "foreign", ts: 4, messageId: "m-9" }] + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + // A non-locking foreign writer lands after the gate passed and + // before the compare runs at commit time. The compare must reject. + verifyCanRestore: async () => { + await fs.writeFile(filePath, JSON.stringify(foreign), "utf8") + return "proceed" as const + }, + }) + + expect(result).toMatchObject({ outcome: "foreign_write" }) + expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual(foreign) + }) + + it("fails closed without writing when the disk file is missing", async () => { + const taskId = "task-restore-missing" + const filePath = path.join(tmpBaseDir, "tasks", taskId, "api_conversation_history.json") + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: proceedGate(), + }) + + expect(result).toMatchObject({ outcome: "unreadable" }) + await expect(fs.access(filePath)).rejects.toMatchObject({ code: "ENOENT" }) + }) + + it("fails closed without writing when the disk content is unreadable", async () => { + const taskId = "task-restore-corrupt" + const filePath = await seedDisk(taskId, []) + await fs.writeFile(filePath, "not json", "utf8") + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: proceedGate(), + }) + + expect(result).toMatchObject({ outcome: "unreadable" }) + expect(await fs.readFile(filePath, "utf8")).toBe("not json") + }) + + it("fails closed without writing when the status gate throws", async () => { + const taskId = "task-restore-gate-error" + const filePath = await seedDisk(taskId, prepared) + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: async () => { + throw new Error("no valid disk record") + }, + }) + + expect(result).toMatchObject({ outcome: "record_state_unknown" }) + expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual(prepared) + }) + + it("skips the write when the gate reports the record committed elsewhere", async () => { + const taskId = "task-restore-active" + const filePath = await seedDisk(taskId, prepared) + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: async () => "committed_elsewhere" as const, + }) + + expect(result).toEqual({ outcome: "committed_elsewhere" }) + expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual(prepared) + }) + + it("skips the write when the gate aborts on a conflicting record state", async () => { + const taskId = "task-restore-conflict" + const filePath = await seedDisk(taskId, prepared) + + const result = await restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: async () => "abort" as const, + }) + + expect(result).toEqual({ outcome: "record_state_conflict" }) + expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual(prepared) + }) + + it("rethrows an unexpected write failure after the rollback restored the disk content", async () => { + const taskId = "task-restore-write-error" + const filePath = await seedDisk(taskId, prepared) + const realRename = fs.rename.bind(fs) + const renameSpy = vi.spyOn(fs, "rename").mockImplementation(async (oldPath, newPath) => { + if (String(newPath).endsWith("api_conversation_history.json") && String(oldPath).includes(".new_")) { + throw new Error("injected disk full") + } + return realRename(oldPath as string, newPath as string) + }) + + await expect( + restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath: tmpBaseDir, + expected: prepared, + replacement: original, + verifyCanRestore: proceedGate(), + }), + ).rejects.toThrow("injected disk full") + + renameSpy.mockRestore() + // The safeWriteJson rollback put the prepared content back. + expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual(prepared) + }) +}) diff --git a/src/core/task-persistence/apiMessages.ts b/src/core/task-persistence/apiMessages.ts index 51749e50fe..4b1e1a7b00 100644 --- a/src/core/task-persistence/apiMessages.ts +++ b/src/core/task-persistence/apiMessages.ts @@ -1,10 +1,12 @@ import * as path from "path" import * as fs from "fs/promises" +import crypto from "crypto" import { Anthropic } from "@anthropic-ai/sdk" import { safeWriteJson } from "../../utils/safeWriteJson" import { GlobalFileNames } from "../../shared/globalFileNames" +import { withFileLock } from "../../utils/fileLock" import { getTaskDirectoryPath } from "../../utils/storage" import { ensureMessageIdentifiers, mergeApiMessageSnapshots } from "./mergeMessageSnapshots" import { getErrorCode, readFileWithMissingRetry } from "./readFileWithMissingRetry" @@ -91,6 +93,17 @@ async function readApiMessagesFile(taskId: string, filePath: string): Promise { + const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId) + return path.join(taskDir, GlobalFileNames.apiConversationHistory) +} + export async function readApiMessages({ taskId, globalStoragePath, @@ -98,15 +111,14 @@ export async function readApiMessages({ taskId: string globalStoragePath: string }): Promise { - const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId) - const filePath = path.join(taskDir, GlobalFileNames.apiConversationHistory) + const filePath = await getApiConversationHistoryFilePath({ taskId, globalStoragePath }) const currentMessages = await readApiMessagesFile(taskId, filePath) if (currentMessages !== undefined) { return currentMessages } - const oldPath = path.join(taskDir, "claude_messages.json") + const oldPath = path.join(path.dirname(filePath), "claude_messages.json") const legacyMessages = await readApiMessagesFile(taskId, oldPath) if (legacyMessages === undefined) { return [] @@ -144,8 +156,7 @@ export async function saveApiMessages({ merge?: boolean }): Promise { ensureMessageIdentifiers(messages) - const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId) - const filePath = path.join(taskDir, GlobalFileNames.apiConversationHistory) + const filePath = await getApiConversationHistoryFilePath({ taskId, globalStoragePath }) let savedMessages = messages await safeWriteJson( filePath, @@ -161,3 +172,162 @@ export async function saveApiMessages({ ) return savedMessages } + +/** + * Writes one durable copy of the pre-resume API conversation history into the + * task directory before a resume prepares its slice. The name embeds a random + * UUID, so concurrent resumes on different hosts never overwrite each other's + * copies. Callers keep the returned path and remove the file only after the + * resume outcome makes the copy obsolete. + * + * @throws When the copy cannot be written, so the caller can stop the resume + * before its destructive preparation write. + */ +export async function createResumeHistoryRecoveryCopy({ + taskId, + globalStoragePath, + messages, +}: { + taskId: string + globalStoragePath: string + messages: ApiMessage[] +}): Promise { + const historyPath = await getApiConversationHistoryFilePath({ taskId, globalStoragePath }) + const recoveryCopyPath = `${historyPath}.pre-resume-${crypto.randomUUID()}` + await safeWriteJson(recoveryCopyPath, ensureMessageIdentifiers(structuredClone(messages))) + return recoveryCopyPath +} + +/** + * Removes one pre-resume recovery copy. A missing file is already the target + * state, so only unexpected removal failures are logged. The copy stays on + * disk on failure, which keeps the original history recoverable. + */ +export async function removeResumeHistoryRecoveryCopy(recoveryCopyPath: string): Promise { + try { + await fs.unlink(recoveryCopyPath) + } catch (error) { + if (getErrorCode(error) !== "ENOENT") { + console.warn( + `[resume-history] Failed to remove the pre-resume recovery copy at ${recoveryCopyPath}:`, + error, + ) + } + } +} + +/** + * What the status gate decided while the API-history file lock was held. + * `proceed` allows the restore, `committed_elsewhere` means the record is + * active and another writer owns the resumed attempt, and `abort` means the + * persisted record no longer shows the status the resume started from. + */ +export type ResumeRestoreStatusGate = "proceed" | "committed_elsewhere" | "abort" + +export type ResumeHistoryRestoreOutcome = + | "restored" + | "committed_elsewhere" + | "record_state_unknown" + | "record_state_conflict" + | "foreign_write" + | "unreadable" + +export interface ResumeHistoryRestoreResult { + outcome: ResumeHistoryRestoreOutcome + error?: unknown +} + +export interface RestoreApiMessagesIfUnchangedOptions { + taskId: string + globalStoragePath: string + /** + * The prepared resume slice. The restore commits only when the disk file + * still holds exactly this content. + */ + expected: ApiMessage[] + /** The pre-resume history written back when the disk still holds `expected`. */ + replacement: ApiMessage[] + /** + * Status gate, called inside the API-history file advisory lock before the + * write. It must read the authoritative task record under the record + * file's own advisory lock, which gives the locked order API-history file + * before record file. No existing writer takes the reverse order, so the + * nesting cannot deadlock. Throw to fail closed on an unreadable record. + */ + verifyCanRestore: () => Promise +} + +/** + * Conditionally restores the pre-resume API conversation history in one + * advisory-lock scope. The status gate, the content compare, and the write + * commit run under the API-history file lock, and the compare itself runs in + * the write's merge callback, so a lock-honoring writer cannot slip a newer + * history between the compare and the commit. The restore writes only when + * the disk still holds the prepared slice; any other content, including a + * newer history from a competing host, is left untouched. Callers keep the + * pre-resume recovery copy for every outcome except `restored` and + * `committed_elsewhere`. + * + * The guarantee covers writers that honor the shared advisory lock, which is + * every in-repo writer of this file. A process that edits the file without + * the lock can still race the commit window; no file-level fix covers that. + * + * @throws Only on an unexpected write failure, for example a full disk. The + * caller must then treat the original history as not yet restored. + */ +export async function restoreApiMessagesIfUnchanged({ + taskId, + globalStoragePath, + expected, + replacement, + verifyCanRestore, +}: RestoreApiMessagesIfUnchangedOptions): Promise { + const filePath = await getApiConversationHistoryFilePath({ taskId, globalStoragePath }) + // ensureMessageIdentifiers mutates its input, so both sides work on clones. + const expectedContent = ensureMessageIdentifiers(structuredClone(expected)) + const replacementContent = ensureMessageIdentifiers(structuredClone(replacement)) + const expectedText = JSON.stringify(expectedContent) + + return withFileLock(filePath, async () => { + try { + const gate = await verifyCanRestore() + if (gate === "committed_elsewhere") { + return { outcome: "committed_elsewhere" } + } + if (gate === "abort") { + return { outcome: "record_state_conflict" } + } + } catch (error) { + return { outcome: "record_state_unknown", error } + } + + let diskState: "unreadable" | "mismatched" | undefined + try { + await safeWriteJson(filePath, replacementContent, { + assumeLockHeld: true, + merge: (existing) => { + if (existing === null) { + diskState = "unreadable" + throw new Error(`the disk API conversation history for task ${taskId} is missing or unreadable`) + } + if (JSON.stringify(existing) !== expectedText) { + diskState = "mismatched" + throw new Error( + `the disk API conversation history for task ${taskId} changed after preparation`, + ) + } + return replacementContent + }, + }) + } catch (error) { + if (diskState === "unreadable") { + return { outcome: "unreadable", error } + } + if (diskState === "mismatched") { + return { outcome: "foreign_write", error } + } + throw error + } + return { outcome: "restored" } + }) +} diff --git a/src/core/task-persistence/index.ts b/src/core/task-persistence/index.ts index 6afd1e7d9b..d0dab9fc2a 100644 --- a/src/core/task-persistence/index.ts +++ b/src/core/task-persistence/index.ts @@ -2,7 +2,14 @@ export { type ApiMessage, ApiMessagesReadError, type ApiMessagesReadErrorKind, + createResumeHistoryRecoveryCopy, + getApiConversationHistoryFilePath, readApiMessages, + removeResumeHistoryRecoveryCopy, + restoreApiMessagesIfUnchanged, + type ResumeHistoryRestoreOutcome, + type ResumeHistoryRestoreResult, + type ResumeRestoreStatusGate, saveApiMessages, } from "./apiMessages" export { diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index ecef4016e7..c69723bbeb 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -116,8 +116,12 @@ import { ClineProvider } from "../webview/ClineProvider" import { MultiSearchReplaceDiffStrategy } from "../diff/strategies/multi-search-replace" import { type ApiMessage, + createResumeHistoryRecoveryCopy, ensureMessageIdentifiers, readApiMessages, + removeResumeHistoryRecoveryCopy, + restoreApiMessagesIfUnchanged, + type ResumeHistoryRestoreResult, saveApiMessages, readTaskMessages, saveTaskMessages, @@ -1489,12 +1493,18 @@ export class Task extends EventEmitter implements TaskLike { // For API requests, consecutive same-role messages are merged via mergeConsecutiveApiMessages() // so rewind/edit behavior can still reference original message boundaries. - /** Replaces the entire API conversation history and persists the new state. */ - async overwriteApiConversationHistory(newHistory: ApiMessage[], persist = true) { + /** + * Replaces the entire API conversation history and persists the new state. + * Returns false when the requested disk write failed. The in-memory + * history always reflects `newHistory` after the call, so a caller that + * must not keep an unpersisted rewrite restores it explicitly. + */ + async overwriteApiConversationHistory(newHistory: ApiMessage[], persist = true): Promise { this.hydrateApiConversationHistory(newHistory) - if (persist) { - await this.saveApiConversationHistory(false) + if (!persist) { + return true } + return this.saveApiConversationHistory(false) } /** @@ -3381,22 +3391,69 @@ export class Task extends EventEmitter implements TaskLike { }) } - await this.overwriteApiConversationHistory(modifiedApiConversationHistory) + // Preserve the authoritative history durably before the destructive + // preparation write, so a failed preparation or restore can never + // lose the original across a restart. + const recoveryCopyPath = await createResumeHistoryRecoveryCopy({ + taskId: this.taskId, + globalStoragePath: this.globalStoragePath, + messages: existingApiConversationHistory, + }) + + const persisted = await this.overwriteApiConversationHistory(modifiedApiConversationHistory) + if (!persisted) { + // The failed write leaves the disk file with the authoritative + // history or its safeWriteJson backup. Keep the recovery copy so + // the original stays recoverable on disk either way. Put the + // original back in memory so no later save from this instance + // can persist the unverified slice. + this.hydrateApiConversationHistory(existingApiConversationHistory) + throw new Error( + `[Task#resumeTaskFromHistory] Failed to persist the prepared API conversation history for task ${this.taskId}. The resume stopped before activation, the record stays interrupted, and the original history is preserved at ${recoveryCopyPath}.`, + ) + } // All fallible resume preparation is complete. Activate the // interrupted record last, so a preparation failure never consumes // the one interrupted → active resume. if (this.abort || this.abandoned) { + await this.restorePreparedResumeHistory( + recoveryCopyPath, + existingApiConversationHistory, + modifiedApiConversationHistory, + ) return } if (this.initialStatus === "interrupted") { const provider = this.providerRef.deref() if (!provider) { + await this.restorePreparedResumeHistory( + recoveryCopyPath, + existingApiConversationHistory, + modifiedApiConversationHistory, + ) throw new Error(`[Task#resumeTaskFromHistory] Provider unavailable for task ${this.taskId}`) } - await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + try { + await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + } catch (error) { + // The record stays interrupted when the locked commit rejects. + // Put the authoritative history back so a retry resumes from + // it. The restore never runs when the disk record is already + // active, because that attempt belongs to another writer. + await this.restorePreparedResumeHistory( + recoveryCopyPath, + existingApiConversationHistory, + modifiedApiConversationHistory, + ) + throw error + } } + // The resume committed, so the prepared history is the accepted + // state and the pre-resume copy is no longer needed. + await removeResumeHistoryRecoveryCopy(recoveryCopyPath) + // Task resuming from history item. await this.initiateTaskLoop(newUserContent) } catch (error) { @@ -3409,6 +3466,92 @@ export class Task extends EventEmitter implements TaskLike { } } + /** + * Restores the authoritative API history after the resume persisted + * prepared history but never activated the record. The interrupted record + * stays retryable, so the disk history must keep its pre-resume content. + * The status gate, the content compare, and the write commit run under one + * advisory lock on the API-history file, so a lock-honoring writer cannot + * slip a newer history between the compare and the commit. The gate fails + * closed: the restore runs only while the persisted record still shows the + * status this resume started from, and an active record always keeps the + * prepared history. Every skipped, rejected, or failed outcome keeps the + * pre-resume recovery copy on disk, so the original history stays + * recoverable after a restart. + */ + private async restorePreparedResumeHistory( + recoveryCopyPath: string, + original: ApiMessage[], + prepared: ApiMessage[], + ): Promise { + this.hydrateApiConversationHistory(original) + + const provider = this.providerRef.deref() + if (!provider) { + console.error( + `[Task#${this.taskId}] restorePreparedResumeHistory: no provider is available, so the authoritative record status is unknown. The restore did not run and the original history is preserved at ${recoveryCopyPath}.`, + ) + return + } + + let result: ResumeHistoryRestoreResult + try { + result = await restoreApiMessagesIfUnchanged({ + taskId: this.taskId, + globalStoragePath: this.globalStoragePath, + expected: prepared, + replacement: original, + verifyCanRestore: async () => { + const diskStatus = await provider.taskHistoryStore.readValidatedDiskStatus(this.taskId) + if (diskStatus === "active") { + return "committed_elsewhere" + } + return diskStatus === (this.initialStatus ?? "active") ? "proceed" : "abort" + }, + }) + } catch (error) { + console.error( + `[Task#${this.taskId}] restorePreparedResumeHistory: partial failure. The restore write failed, so the disk keeps the prepared history. The original history is preserved at ${recoveryCopyPath} and in this Task memory, and the record stays resumable.`, + error, + ) + return + } + + switch (result.outcome) { + case "restored": + await removeResumeHistoryRecoveryCopy(recoveryCopyPath) + return + case "committed_elsewhere": + await removeResumeHistoryRecoveryCopy(recoveryCopyPath) + console.error( + `[Task#${this.taskId}] restorePreparedResumeHistory: the record is active on disk, so the resume commit landed before the failure. The prepared API history is kept and no restore ran.`, + ) + return + case "record_state_unknown": + console.error( + `[Task#${this.taskId}] restorePreparedResumeHistory: the authoritative record status is unreadable, so the restore failed closed and no restore ran. The original history is preserved at ${recoveryCopyPath}.`, + result.error, + ) + return + case "record_state_conflict": + console.error( + `[Task#${this.taskId}] restorePreparedResumeHistory: the record status on disk no longer matches the status this resume started from, so the restore failed closed and no restore ran. The original history is preserved at ${recoveryCopyPath}.`, + ) + return + case "foreign_write": + console.warn( + `[Task#${this.taskId}] restorePreparedResumeHistory: the disk API history changed after preparation, so no restore ran. The original history is preserved at ${recoveryCopyPath}.`, + ) + return + case "unreadable": + console.error( + `[Task#${this.taskId}] restorePreparedResumeHistory: the disk API history is missing or unreadable, so the restore failed closed and no restore ran. The original history is preserved at ${recoveryCopyPath}.`, + result.error, + ) + return + } + } + private async resumePendingTaskAction(action: PendingTaskAction): Promise { const provider = this.providerRef.deref() if (!provider) { diff --git a/src/core/task/__tests__/Task.ignored-disabled-tools-notice.spec.ts b/src/core/task/__tests__/Task.ignored-disabled-tools-notice.spec.ts index a83586122a..ee3736b06d 100644 --- a/src/core/task/__tests__/Task.ignored-disabled-tools-notice.spec.ts +++ b/src/core/task/__tests__/Task.ignored-disabled-tools-notice.spec.ts @@ -102,6 +102,16 @@ vi.mock("delay", () => ({ default: vi.fn().mockResolvedValue(undefined), })) +// The resume path persists the prepared API history through the real +// task-persistence module. Stub the two disk-writing recovery helpers so the +// in-memory fixture never touches a real storage path. Every other export +// stays real. +vi.mock("../../task-persistence", async (importOriginal) => ({ + ...(await importOriginal()), + createResumeHistoryRecoveryCopy: vi.fn().mockResolvedValue("/mock/pre-resume-copy"), + removeResumeHistoryRecoveryCopy: vi.fn().mockResolvedValue(undefined), +})) + type StartTaskAccess = { startTask: (task?: string, images?: string[]) => Promise resumeTaskFromHistory: () => Promise @@ -255,7 +265,7 @@ async function resumeTaskWithDisabledTools(disabledTools: string[] | undefined) { role: "user", content: [{ type: "text", text: "historical task" }], ts: historyItem.ts }, { role: "assistant", content: [{ type: "text", text: "Working on it." }], ts: historyItem.ts + 1 }, ]) - vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(undefined) + vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(true) const initiateLoopSpy = vi.spyOn(taskAccess, "initiateTaskLoop").mockResolvedValue(undefined) await taskAccess.resumeTaskFromHistory() diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index ff9df8b976..a3c61f3a04 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -32,6 +32,7 @@ type TaskPersistenceAccess = { resumePendingTaskAction: (action: PendingTaskAction) => Promise saveClineMessages: () => Promise initiateTaskLoop: (userContent: Anthropic.Messages.ContentBlockParam[]) => Promise + retrySaveApiConversationHistory: () => Promise } function getTaskPersistenceAccess(task: Task): TaskPersistenceAccess { @@ -55,6 +56,9 @@ const { mockReadTaskMessages, mockTaskMetadata, mockPWaitFor, + mockCreateResumeHistoryRecoveryCopy, + mockRemoveResumeHistoryRecoveryCopy, + mockRestoreApiMessagesIfUnchanged, } = vi.hoisted(() => ({ mockSaveApiMessages: vi.fn().mockResolvedValue(undefined), mockSaveTaskMessages: vi.fn().mockResolvedValue(undefined), @@ -72,6 +76,11 @@ const { }, }), mockPWaitFor: vi.fn().mockResolvedValue(undefined), + mockCreateResumeHistoryRecoveryCopy: vi + .fn() + .mockResolvedValue("/mock/tasks/test-id/api_conversation_history.json.pre-resume-mock"), + mockRemoveResumeHistoryRecoveryCopy: vi.fn().mockResolvedValue(undefined), + mockRestoreApiMessagesIfUnchanged: vi.fn().mockResolvedValue({ outcome: "restored" }), })) // ─── Module mocks ──────────────────────────────────────────────────────────── @@ -117,12 +126,17 @@ vi.mock("../../task-persistence", async (importOriginal) => { readApiMessages: mockReadApiMessages, readTaskMessages: mockReadTaskMessages, taskMetadata: mockTaskMetadata, + createResumeHistoryRecoveryCopy: mockCreateResumeHistoryRecoveryCopy, + removeResumeHistoryRecoveryCopy: mockRemoveResumeHistoryRecoveryCopy, + restoreApiMessagesIfUnchanged: mockRestoreApiMessagesIfUnchanged, TaskHistoryStore: vi.fn().mockImplementation(function () { return { initialize: vi.fn().mockResolvedValue(undefined), dispose: vi.fn(), get: vi.fn(), getAll: vi.fn().mockReturnValue([]), + invalidate: vi.fn().mockResolvedValue(undefined), + readValidatedDiskStatus: vi.fn().mockResolvedValue("interrupted"), upsert: vi.fn().mockResolvedValue([]), delete: vi.fn().mockResolvedValue(undefined), deleteMany: vi.fn().mockResolvedValue(undefined), @@ -2461,6 +2475,236 @@ describe("Task persistence", () => { }) }) + // ── resumeTaskFromHistory — prepared api history persistence gate and rollback ── + + describe("resumeTaskFromHistory api-history persistence gate and rollback", () => { + function makeResumeTask(): Task { + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + startTask: false, + initialStatus: "interrupted", + }) + mockProvider.taskHistoryStore.get = vi.fn().mockReturnValue({ id: task.taskId, status: "interrupted" }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) + vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop").mockResolvedValue(undefined) + return task + } + + /** + * Mirrors the production gate handling inside the real conditional + * restore, so the tests exercise the gate the Task passes into the + * lock scope. + */ + function runGateInsideRestore(): void { + mockRestoreApiMessagesIfUnchanged.mockImplementationOnce( + async (options: { verifyCanRestore: () => Promise<"proceed" | "committed_elsewhere" | "abort"> }) => { + try { + const gate = await options.verifyCanRestore() + return { outcome: gate === "proceed" ? "restored" : gate } + } catch (error) { + return { outcome: "record_state_unknown", error } + } + }, + ) + } + + it("stops before activation when the prepared api history fails to persist", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) + mockSaveApiMessages.mockRejectedValueOnce(new Error("disk full")) + + const task = makeResumeTask() + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow( + "Failed to persist the prepared API conversation history", + ) + + expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + // Only the failed preparation write ran. The disk file never changed. + expect(mockSaveApiMessages).toHaveBeenCalledTimes(1) + // The durable pre-resume copy exists and stays, because the + // safeWriteJson rollback that restored the original on disk may + // itself have failed. + expect(mockCreateResumeHistoryRecoveryCopy).toHaveBeenCalledTimes(1) + expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() + // The in-memory history holds the authoritative content again, so no + // later save from this instance can persist the unverified slice. + expect(task.apiConversationHistory).toEqual([ + { role: "user", content: "Continue", messageId: expect.any(String) }, + ]) + }) + + it("restores the authoritative api history through the locked conditional restore when the resume rejects a stale parent", async () => { + mockReadTaskMessages.mockResolvedValue([]) + // Reads: hydration and the post-acceptance derivation. The restore + // compares inside its own lock scope and reads the disk directly. + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + mockProvider.resumeInterruptedTask = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + runGateInsideRestore() + + const task = makeResumeTask() + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledTimes(1) + expect(loop).not.toHaveBeenCalled() + // The preparation slice is the only content that reached the disk layer. + const savedMessages = vi.mocked(mockSaveApiMessages).mock.calls.map((call) => call[0].messages) + expect(savedMessages).toEqual([[]]) + // The restore compared against the prepared slice and carried the + // authoritative history as its replacement. + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) + const restoreOptions = vi.mocked(mockRestoreApiMessagesIfUnchanged).mock.calls[0][0] + expect(restoreOptions.expected).toEqual([]) + expect(restoreOptions.replacement).toEqual([ + { role: "user", content: "Continue", messageId: expect.any(String) }, + ]) + // The restore succeeded, so the pre-resume copy was removed. + expect(mockRemoveResumeHistoryRecoveryCopy).toHaveBeenCalledTimes(1) + }) + + it("reports a partial failure and keeps the recovery copy when the restore write fails", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + // The preparation write lands. The locked restore write then fails. + mockSaveApiMessages.mockResolvedValueOnce(undefined) + mockRestoreApiMessagesIfUnchanged.mockRejectedValueOnce(new Error("restore disk full")) + mockProvider.resumeInterruptedTask = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + + const task = makeResumeTask() + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") + const retry = vi + .spyOn(getTaskPersistenceAccess(task), "retrySaveApiConversationHistory") + .mockResolvedValue(false) + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(loop).not.toHaveBeenCalled() + // The restore write runs inside the lock scope and does not retry + // through the Task save helper. + expect(retry).not.toHaveBeenCalled() + // The prepared slice is the last content that reached the disk layer. + const savedMessages = vi.mocked(mockSaveApiMessages).mock.calls.map((call) => call[0].messages) + expect(savedMessages).toEqual([[]]) + const recoveryCopyPath = await mockCreateResumeHistoryRecoveryCopy.mock.results[0].value + // The partial-failure log names the durable recovery copy, so the + // original history stays identifiable on disk after a restart. + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("partial failure"), expect.anything()) + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining(recoveryCopyPath), expect.anything()) + expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() + consoleError.mockRestore() + }) + + it("fails closed without a restore when the record status is unreadable", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + mockProvider.resumeInterruptedTask = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + mockProvider.taskHistoryStore.readValidatedDiskStatus = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("no valid disk record")) + runGateInsideRestore() + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + + const task = makeResumeTask() + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) + // The gate rejected inside the lock scope, so the restore never + // compared or wrote, and the recovery copy stays on disk. + expect(mockSaveApiMessages).toHaveBeenCalledTimes(1) + expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("failed closed"), expect.anything()) + consoleError.mockRestore() + }) + + it("keeps the prepared history and removes the recovery copy when the record is already active", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + mockProvider.resumeInterruptedTask = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + mockProvider.taskHistoryStore.readValidatedDiskStatus = vi.fn().mockResolvedValue("active") + runGateInsideRestore() + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + + const task = makeResumeTask() + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledTimes(1) + // The prepared history belongs to the writer that committed the + // active record, so the recovery copy is obsolete and removed. + expect(mockRemoveResumeHistoryRecoveryCopy).toHaveBeenCalledTimes(1) + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("active on disk")) + consoleError.mockRestore() + }) + + it("keeps the recovery copy when the disk history changed after preparation", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + mockProvider.resumeInterruptedTask = vi + .fn() + .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + mockRestoreApiMessagesIfUnchanged.mockResolvedValueOnce({ outcome: "foreign_write" }) + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}) + + const task = makeResumeTask() + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) + // The newer disk content is never clobbered, and the original stays + // recoverable in the copy. + expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() + const recoveryCopyPath = await mockCreateResumeHistoryRecoveryCopy.mock.results[0].value + expect(consoleWarn).toHaveBeenCalledWith(expect.stringContaining(recoveryCopyPath)) + consoleWarn.mockRestore() + }) + + it("removes the recovery copy when the resume activates and starts the loop", async () => { + mockReadTaskMessages.mockResolvedValue([]) + mockReadApiMessages + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) + + const task = makeResumeTask() + const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") + + await task["resumeTaskFromHistory"]() + + expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledTimes(1) + expect(loop).toHaveBeenCalledTimes(1) + expect(mockRestoreApiMessagesIfUnchanged).not.toHaveBeenCalled() + expect(mockRemoveResumeHistoryRecoveryCopy).toHaveBeenCalledTimes(1) + // The copy is removed only after the activation commit landed. + expect(mockRemoveResumeHistoryRecoveryCopy.mock.invocationCallOrder[0]).toBeLessThan( + loop.mock.invocationCallOrder[0], + ) + }) + }) + describe("resumeTaskFromHistory", () => { it.each(["not_found", "invalid", "io_error"] as const)( "does not persist when hydration fails with %s", diff --git a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts new file mode 100644 index 0000000000..e64cea8592 --- /dev/null +++ b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts @@ -0,0 +1,521 @@ +// cd src && npx vitest run core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts + +import * as fs from "fs/promises" +import * as os from "os" +import * as path from "path" + +import * as vscode from "vscode" + +import type { HistoryItem } from "@roo-code/types" +import { TelemetryService } from "@roo-code/telemetry" + +import { Task } from "../Task" +import { ClineProvider } from "../../webview/ClineProvider" +import { ContextProxy } from "../../config/ContextProxy" +import { WebviewFocusTracker } from "../../webview/WebviewFocusTracker" +import { providerIdentifiers } from "@roo-code/types/provider-identifiers" +import { GlobalFileNames } from "../../../shared/globalFileNames" +import { LifecycleTransitionError } from "../../task-persistence" +import type { ApiMessage } from "../../task-persistence/apiMessages" + +// The store, the message persistence helpers, and the per-task files all run +// for real. Only the editor shell and external integrations are mocked. The +// fs/promises rename wrapper stays real unless a test injects a failure. +vi.mock("fs/promises", async () => { + const actual = await vi.importActual("fs/promises") + return { ...actual, rename: vi.fn(actual.rename) } +}) + +vi.mock("delay", () => ({ + __esModule: true, + default: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock("execa", () => ({ + execa: vi.fn(), +})) + +vi.mock("p-wait-for", () => ({ + default: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock("vscode", () => { + const mockDisposable = { dispose: vi.fn() } + const mockEventEmitter = { event: vi.fn(), fire: vi.fn() } + const mockTextDocument = { uri: { fsPath: "/mock/workspace/path/file.ts" } } + const mockTextEditor = { document: mockTextDocument } + const mockTab = { input: { uri: { fsPath: "/mock/workspace/path/file.ts" } } } + const mockTabGroup = { tabs: [mockTab] } + + return { + TabInputTextDiff: vi.fn(), + CodeActionKind: { + QuickFix: { value: "quickfix" }, + RefactorRewrite: { value: "refactor.rewrite" }, + }, + window: { + createTextEditorDecorationType: vi.fn().mockReturnValue({ dispose: vi.fn() }), + visibleTextEditors: [mockTextEditor], + tabGroups: { + all: [mockTabGroup], + close: vi.fn(() => ({ dispose: vi.fn() })), + onDidChangeTabs: vi.fn(() => ({ dispose: vi.fn() })), + showErrorMessage: vi.fn(), + }, + showErrorMessage: vi.fn(), + }, + workspace: { + workspaceFolders: [ + { + uri: { fsPath: "/mock/workspace/path" }, + name: "mock-workspace", + index: 0, + }, + ], + createFileSystemWatcher: vi.fn(() => ({ + onDidCreate: vi.fn(() => mockDisposable), + onDidDelete: vi.fn(() => mockDisposable), + onDidChange: vi.fn(() => mockDisposable), + dispose: vi.fn(), + })), + fs: { + stat: vi.fn().mockResolvedValue({ type: 1 }), + }, + onDidSaveTextDocument: vi.fn(() => mockDisposable), + getConfiguration: vi.fn(() => ({ get: (_key: string, defaultValue: unknown) => defaultValue })), + }, + env: { + uriScheme: "vscode", + language: "en", + }, + EventEmitter: vi.fn().mockImplementation(function () { + return mockEventEmitter + }), + Disposable: { + from: vi.fn(), + }, + TabInputText: vi.fn(), + } +}) + +vi.mock("../../mentions", () => ({ + parseMentions: vi.fn().mockImplementation((text: string) => { + return Promise.resolve({ text, mode: undefined, contentBlocks: [] }) + }), + openMention: vi.fn(), + getLatestTerminalOutput: vi.fn(), +})) + +vi.mock("../../../integrations/misc/extract-text", () => ({ + extractTextFromFile: vi.fn().mockResolvedValue("Mock file content"), +})) + +vi.mock("../../environment/getEnvironmentDetails", () => ({ + getEnvironmentDetails: vi.fn().mockResolvedValue(""), +})) + +vi.mock("../../ignore/RooIgnoreController") + +vi.mock("../../condense", () => ({ + CondenseContext: vi.fn(), + summarizeTask: vi.fn().mockResolvedValue({ success: false }), +})) + +type TaskTestAccess = { + initiateTaskLoop: (userContent: unknown[]) => Promise + retrySaveApiConversationHistory: () => Promise +} + +function taskAccess(task: Task): TaskTestAccess { + return task as unknown as TaskTestAccess +} + +const childId = "resume-child" +const parentId = "resume-parent" + +const authoritativeApiHistory: ApiMessage[] = [ + { role: "user", content: [{ type: "text", text: "first user message" }], ts: 1, messageId: "m-1" }, + { role: "assistant", content: "assistant reply", ts: 2, messageId: "m-2" }, + { role: "user", content: [{ type: "text", text: "last user message" }], ts: 3, messageId: "m-3" }, +] + +// The preparation slice the resume writes before activation: the prepared +// history drops the last user message, which moves into the task loop input. +const preparedApiHistory = authoritativeApiHistory.slice(0, -1) + +function makeChildRecord(overrides: Partial = {}): HistoryItem { + return { + id: childId, + number: 1, + ts: 1, + task: "Child task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + workspace: "/mock/workspace/path", + status: "interrupted", + parentTaskId: parentId, + rootTaskId: parentId, + ...overrides, + } +} + +async function seedTaskFiles(storagePath: string, taskId: string, apiMessages: ApiMessage[]): Promise { + const taskDir = path.join(storagePath, "tasks", taskId) + await fs.mkdir(taskDir, { recursive: true }) + await fs.writeFile(path.join(taskDir, GlobalFileNames.uiMessages), "[]", "utf8") + await fs.writeFile(path.join(taskDir, GlobalFileNames.apiConversationHistory), JSON.stringify(apiMessages), "utf8") + return taskDir +} + +async function readPersistedApiHistory(storagePath: string, taskId: string): Promise { + const filePath = path.join(storagePath, "tasks", taskId, GlobalFileNames.apiConversationHistory) + return JSON.parse(await fs.readFile(filePath, "utf8")) +} + +async function readPersistedRecord(storagePath: string, taskId: string): Promise> { + const filePath = path.join(storagePath, "tasks", taskId, GlobalFileNames.historyItem) + return JSON.parse(await fs.readFile(filePath, "utf8")) +} + +function findRecoveryCopyName(taskDir: string): Promise { + return fs.readdir(taskDir).then((names) => names.find((name) => name.includes(".pre-resume-"))) +} + +describe("Task.resumeTaskFromHistory api-history persistence on real files", () => { + let storagePath: string + let provider: ClineProvider & Record + const realRename = vi.mocked(fs.rename).getMockImplementation()! + + beforeEach(async () => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + + storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-resume-api-history-")) + + const mockExtensionContext = { + globalState: { + get: vi.fn().mockReturnValue(undefined), + update: vi.fn().mockResolvedValue(undefined), + keys: vi.fn().mockReturnValue([]), + }, + globalStorageUri: { fsPath: storagePath }, + workspaceState: { + get: vi.fn().mockReturnValue(undefined), + update: vi.fn().mockResolvedValue(undefined), + keys: vi.fn().mockReturnValue([]), + }, + secrets: { + get: vi.fn().mockResolvedValue(undefined), + store: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + }, + extensionUri: { fsPath: "/mock/extension/path" }, + extension: { packageJSON: { version: "1.0.0" } }, + } as unknown as vscode.ExtensionContext + + const mockOutputChannel = { + appendLine: vi.fn(), + append: vi.fn(), + clear: vi.fn(), + show: vi.fn(), + hide: vi.fn(), + dispose: vi.fn(), + } as unknown as vscode.OutputChannel + + provider = new ClineProvider( + mockExtensionContext, + mockOutputChannel, + "sidebar", + new ContextProxy(mockExtensionContext), + new WebviewFocusTracker(), + ) as ClineProvider & Record + + provider.postMessageToWebview = vi.fn().mockResolvedValue(undefined) + provider.postStateToWebview = vi.fn().mockResolvedValue(undefined) + provider.log = vi.fn() + + await provider.taskHistoryStore.initialize() + }) + + afterEach(async () => { + provider.taskHistoryStore.dispose() + vi.mocked(fs.rename).mockImplementation(realRename) + await fs.rm(storagePath, { recursive: true, force: true }).catch(() => {}) + }) + + function prepareResumedTask(history: HistoryItem): { + task: Task + loop: ReturnType + resumeActivation: ReturnType + } { + const task = new Task({ + provider, + apiConfiguration: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-3-5-sonnet-20241022", + apiKey: "test-api-key", + }, + historyItem: history, + startTask: false, + }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" }) + const loop = vi.fn().mockResolvedValue(undefined) + vi.spyOn(taskAccess(task), "initiateTaskLoop").mockImplementation(loop) + const resumeActivation = vi.spyOn(provider, "resumeInterruptedTask") + return { task, loop, resumeActivation } + } + + it("commits the resume and keeps the prepared history when the parent still awaits the child", async () => { + await provider.taskHistoryStore.upsert({ + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: childId, + delegatedToId: childId, + childIds: [childId], + } as HistoryItem) + await provider.taskHistoryStore.upsert(makeChildRecord()) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + const { task, loop } = prepareResumedTask(makeChildRecord()) + await task["resumeTaskFromHistory"]() + + expect(loop).toHaveBeenCalledTimes(1) + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(preparedApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("active") + expect(provider.taskHistoryStore.get(childId)?.status).toBe("active") + // The resume committed, so no pre-resume recovery copy remains. + expect(await findRecoveryCopyName(taskDir)).toBeUndefined() + }) + + it("restores the persisted api history and keeps the record interrupted when the locked resume rejects a stale parent", async () => { + // The parent no longer awaits the child, so the locked commit rejects. + await provider.taskHistoryStore.upsert({ + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "active", + } as HistoryItem) + await provider.taskHistoryStore.upsert(makeChildRecord()) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + const { task, loop, resumeActivation } = prepareResumedTask(makeChildRecord()) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("no longer awaits it") + + expect(resumeActivation).toHaveBeenCalledTimes(1) + expect(loop).not.toHaveBeenCalled() + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(authoritativeApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + expect(persistedRecord.parentTaskId).toBe(parentId) + expect(provider.taskHistoryStore.get(childId)?.status).toBe("interrupted") + // The restore succeeded, so the pre-resume copy was removed. + expect(await findRecoveryCopyName(taskDir)).toBeUndefined() + }) + + it("stops before activation and keeps the persisted api history when the preparation write fails", async () => { + await provider.taskHistoryStore.upsert(makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined })) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + vi.mocked(fs.rename).mockImplementation(async (oldPath, newPath) => { + const targetsApiHistory = + String(newPath).endsWith(GlobalFileNames.apiConversationHistory) && String(oldPath).includes(".new_") + if (targetsApiHistory) { + throw new Error("injected api history rename failure") + } + return realRename(oldPath, newPath) + }) + + const { task, loop, resumeActivation } = prepareResumedTask( + makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined }), + ) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow( + "Failed to persist the prepared API conversation history", + ) + + expect(resumeActivation).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(authoritativeApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + // The safeWriteJson rollback restored the original on disk, so the + // durable pre-resume copy is no longer the only recoverable original. + // It stays on disk anyway, because the rollback itself could have failed. + const recoveryCopyName = await findRecoveryCopyName(taskDir) + expect(recoveryCopyName).toBeDefined() + expect(await fs.readFile(path.join(taskDir, recoveryCopyName!), "utf8")).toBe( + JSON.stringify(authoritativeApiHistory), + ) + }) + + it("restores the persisted api history when the task aborts after preparation and before activation", async () => { + await provider.taskHistoryStore.upsert(makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined })) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + const { task, loop, resumeActivation } = prepareResumedTask( + makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined }), + ) + const realOverwrite = task.overwriteApiConversationHistory.bind(task) + vi.spyOn(task, "overwriteApiConversationHistory").mockImplementation(async (history, persist) => { + const result = await realOverwrite(history, persist) + await task.abortTask(true) + return result + }) + + await task["resumeTaskFromHistory"]() + + expect(resumeActivation).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(authoritativeApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + expect(await findRecoveryCopyName(taskDir)).toBeUndefined() + }) + + it("reports a partial failure and proves the original bytes in the recovery copy when the restore write fails", async () => { + await provider.taskHistoryStore.upsert({ + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "active", + } as HistoryItem) + await provider.taskHistoryStore.upsert(makeChildRecord()) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + // The preparation commit lands. The restore commit fails once and the + // safeWriteJson rollback restores the prepared slice on disk. + let apiCommitRenames = 0 + vi.mocked(fs.rename).mockImplementation(async (oldPath, newPath) => { + const targetsApiHistory = + String(newPath).endsWith(GlobalFileNames.apiConversationHistory) && String(oldPath).includes(".new_") + if (targetsApiHistory) { + apiCommitRenames++ + if (apiCommitRenames > 1) { + throw new Error("injected restore rename failure") + } + } + return realRename(oldPath, newPath) + }) + + const { task, loop } = prepareResumedTask(makeChildRecord()) + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("no longer awaits it") + + expect(loop).not.toHaveBeenCalled() + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("partial failure"), expect.anything()) + consoleError.mockRestore() + // The prepared slice is what remains on disk. + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(preparedApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + const leftovers = await fs.readdir(taskDir) + expect(leftovers.some((name) => name.includes(".bak_"))).toBe(false) + // The durable pre-resume copy holds the exact original bytes under an + // identifiable name, so a restart can recover the interrupted task. + const recoveryCopyName = leftovers.find((name) => name.includes(".pre-resume-")) + expect(recoveryCopyName).toBeDefined() + expect(await fs.readFile(path.join(taskDir, recoveryCopyName!), "utf8")).toBe( + JSON.stringify(authoritativeApiHistory), + ) + }) + + it("never clobbers a newer foreign history that lands before the restore commit", async () => { + await provider.taskHistoryStore.upsert({ + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "active", + } as HistoryItem) + await provider.taskHistoryStore.upsert(makeChildRecord()) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + // A competing host writes newer history after our preparation and + // right before our activation rejects. The restore must honor it. + const foreignHistory: ApiMessage[] = [ + ...preparedApiHistory, + { role: "assistant", content: "foreign host progress", ts: 4, messageId: "m-foreign" }, + ] + const { task, loop } = prepareResumedTask(makeChildRecord()) + vi.mocked(provider.resumeInterruptedTask as ReturnType).mockImplementation(async () => { + await fs.writeFile( + path.join(taskDir, GlobalFileNames.apiConversationHistory), + JSON.stringify(foreignHistory), + "utf8", + ) + throw new LifecycleTransitionError("parent linkage changed") + }) + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(loop).not.toHaveBeenCalled() + // The foreign history is the last content on disk, so the restore did + // not clobber it with the pre-resume content. + expect(await readPersistedApiHistory(storagePath, childId)).toEqual(foreignHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + // The original stays recoverable in the named copy, because the + // foreign content won the file and the pre-resume history did not. + const recoveryCopyName = await findRecoveryCopyName(taskDir) + expect(recoveryCopyName).toBeDefined() + expect(await fs.readFile(path.join(taskDir, recoveryCopyName!), "utf8")).toBe( + JSON.stringify(authoritativeApiHistory), + ) + consoleWarn.mockRestore() + }) + + it("fails closed and keeps the recovery copy when the record status is unreadable", async () => { + await provider.taskHistoryStore.upsert(makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined })) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + // The record becomes unreadable right before our activation rejects, + // so the restore cannot learn the authoritative status. + const { task, loop } = prepareResumedTask(makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined })) + vi.mocked(provider.resumeInterruptedTask as ReturnType).mockImplementation(async () => { + await fs.writeFile(path.join(taskDir, GlobalFileNames.historyItem), "not json", "utf8") + throw new LifecycleTransitionError("parent linkage changed") + }) + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") + + expect(loop).not.toHaveBeenCalled() + // The restore failed closed: the disk keeps the prepared slice and the + // pre-resume copy keeps the original bytes. + expect(await readPersistedApiHistory(storagePath, childId)).toEqual(preparedApiHistory) + const recoveryCopyName = await findRecoveryCopyName(taskDir) + expect(recoveryCopyName).toBeDefined() + expect(await fs.readFile(path.join(taskDir, recoveryCopyName!), "utf8")).toBe( + JSON.stringify(authoritativeApiHistory), + ) + consoleError.mockRestore() + }) +}) diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 3066613263..20c1e561d4 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -2541,7 +2541,7 @@ describe("Cline", () => { newContextTokens: 1, condenseId: "condense-id", }) - const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(undefined) + const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(true) vi.spyOn(task, "submitUserMessage").mockResolvedValue(undefined) await expect(task.condenseContext()).resolves.toBeUndefined() @@ -5978,7 +5978,7 @@ describe("Cline", () => { // the aborted task's post-overwrite say from throwing before the // summarize/overwrite assertions can report a regression. vi.spyOn(task, "say").mockResolvedValue(undefined) - const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(undefined) + const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(true) // The summarizeConversation module mock is never cleared, so pin the // call count this condense starts from. const summarizeCallsBefore = vi.mocked(summarizeConversation).mock.calls.length @@ -6030,7 +6030,7 @@ describe("Cline", () => { // the aborted task's post-overwrite say from throwing before the // overwrite assertion can report the regression. const saySpy = vi.spyOn(task, "say").mockResolvedValue(undefined) - const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(undefined) + const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(true) // The summarizeConversation module mock is never cleared, so pin the // call count this condense starts from. const summarizeCallsBefore = vi.mocked(summarizeConversation).mock.calls.length @@ -6080,7 +6080,7 @@ describe("Cline", () => { ] vi.spyOn(getTaskTestAccess(task), "getSystemPrompt").mockResolvedValue("mock system prompt") vi.spyOn(task, "say").mockResolvedValue(undefined) - const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(undefined) + const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(true) // Suspend inside the collector so the abort lands while the // summarization request cannot have started yet. let releaseEnvDetails!: (value: string) => void @@ -6127,7 +6127,7 @@ describe("Cline", () => { ] vi.spyOn(getTaskTestAccess(task), "getSystemPrompt").mockResolvedValue("mock system prompt") vi.spyOn(task, "say").mockResolvedValue(undefined) - const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(undefined) + const overwriteSpy = vi.spyOn(task, "overwriteApiConversationHistory").mockResolvedValue(true) let releaseFilesRead!: (value: string[] | undefined) => void const filesReadGate = new Promise((resolve) => { releaseFilesRead = resolve From c5ed40e0689a94a85ec28d265125977d2d962fc6 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 16 Oct 2026 22:43:53 +0000 Subject: [PATCH 09/14] test: disable transitions when applying visual themes --- webview-ui/playwright/themes.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/webview-ui/playwright/themes.ts b/webview-ui/playwright/themes.ts index 65ec5ee384..bd329ada08 100644 --- a/webview-ui/playwright/themes.ts +++ b/webview-ui/playwright/themes.ts @@ -24,5 +24,15 @@ export async function applyVisualTheme(page: Page, theme: VisualTheme) { document.body.className = bodyClass document.body.removeAttribute("style") document.body.dataset.vscodeThemeId = themeId + // Contrast and layout assertions sample computed styles immediately + // after the theme flip. Remove transitions so a sampled color is the + // settled theme value, never mid-flight interpolation. + let styleElement = document.getElementById("visual-theme-disable-transitions") + if (!styleElement) { + styleElement = document.createElement("style") + styleElement.id = "visual-theme-disable-transitions" + document.head.appendChild(styleElement) + } + styleElement.textContent = "*, *::before, *::after { transition: none !important; }" }, theme) } From be2c3ebb55910889d47c355b15492f7a928d4222 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 17 Oct 2026 01:14:11 +0000 Subject: [PATCH 10/14] fix: harden interrupted-resume recovery and denial persistence --- docs/architecture/task-lifecycle-model.md | 7 +- scripts/check-task-lifecycle.ts | 27 ++ src/core/task-persistence/TaskHistoryStore.ts | 17 +- .../TaskHistoryStore.realConcurrency.spec.ts | 124 ++++++++++ src/core/task-persistence/apiMessages.ts | 3 +- src/core/task/Task.ts | 93 +++++-- .../__tests__/Task.pending-action.spec.ts | 67 +++++ .../task/__tests__/Task.persistence.spec.ts | 13 +- .../Task.resumeApiHistoryPersistence.spec.ts | 232 +++++++++++++++++- 9 files changed, 550 insertions(+), 33 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 4868504c95..b5721dba71 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -51,18 +51,19 @@ TLA+/PlusCal or Quint with TLC becomes a better fit when the lifecycle needs tem | `complete(child)` | `ClineProvider.reopenParentFromDelegation` | | `abandon(child)` | `ClineProvider.abandonSubtask` | | Pending-action settlement | `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear in the rejected-delegation settlement path (#1714) | +| Denied replay | denied pending-action replay in `Task.resumePendingTaskAction`, which persists the denial result before `TaskHistoryStore.resumeInterruptedTask` | | Atomic event step | `atomicReadAndUpdate`, `atomicUpdatePair`, `atomicUpdateParentDelegation`, the parent-file-locked resume, and the per-parent delegation transition lock | | Event interleaving | Competing completion, cancellation, abandonment, resume, and new delegation calls | The model has three fixed task slots, enough to cover competing siblings and a nested parent-child-grandchild chain. It explores every reachable interleaving through depth 13, deduplicating canonical states. Representative checks also exercise rejected operations that do not create a new state: a second concurrent delegation while the first child is active, stale completion after re-delegation, late completion after abandonment, completion after interruption, and nested completion. Named semantic landmarks require the graph to retain interrupted-child re-delegation, nested delegation, and an explicitly resumed interrupted child delegating to a grandchild even when the raw state total changes. The resume action calls the production `resumeInterruptedChild` composition, which validates the parent's delegation pointer before applying the child reducer, and carries a model-only provenance bit so the explorer can distinguish that path from a child that was never interrupted. Generic persisted transitions still reject `interrupted → active`. A `resume-stale` rejected action covers the competing-host class: after the parent re-delegates elsewhere, the parent-locked ownership check rejects the resume and every record stays unchanged. -Each task slot can also hold one of two pending `create_subtask` actions. A `stage` action mirrors `setPendingTaskAction` overwrite semantics, delegation clears the action its request carried, completion clears the child's action only when its event carries the matching action ID, and a `settle-rejected` action models the settlement that follows an authoritative delegation rejection (#1714). Production settles through the typed `LifecycleTransitionError` from the shared guards: the provider calls the disk-authoritative `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear under the per-file lock, then propagates the original rejection. Six named witnesses must remain reachable: settlement from an interrupted record after rejection, settlement through a successful active delegation, unrelated-action preservation during completion, stale-action protection where a settlement targeting one action ID leaves a replacement action intact, matching-ID completion clearing, and replacement-ID completion preservation. A mismatched pending-action request keeps its production behavior: the atomic update throws before any transition, and no settlement runs. +Each task slot can also hold one of two pending `create_subtask` actions. A `stage` action mirrors `setPendingTaskAction` overwrite semantics, delegation clears the action its request carried, completion clears the child's action only when its event carries the matching action ID, and a `settle-rejected` action models the settlement that follows an authoritative delegation rejection (#1714). Production settles through the typed `LifecycleTransitionError` from the shared guards: the provider calls the disk-authoritative `TaskHistoryStore.clearPendingActionIfMatching` compare-and-clear under the per-file lock, then propagates the original rejection. Seven named witnesses must remain reachable: settlement from an interrupted record after rejection, settlement through a successful active delegation, unrelated-action preservation during completion, stale-action protection where a settlement targeting one action ID leaves a replacement action intact, matching-ID completion clearing, replacement-ID completion preservation, and a denied-replay activation that settles only its matching action. A mismatched pending-action request keeps its production behavior: the atomic update throws before any transition, and no settlement runs. Production completion also accepts a recovery-compatible `active` parent that still awaits the returning child, then clears the stale pointers. Normal model transitions never create that intermediate state, so it is covered by a focused reducer test rather than admitted as a generally valid reachable state. -Explicit resume validates both records under file locks. A linked child resume holds the parent file's advisory lock across the parent ownership check and the child activation, so a competing host cannot re-delegate the parent in between. The re-delegation write (`atomicUpdateParentDelegation`) re-reads the awaited child's on-disk record inside the parent-file lock and rejects the delegation unless that child is still interrupted. The two guards share one lock order, parent file before child file, so they close the competing resume and re-delegation interleavings in both directions. The child-file merge still compares the caller's expected parent with the authoritative child backlink. The resume rejects an abandonment that committed before it, even in its partial-write window. +Explicit resume validates both records under file locks. A linked child resume holds the parent file's advisory lock across the parent ownership check and the child activation, so a competing host cannot re-delegate the parent in between. The re-delegation write (`atomicUpdateParentDelegation`) re-reads the awaited child's on-disk record under the child record's own advisory lock inside the parent-file lock, so a peer write's backup-rename window cannot make a valid child record look missing, and rejects the delegation unless that child is still interrupted. The two guards share one lock order, parent file before child file, so they close the competing resume and re-delegation interleavings in both directions. The child-file merge still compares the caller's expected parent with the authoritative child backlink. The resume rejects an abandonment that committed before it, even in its partial-write window. -Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A failed preparation write also stops the resume, and the Task restores the authoritative API conversation history in memory. Before its destructive preparation write the resume first stores a durable pre-resume recovery copy of the original history in the task directory under a UUID-distinct name, so a failed preparation or restore cannot lose the original across a restart. When the activation commit rejects, the Task restores the persisted history through a conditional write whose status gate, content compare, and commit all run under the API-history file's advisory lock; the gate reads the task record under the record file's lock inside that scope, and no existing writer takes the reverse lock order. The restore commits only while the persisted record still shows the status the resume started from and the disk still holds the prepared slice, so an active record and a newer foreign history always win. An unreadable record status or an unreadable history fails the restore closed, and the restore never runs on unknown state. A record that is already active on disk keeps the prepared history, and the skipped restore is logged as a partial failure with the now-obsolete recovery copy removed. A restore write that fails logs the same partial failure and names the recovery copy that still holds the original bytes. These guarantees bind writers that honor the shared advisory locks, which covers every in-repo writer of both files. A denied pending-action replay activates the interrupted record through the same locked path before its loop starts. An accepted non-interrupted resume still runs without a store transition. +Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A failed preparation write also stops the resume, and the Task restores the authoritative API conversation history in memory. Before its destructive preparation write the resume first stores a durable pre-resume recovery copy of the original history in the task directory under a UUID-distinct name, so a failed preparation or restore cannot lose the original across a restart. When the activation commit rejects, the Task restores the persisted history through a conditional write whose status gate, content compare, and commit all run under the API-history file's advisory lock; the gate reads the task record under the record file's lock inside that scope, and no existing writer takes the reverse lock order. The restore commits only while the persisted record still shows the status the resume started from and the disk still holds the prepared slice, so an active record and a newer foreign history always win. An unreadable record status or an unreadable history fails the restore closed, and the restore never runs on unknown state. A record that is already active on disk keeps the prepared history, and the skipped restore is logged as a partial failure that names the recovery copy, which is retained because the active status alone does not prove this attempt's prepared history was the one accepted. A restore write that fails logs the same partial failure and names the recovery copy that still holds the original bytes. These guarantees bind writers that honor the shared advisory locks, which covers every in-repo writer of both files. The denied pending-action replay uses the same checked order: it persists the denial tool result before the activation commit, a failed append stops before activation and leaves the record interrupted with its durable pending action, and an activation rejection keeps the committed denial without an API-history restore, because the denial is a settled user decision rather than an unaccepted prepared slice. An accepted non-interrupted resume still runs without a store transition. Rehydrated interrupted tasks omit lineage from message metadata saves, so their stale construction snapshots cannot restore severed links after rejection. Reverse-order stale abandonment and other live-task writers remain within the ownership/generation gaps below. diff --git a/scripts/check-task-lifecycle.ts b/scripts/check-task-lifecycle.ts index dcd1e5fc22..8478803abb 100644 --- a/scripts/check-task-lifecycle.ts +++ b/scripts/check-task-lifecycle.ts @@ -26,6 +26,7 @@ interface Transition { delegation?: { parentId: TaskId } completion?: { childId: TaskId; pendingActionId?: string } settlement?: { taskId: TaskId; actionId: string } + deniedReplay?: { taskId: TaskId; actionId: string } } interface TraceStep { @@ -47,6 +48,7 @@ const expectedActions = [ "interrupt", "resume", "resume-stale", + "resume-denied", "complete", "abandon", "stage", @@ -120,6 +122,20 @@ const semanticWitnesses = { canonicalTask(after.pendingAction) === canonicalTask(beforeAction) ) }, + // The denied pending-action replay persists its denial result before the + // activation commit, so the durable denial settles exactly its matching + // action and the record activates without it. + "denied-replay-settles-matching-action": ({ prev, next, transition }: WitnessContext) => { + if (transition.deniedReplay === undefined) return false + const before = prev[transition.deniedReplay.taskId] + const after = next[transition.deniedReplay.taskId] + return ( + before?.status === "interrupted" && + before.pendingAction?.actionId === transition.deniedReplay.actionId && + after?.status === "active" && + after.pendingAction === undefined + ) + }, } satisfies Record boolean> function task(id: TaskId, parentTaskId?: TaskId): ModelTask { @@ -212,6 +228,17 @@ function transitions(state: ModelState): Transition[] { name: `resume(${parentId})`, next: replace(state, { ...resumed, modelWasResumed: true }), }) + // The denied replay uses the same production resume reducers, and + // its denial result is durable before the activation commit, so + // the matching pending action is settled in the same step. + for (const actionId of actionIds) { + if (parent.pendingAction?.actionId !== actionId) continue + result.push({ + name: `resume-denied(${parentId}, ${actionId})`, + next: replace(state, { ...resumed, pendingAction: undefined }), + deniedReplay: { taskId: parentId, actionId }, + }) + } } else if (parent.status === "interrupted" && parent.parentTaskId && lineageParent !== undefined) { // Representative rejected event: the parent re-delegated elsewhere, // so the store's parent-locked ownership check must reject the diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index b18a19816a..d145d6c772 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -1255,7 +1255,22 @@ export class TaskHistoryStore { `[TaskHistoryStore] atomicUpdateParentDelegation: delegated task ${parentTaskId} has no awaited child`, ) } - const awaitedChild = await this.readValidatedTaskFile(awaitedChildId) + if (awaitedChildId === parentTaskId) { + // The lock below is not reentrant, so a self-await + // would deadlock instead of failing closed. + throw new LifecycleTransitionError( + `[TaskHistoryStore] atomicUpdateParentDelegation: delegated task ${parentTaskId} awaits itself`, + ) + } + // Read under the child record's advisory lock, in the + // same parent-before-child order as + // resumeInterruptedTask. A peer safeWriteJson renames + // the target to its backup before the commit rename, + // and an unlocked read inside that window rejects a + // valid delegation with a missing-record error. + const awaitedChild = await withFileLock(await this.getTaskFilePath(awaitedChildId), () => + this.readValidatedTaskFile(awaitedChildId), + ) awaitedChildDiskStatus = awaitedChild.status if (awaitedChildDiskStatus !== "interrupted") { throw new LifecycleTransitionError( diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index 6d338e0124..2df6de4ef4 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -6,6 +6,7 @@ import type { HistoryItem } from "@roo-code/types" import { TaskHistoryStore } from "../TaskHistoryStore" import { abandonDelegatedChild, delegateTaskToChild, LifecycleTransitionError } from "../taskLifecycle" +import { acquireFileLock } from "../../../utils/fileLock" // Wrap the real fs/promises so a test can pause inside the per-file lock // while every call still runs against the real filesystem. @@ -300,6 +301,129 @@ describe("TaskHistoryStore real cross-host locking", () => { } }) + it("waits for the awaited child's file lock while a peer write hides the record", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "redelegation-child-lock-window-")) + const store = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await store.initialize() + await store.upsert(parent) + await store.upsert(child) + + // A peer safeWriteJson holds the child file lock and has renamed + // the record to its backup, so the target is missing until the + // commit rename. The delegation must wait for the lock instead of + // rejecting the hidden record as missing. + const childFilePath = path.join(storagePath, "tasks", "child", "history_item.json") + const releaseChildLock = await acquireFileLock(childFilePath) + await fs.unlink(childFilePath) + + const originalRead = store["readValidatedTaskFile"].bind(store) + let markParentRecordRead!: () => void + const parentRecordRead = new Promise((resolve) => { + markParentRecordRead = resolve + }) + store["readValidatedTaskFile"] = async (taskId: string) => { + if (taskId === "parent") markParentRecordRead() + return originalRead(taskId) + } + + let settled = false + const delegation = store + .atomicUpdateParentDelegation("parent", (diskParent, awaitedChildDiskStatus) => + delegateTaskToChild(diskParent, "child-b", awaitedChildDiskStatus), + ) + .then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + // The parent record is read first. After the flush, an unlocked + // child read would already have rejected the hidden record, while + // the locked read cannot settle before the release below. + await parentRecordRead + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(settled).toBe(false) + + // The peer commit makes the still-interrupted record visible again. + await fs.writeFile(childFilePath, JSON.stringify(child), "utf8") + await releaseChildLock() + await delegation + + await store.invalidate("parent") + expect(store.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child-b" }) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + + it("re-reads the awaited child behind its lock and rejects a peer write that resumed it", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "redelegation-child-lock-resumed-")) + const store = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await store.initialize() + await store.upsert(parent) + await store.upsert(child) + + // The peer commits a resumed (active) child during the hidden + // window. The locked re-read must observe that committed record + // and reject the re-delegation. + const childFilePath = path.join(storagePath, "tasks", "child", "history_item.json") + const releaseChildLock = await acquireFileLock(childFilePath) + await fs.unlink(childFilePath) + + const originalRead = store["readValidatedTaskFile"].bind(store) + let markParentRecordRead!: () => void + const parentRecordRead = new Promise((resolve) => { + markParentRecordRead = resolve + }) + store["readValidatedTaskFile"] = async (taskId: string) => { + if (taskId === "parent") markParentRecordRead() + return originalRead(taskId) + } + + const delegation = store.atomicUpdateParentDelegation("parent", (diskParent, awaitedChildDiskStatus) => + delegateTaskToChild(diskParent, "child-b", awaitedChildDiskStatus), + ) + // The parent record is read first. After the flush, an unlocked + // child read would already have rejected with a missing-record + // error, while the locked read waits for the release below. + await parentRecordRead + await new Promise((resolve) => setTimeout(resolve, 20)) + await fs.writeFile(childFilePath, JSON.stringify({ ...child, status: "active" }), "utf8") + await releaseChildLock() + + await expect(delegation).rejects.toThrow("not interrupted") + + await store.invalidate("parent") + expect(store.get("parent")).toMatchObject({ status: "delegated", awaitingChildId: "child" }) + } finally { + store.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + it("commits a first delegation through the same lock-held parent write", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "delegation-first-lock-")) const storeA = new TaskHistoryStore(storagePath) diff --git a/src/core/task-persistence/apiMessages.ts b/src/core/task-persistence/apiMessages.ts index 4b1e1a7b00..204ef9c75b 100644 --- a/src/core/task-persistence/apiMessages.ts +++ b/src/core/task-persistence/apiMessages.ts @@ -265,8 +265,7 @@ export interface RestoreApiMessagesIfUnchangedOptions { * history between the compare and the commit. The restore writes only when * the disk still holds the prepared slice; any other content, including a * newer history from a competing host, is left untouched. Callers keep the - * pre-resume recovery copy for every outcome except `restored` and - * `committed_elsewhere`. + * pre-resume recovery copy for every outcome except `restored`. * * The guarantee covers writers that honor the shared advisory lock, which is * every in-repo writer of this file. A process that edits the file without diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index c69723bbeb..dcb2c3a7f3 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1398,8 +1398,10 @@ export class Task extends EventEmitter implements TaskLike { /** * Appends an API turn and records whether an assistant turn reached persistent storage. * If the message resolves a pending action, retries the save on initial failure before clearing the action. + * Returns whether the turn reached persistent storage, so a caller that must not run before a + * durable result can stop instead of ignoring the save outcome. */ - private async addToApiConversationHistory(message: Anthropic.MessageParam, reasoning?: string): Promise { + private async addToApiConversationHistory(message: Anthropic.MessageParam, reasoning?: string): Promise { const resolvesPendingAction = this.pendingAction && message.role === "user" && @@ -1435,6 +1437,7 @@ export class Task extends EventEmitter implements TaskLike { this.assistantMessageSavedToHistory = saved this.resolveAssistantMessagePersistence(saved) } + return saved } /** Cancels the current persistence generation before creating the next assistant-turn boundary. */ @@ -3522,9 +3525,12 @@ export class Task extends EventEmitter implements TaskLike { await removeResumeHistoryRecoveryCopy(recoveryCopyPath) return case "committed_elsewhere": - await removeResumeHistoryRecoveryCopy(recoveryCopyPath) + // The active status alone does not prove this instance's prepared + // history is the one on disk: another writer can prepare different + // content and activate between our write and this gate. The copy + // stays, so this attempt's original history stays recoverable. console.error( - `[Task#${this.taskId}] restorePreparedResumeHistory: the record is active on disk, so the resume commit landed before the failure. The prepared API history is kept and no restore ran.`, + `[Task#${this.taskId}] restorePreparedResumeHistory: the record is active on disk, so no restore ran and this attempt's prepared history is not proven accepted. The original history is preserved at ${recoveryCopyPath}.`, ) return case "record_state_unknown": @@ -3604,21 +3610,45 @@ export class Task extends EventEmitter implements TaskLike { await this.say("user_feedback", text ?? "", images) } - // A denied pending action still starts the task loop. Activate the - // interrupted record first, so the persisted status matches the running - // task. Approval paths above return early and run their own transitions. - if (this.initialStatus === "interrupted") { - await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) - } - const deniedContent = text ? formatResponse.toolDeniedWithFeedback(text) : formatResponse.toolDenied() - await this.initiateTaskLoop([ + const denialUserContent: Anthropic.Messages.ContentBlockParam[] = [ { type: "tool_result", tool_use_id: action.actionId, content: formatResponse.toolResult(deniedContent, images), }, - ]) + ] + + // A denied pending action still starts the task loop. For an + // interrupted record the denial result is persisted before the one + // interrupted → active commit, so a failed save cannot leave an active + // record whose API history never shows the denial. Approval paths + // above return early and run their own transitions. + if (this.initialStatus === "interrupted") { + const persisted = await this.addToApiConversationHistory({ role: "user", content: denialUserContent }) + if (!persisted) { + // Drop the unpersisted append so no later save from this + // instance can persist it, and stop before activation. The + // record stays interrupted with its durable pending action, so + // the replay stays retryable. + this.apiConversationHistory.pop() + throw new Error( + `[Task#resumePendingTaskAction] Failed to persist the denial result for task ${this.taskId}. The resume stopped before activation, the record stays interrupted, and the pending action stays replayable.`, + ) + } + // An activation rejection, for example a stale parent linkage, + // leaves the record interrupted with the denial durable and the + // action settled, so a replay proceeds from the committed denial. + // No API-history restore runs: the denial is a committed user + // decision, not an unaccepted prepared slice. + await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + // The denial result is already persisted, so the loop must not + // append it again. + await this.initiateTaskLoop(denialUserContent, { userContentAlreadyPersisted: true }) + return + } + + await this.initiateTaskLoop(denialUserContent) } /** @@ -3978,7 +4008,10 @@ export class Task extends EventEmitter implements TaskLike { // Task Loop - private async initiateTaskLoop(userContent: Anthropic.Messages.ContentBlockParam[]): Promise { + private async initiateTaskLoop( + userContent: Anthropic.Messages.ContentBlockParam[], + options: { userContentAlreadyPersisted?: boolean } = {}, + ): Promise { // Kicks off the checkpoints initialization process in the background. // `getCheckpointService` wraps its full body in a try/catch and returns // `undefined` on failure (see src/core/checkpoints/index.ts), so the @@ -3991,8 +4024,14 @@ export class Task extends EventEmitter implements TaskLike { this.emit(RooCodeEventName.TaskStarted) + let firstTurnOptions: typeof options | undefined = options while (!this.abort) { - const didEndLoop = await this.recursivelyMakeClineRequests(nextUserContent, includeFileDetails) + const didEndLoop = await this.recursivelyMakeClineRequests( + nextUserContent, + includeFileDetails, + firstTurnOptions, + ) + firstTurnOptions = undefined includeFileDetails = false // We only need file details the first time. // The way this agentic loop works is that cline will be given a @@ -4019,15 +4058,24 @@ export class Task extends EventEmitter implements TaskLike { public async recursivelyMakeClineRequests( userContent: Anthropic.Messages.ContentBlockParam[], includeFileDetails: boolean = false, + options: { userContentAlreadyPersisted?: boolean } = {}, ): Promise { interface StackItem { userContent: Anthropic.Messages.ContentBlockParam[] includeFileDetails: boolean retryAttempt?: number userMessageWasRemoved?: boolean // Track if user message was removed due to empty response + userContentAlreadyPersisted?: boolean // First turn of a denial replay: the content was persisted before activation } - const stack: StackItem[] = [{ userContent, includeFileDetails, retryAttempt: 0 }] + const stack: StackItem[] = [ + { + userContent, + includeFileDetails, + retryAttempt: 0, + userContentAlreadyPersisted: options.userContentAlreadyPersisted, + }, + ] while (stack.length > 0) { const currentItem = stack.pop()! @@ -4155,11 +4203,16 @@ export class Task extends EventEmitter implements TaskLike { const finalUserContent = [...contentWithoutEnvDetails, { type: "text" as const, text: environmentDetails }] // See shouldAddUserMessageToHistory for the full add/skip rules (retry/empty/removed). const isEmptyUserContent = currentUserContent.length === 0 - const shouldAddUserMessage = shouldAddUserMessageToHistory({ - retryAttempt: currentItem.retryAttempt, - isEmptyUserContent, - userMessageWasRemoved: currentItem.userMessageWasRemoved, - }) + // A persisted denial replay must not append its first turn again, + // so the already-persisted flag suppresses the history write and + // the empty-response removal that keys off it. + const shouldAddUserMessage = + !currentItem.userContentAlreadyPersisted && + shouldAddUserMessageToHistory({ + retryAttempt: currentItem.retryAttempt, + isEmptyUserContent, + userMessageWasRemoved: currentItem.userMessageWasRemoved, + }) if (shouldAddUserMessage) { await this.addToApiConversationHistory({ role: "user", content: finalUserContent }) this.messageCounts.user++ diff --git a/src/core/task/__tests__/Task.pending-action.spec.ts b/src/core/task/__tests__/Task.pending-action.spec.ts index 2299ae644d..4e6aedc5b0 100644 --- a/src/core/task/__tests__/Task.pending-action.spec.ts +++ b/src/core/task/__tests__/Task.pending-action.spec.ts @@ -212,6 +212,73 @@ describe("Task pending action replay", () => { ]) }) + it("persists the denial result before activating an interrupted record", async () => { + const provider = { resumeInterruptedTask: vi.fn().mockResolvedValue(undefined) } + const task = createTask(provider) + const state = task as unknown as { + initialStatus?: string + apiConversationHistory: unknown[] + addToApiConversationHistory: ReturnType + } + state.initialStatus = "interrupted" + state.apiConversationHistory = [] + state.addToApiConversationHistory = vi.fn().mockResolvedValue(true) + task.ask = vi.fn().mockResolvedValue({ response: "noButtonClicked" }) + const initiateTaskLoop = (task as unknown as { initiateTaskLoop: ReturnType }).initiateTaskLoop + + await getPendingActionAccess(task).resumePendingTaskAction(createAction) + + expect(state.addToApiConversationHistory).toHaveBeenCalledWith({ + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "create-action", + content: JSON.stringify({ status: "denied", message: "The user denied this operation." }), + }, + ], + }) + expect(provider.resumeInterruptedTask).toHaveBeenCalledWith("task-1", undefined) + expect(initiateTaskLoop).toHaveBeenCalledWith( + [expect.objectContaining({ type: "tool_result", tool_use_id: "create-action" })], + { userContentAlreadyPersisted: true }, + ) + const add = vi.mocked(state.addToApiConversationHistory) + const resume = vi.mocked(provider.resumeInterruptedTask) + expect(add.mock.invocationCallOrder[0]).toBeLessThan(resume.mock.invocationCallOrder[0]) + expect(resume.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(initiateTaskLoop).mock.invocationCallOrder[0]) + }) + + it("stops before activation when the denial result fails to persist", async () => { + const provider = { resumeInterruptedTask: vi.fn().mockResolvedValue(undefined) } + const task = createTask(provider) + const persistedHistory = [{ role: "user", content: "prior turn" }] + const state = task as unknown as { + initialStatus?: string + apiConversationHistory: unknown[] + addToApiConversationHistory: ReturnType + } + state.initialStatus = "interrupted" + state.apiConversationHistory = persistedHistory + state.addToApiConversationHistory = vi.fn().mockImplementation(async (message: unknown) => { + state.apiConversationHistory.push(message) + return false + }) + task.ask = vi.fn().mockResolvedValue({ response: "noButtonClicked" }) + const initiateTaskLoop = (task as unknown as { initiateTaskLoop: ReturnType }).initiateTaskLoop + + await expect(getPendingActionAccess(task).resumePendingTaskAction(createAction)).rejects.toThrow( + "Failed to persist the denial result", + ) + + expect(provider.resumeInterruptedTask).not.toHaveBeenCalled() + expect(initiateTaskLoop).not.toHaveBeenCalled() + // The unpersisted append is dropped, so the in-memory history matches + // the disk again and no later save can persist the denial from here. + expect(state.apiConversationHistory).toEqual(persistedHistory) + expect(state.apiConversationHistory).toHaveLength(1) + }) + it("fails clearly when the provider is unavailable", async () => { const task = createTask() diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index a3c61f3a04..6a01b58fbe 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -2417,7 +2417,7 @@ describe("Task persistence", () => { role: "user", content: [{ type: "tool_result", tool_use_id: "finish-action", content: "Denied" }], }), - ).resolves.toBeUndefined() + ).resolves.toBe(true) expect((task as unknown as { pendingAction?: PendingTaskAction }).pendingAction).toEqual(pendingAction) expect(consoleError).toHaveBeenCalledWith( @@ -2635,7 +2635,7 @@ describe("Task persistence", () => { consoleError.mockRestore() }) - it("keeps the prepared history and removes the recovery copy when the record is already active", async () => { + it("keeps the prepared history and the recovery copy when the record is already active", async () => { mockReadTaskMessages.mockResolvedValue([]) mockReadApiMessages .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) @@ -2652,10 +2652,13 @@ describe("Task persistence", () => { await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledTimes(1) - // The prepared history belongs to the writer that committed the - // active record, so the recovery copy is obsolete and removed. - expect(mockRemoveResumeHistoryRecoveryCopy).toHaveBeenCalledTimes(1) + // The active status alone does not prove this attempt's prepared + // history was the one accepted, so the recovery copy stays and the + // partial-failure log names its path. + expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() + const recoveryCopyPath = await mockCreateResumeHistoryRecoveryCopy.mock.results[0].value expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("active on disk")) + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining(recoveryCopyPath)) consoleError.mockRestore() }) diff --git a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts index e64cea8592..0b2e277859 100644 --- a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts +++ b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts @@ -6,7 +6,7 @@ import * as path from "path" import * as vscode from "vscode" -import type { HistoryItem } from "@roo-code/types" +import type { HistoryItem, PendingTaskAction } from "@roo-code/types" import { TelemetryService } from "@roo-code/telemetry" import { Task } from "../Task" @@ -119,6 +119,9 @@ vi.mock("../../ignore/RooIgnoreController") vi.mock("../../condense", () => ({ CondenseContext: vi.fn(), summarizeTask: vi.fn().mockResolvedValue({ success: false }), + // The seeded histories hold no condensation markers, so the identity + // mapping matches the real filter for these tests. + getEffectiveApiHistory: vi.fn((messages: unknown[]) => messages), })) type TaskTestAccess = { @@ -182,7 +185,7 @@ function findRecoveryCopyName(taskDir: string): Promise { return fs.readdir(taskDir).then((names) => names.find((name) => name.includes(".pre-resume-"))) } -describe("Task.resumeTaskFromHistory api-history persistence on real files", () => { +describe("Task resume api-history persistence on real files", () => { let storagePath: string let provider: ClineProvider & Record const realRename = vi.mocked(fs.rename).getMockImplementation()! @@ -519,3 +522,228 @@ describe("Task.resumeTaskFromHistory api-history persistence on real files", () consoleError.mockRestore() }) }) + +describe("Task.resumePendingTaskAction denial persistence on real files", () => { + let storagePath: string + let provider: ClineProvider & Record + const realRename = vi.mocked(fs.rename).getMockImplementation()! + + const finishAction: PendingTaskAction = { + kind: "finish_subtask", + actionId: "finish-action-1", + approvalText: "{}", + parentTaskId: parentId, + result: "Done", + } + + // The pending denial answers this tool call, so the seeded history ends + // with the assistant turn that requested it. + const denialApiHistory: ApiMessage[] = [ + { role: "user", content: [{ type: "text", text: "do the thing" }], ts: 1, messageId: "m-1" }, + { + role: "assistant", + content: [{ type: "tool_use", id: finishAction.actionId, name: "finish_subtask", input: {} }], + ts: 2, + messageId: "m-2", + }, + ] + + function makeDelegatedParent(): HistoryItem { + return { + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: childId, + delegatedToId: childId, + childIds: [childId], + } + } + + function makePendingChildRecord(): HistoryItem { + return makeChildRecord({ pendingAction: finishAction }) + } + + function prepareDeniedTask(history: HistoryItem): { task: Task; resumeActivation: ReturnType } { + const task = new Task({ + provider, + apiConfiguration: { + apiProvider: providerIdentifiers.anthropic, + apiModelId: "claude-3-5-sonnet-20241022", + apiKey: "test-api-key", + }, + historyItem: history, + startTask: false, + }) + vi.spyOn(task, "ask").mockResolvedValue({ response: "noButtonClicked" }) + provider.postStateToWebviewWithoutTaskHistory = vi.fn().mockResolvedValue(undefined) + const resumeActivation = vi.spyOn(provider, "resumeInterruptedTask") + return { task, resumeActivation } + } + + function countDenialResults(messages: ApiMessage[]): number { + return messages.filter( + (message) => + message.role === "user" && + Array.isArray(message.content) && + message.content.some( + (block) => + block.type === "tool_result" && + (block as { tool_use_id?: string }).tool_use_id === finishAction.actionId, + ), + ).length + } + + beforeEach(async () => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + + storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "task-denial-persistence-")) + + const mockExtensionContext = { + globalState: { + get: vi.fn().mockReturnValue(undefined), + update: vi.fn().mockResolvedValue(undefined), + keys: vi.fn().mockReturnValue([]), + }, + globalStorageUri: { fsPath: storagePath }, + workspaceState: { + get: vi.fn().mockReturnValue(undefined), + update: vi.fn().mockResolvedValue(undefined), + keys: vi.fn().mockReturnValue([]), + }, + secrets: { + get: vi.fn().mockResolvedValue(undefined), + store: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + }, + extensionUri: { fsPath: "/mock/extension/path" }, + extension: { packageJSON: { version: "1.0.0" } }, + } as unknown as vscode.ExtensionContext + + const mockOutputChannel = { + appendLine: vi.fn(), + append: vi.fn(), + clear: vi.fn(), + show: vi.fn(), + hide: vi.fn(), + dispose: vi.fn(), + } as unknown as vscode.OutputChannel + + provider = new ClineProvider( + mockExtensionContext, + mockOutputChannel, + "sidebar", + new ContextProxy(mockExtensionContext), + new WebviewFocusTracker(), + ) as ClineProvider & Record + + provider.postMessageToWebview = vi.fn().mockResolvedValue(undefined) + provider.postStateToWebview = vi.fn().mockResolvedValue(undefined) + provider.log = vi.fn() + + await provider.taskHistoryStore.initialize() + }) + + afterEach(async () => { + provider.taskHistoryStore.dispose() + vi.mocked(fs.rename).mockImplementation(realRename) + await fs.rm(storagePath, { recursive: true, force: true }).catch(() => {}) + }) + + it("persists the denial result before activation and settles the action once", async () => { + await provider.taskHistoryStore.upsert(makeDelegatedParent()) + await provider.taskHistoryStore.upsert(makePendingChildRecord()) + await seedTaskFiles(storagePath, childId, denialApiHistory) + + const { task, resumeActivation } = prepareDeniedTask(makePendingChildRecord()) + // Drive exactly one real assistant turn through the resumed task; the + // abandoned exit stops the loop right after the stream. The spy also + // keeps the mocked request off the network. + vi.spyOn(task, "attemptApiRequest").mockImplementation(() => + (async function* () { + yield { type: "text" as const, text: "understood" } + task["abandoned"] = true + })(), + ) + // The real model-metadata fetch reaches outside the test scope, so the + // turn runs on the stubbed seam like the other loop-driving suites. + const loopAccess = task as unknown as { safeEnsureModelFetched: () => Promise } + loopAccess.safeEnsureModelFetched = vi.fn().mockResolvedValue(undefined) + + await task["resumeTaskFromHistory"]() + + expect(resumeActivation).toHaveBeenCalledTimes(1) + // The loop entered its first turn: the denial turn posts its + // api_req_started state before the request. + expect(provider.postStateToWebviewWithoutTaskHistory).toHaveBeenCalled() + const persisted = await readPersistedApiHistory(storagePath, childId) + // The denial result is on disk exactly once: the pre-activation write + // committed it, and the loop's first turn did not append it again. + expect(countDenialResults(persisted)).toBe(1) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("active") + // The durable denial settled the pending action. + expect(persistedRecord.pendingAction).toBeUndefined() + expect(provider.taskHistoryStore.get(childId)?.status).toBe("active") + expect(provider.taskHistoryStore.get(childId)?.pendingAction).toBeUndefined() + }) + + it("stops before activation and keeps the pending action durable when the denial result fails to persist", async () => { + await provider.taskHistoryStore.upsert(makeDelegatedParent()) + await provider.taskHistoryStore.upsert(makePendingChildRecord()) + await seedTaskFiles(storagePath, childId, denialApiHistory) + + vi.mocked(fs.rename).mockImplementation(async (oldPath, newPath) => { + const targetsApiHistory = + String(newPath).endsWith(GlobalFileNames.apiConversationHistory) && String(oldPath).includes(".new_") + if (targetsApiHistory) { + throw new Error("injected api history rename failure") + } + return realRename(oldPath, newPath) + }) + + const { task, resumeActivation } = prepareDeniedTask(makePendingChildRecord()) + const request = vi.spyOn(task, "attemptApiRequest") + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("Failed to persist the denial result") + + expect(resumeActivation).not.toHaveBeenCalled() + expect(request).not.toHaveBeenCalled() + // The disk keeps the authoritative history and the retryable record. + expect(await readPersistedApiHistory(storagePath, childId)).toEqual(denialApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + expect(persistedRecord.pendingAction).toMatchObject({ actionId: finishAction.actionId }) + // The unpersisted append is dropped from memory, so no later save from + // this instance can persist the denial from here. + expect(task.apiConversationHistory).toEqual(denialApiHistory) + }) + + it("keeps the denial durable and settled when the stale parent rejects the activation", async () => { + // The parent no longer awaits the child, so the locked commit rejects. + await provider.taskHistoryStore.upsert({ ...makeDelegatedParent(), status: "active" }) + await provider.taskHistoryStore.upsert(makePendingChildRecord()) + await seedTaskFiles(storagePath, childId, denialApiHistory) + + const { task, resumeActivation } = prepareDeniedTask(makePendingChildRecord()) + const request = vi.spyOn(task, "attemptApiRequest") + + await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("no longer awaits it") + + expect(resumeActivation).toHaveBeenCalledTimes(1) + expect(request).not.toHaveBeenCalled() + // The committed denial stays on disk; it is a settled user decision, + // not an unaccepted prepared slice, so no restore removes it. + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(countDenialResults(persisted)).toBe(1) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + expect(persistedRecord.pendingAction).toBeUndefined() + }) +}) From 593ff6eefc68e09428fd4a875ca9c21402c025ac Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 04:09:49 +0000 Subject: [PATCH 11/14] fix: preserve explicit-resume ownership across restart --- ...ask-lifecycle-persisted-ownership-model.md | 3 +- packages/types/src/history.ts | 1 + scripts/check-task-lifecycle.ts | 12 ++++- src/core/task-persistence/TaskHistoryStore.ts | 25 +++++++--- .../TaskHistoryStore.realConcurrency.spec.ts | 48 +++++++++++++++++++ .../TaskHistoryStore.reconciliation.spec.ts | 44 +++++++++++++++++ .../__tests__/taskLifecycle.spec.ts | 14 +++++- src/core/task-persistence/taskLifecycle.ts | 6 ++- 8 files changed, 142 insertions(+), 11 deletions(-) diff --git a/docs/architecture/task-lifecycle-persisted-ownership-model.md b/docs/architecture/task-lifecycle-persisted-ownership-model.md index a72a9c4098..b6b5635c6d 100644 --- a/docs/architecture/task-lifecycle-persisted-ownership-model.md +++ b/docs/architecture/task-lifecycle-persisted-ownership-model.md @@ -23,7 +23,7 @@ All blocks use one vocabulary, established here so that dependent workstreams (P - **Lock-time ownership check.** A required new check on the parent record, specified under LIFE-BLK-P1-001. It runs in the parent file's merge. The gap report's dependency column for `LIFE-GAP-001` calls it the disk-authoritative ownership guard. - **Lock-time generation check.** A required new check on the child record, specified under LIFE-BLK-P1-012. It runs in the child file's merge. It is a separate check because the generation lives in the child file, and each file has its own lock. `mergeHistoryDelta` sees only one disk record per call, so one check cannot compare both files under their locks. - **Cached updater input.** Both `TaskHistoryStore.atomicReadAndUpdate` and `atomicUpdatePair` pass the updater the cached item, not the disk record. The updater result becomes a delta against the cache, and `mergeHistoryDelta` merges that delta into disk. So a reducer check inside an updater validates the cache, not disk. -- **Lifecycle-owned fields.** The fields whose authoritative writer is the lifecycle layer: `status`, `rootTaskId`, `parentTaskId`, `delegatedToId`, `childIds`, `awaitingChildId`, `completedByChildId`, `completionResultSummary`, and `pendingAction`. The block-level ownership table is under LIFE-BLK-P1-002 below. +- **Lifecycle-owned fields.** The fields whose authoritative writer is the lifecycle layer: `status`, `rootTaskId`, `parentTaskId`, `delegatedToId`, `childIds`, `awaitingChildId`, `completedByChildId`, `completionResultSummary`, `explicitResume`, and `pendingAction`. The block-level ownership table is under LIFE-BLK-P1-002 below. - **Attempt generation.** A monotonically increasing per-task token distinguishing one execution attempt of the same task ID from the next after interruption and resume. No such persisted token exists today; `PendingTaskAction.actionId` is pending-approval identity, not attempt identity (see LIFE-BLK-P1-012). ## LIFE-BLK-P1-001 — lock-time ownership check (GAP-001) @@ -71,6 +71,7 @@ Inside each write, `writeTaskFile` runs `mergeHistoryDelta` under that file's ad | `delegatedToId`, `awaitingChildId` | `delegateTaskToChild`, `completeDelegatedChild`, `abandonDelegatedChild`, and the repair paths | No | | `childIds` | The same reducers. The merge unions concurrent additions. | No | | `completedByChildId`, `completionResultSummary` | `completeDelegatedChild` and reconciliation repair. Standalone completion writes `status` only. | No | +| `explicitResume` | `resumeInterruptedTask` sets it on the explicit resume and no writer clears it. Startup reconciliation reads it in `reconcileDelegationStateCore`; the reducer model strips it because reconciliation sits outside the persisted reducer graph. | No | | `pendingAction` | `setPendingTaskAction` (`ClineProvider.ts:740`), the compare-and-clear settlement (`:752-757`), and the delegation and completion paths | No | | `id`, `number`, `task` | Creation | Creation only | | `ts` | Metadata. `taskMetadata` sets it from the last relevant message on every save (`taskMetadata.ts:69-73`). | Yes | diff --git a/packages/types/src/history.ts b/packages/types/src/history.ts index 5d9671842e..68710e5632 100644 --- a/packages/types/src/history.ts +++ b/packages/types/src/history.ts @@ -48,6 +48,7 @@ export const historyItemSchema = z.object({ awaitingChildId: z.string().optional(), // Child currently awaited (set when delegated) completedByChildId: z.string().optional(), // Child that completed and resumed this parent completionResultSummary: z.string().optional(), // Summary from completed child + explicitResume: z.boolean().optional(), // Set when the explicit user-resume path reactivated this task; startup recovery keeps the parent's awaiting link pendingAction: pendingTaskActionSchema.optional(), }) diff --git a/scripts/check-task-lifecycle.ts b/scripts/check-task-lifecycle.ts index 8478803abb..e27d1d3468 100644 --- a/scripts/check-task-lifecycle.ts +++ b/scripts/check-task-lifecycle.ts @@ -341,7 +341,17 @@ function invariantViolations(state: ModelState): string[] { } function canonical(state: ModelState): string { - return JSON.stringify(taskIds.map((id) => state[id] ?? null)) + return JSON.stringify( + taskIds.map((id) => { + const task = state[id] + if (!task) return null + // explicitResume only steers store-level startup recovery after a + // restart, which this reducer model does not represent. No modeled + // transition reads it, so it is not part of the canonical state. + const { explicitResume: _explicitResume, ...modeled } = task + return modeled + }), + ) } function formatCounterexample(message: string, trace: TraceStep[]): string { diff --git a/src/core/task-persistence/TaskHistoryStore.ts b/src/core/task-persistence/TaskHistoryStore.ts index d145d6c772..789008d60c 100644 --- a/src/core/task-persistence/TaskHistoryStore.ts +++ b/src/core/task-persistence/TaskHistoryStore.ts @@ -454,11 +454,14 @@ export class TaskHistoryStore { * - Parent `delegated` with no `awaitingChildId` → parent → `active` (invalid state) * - Parent `delegated`, child not found → parent → `active` (orphaned delegation) * - Parent `delegated`, child `completed` → parent → `active` (interrupted handoff) - * - Parent `delegated`, child `active` → child → `interrupted`, parent → `active` + * - Parent `delegated`, child `active` without `explicitResume` → child → `interrupted`, parent → `active` + * - Parent `delegated`, child `active` with `explicitResume` → child → `interrupted`, parent stays `delegated` * * A parent awaiting an `interrupted` or `delegated` child is left as-is — the child is * resumable. An `active` child is treated as orphaned during startup recovery because - * no live task session exists to own it. + * no live task session exists to own it. An `active` child marked `explicitResume` was + * reactivated through the user-resume path, so the restart only ends the lost session: + * the parent keeps its awaiting link and the next linked resume succeeds. */ private async reconcileDelegationState(persistedActiveIds: ReadonlySet): Promise { return this.withLock(() => this.reconcileDelegationStateCore(persistedActiveIds)) @@ -524,10 +527,20 @@ export class TaskHistoryStore { // behind it. Mark it interrupted before releasing the parent's delegation // link so the normal resume/re-delegate flow can take over. This is an // administrative recovery, not a runtime delegation transition. - await this.repairActiveDelegation(item, child) - console.warn( - `[TaskHistoryStore] Reconciled orphaned active child: child ${child.id} → interrupted, task ${item.id} → active`, - ) + // An explicitly resumed child keeps the parent's awaiting link: the user + // approved that pairing, so the restart only ends the lost session and + // the next linked resume must still succeed. + if (child.explicitResume) { + await this.upsertCore({ ...child, status: "interrupted" }, { skipTransitionCheck: true }) + console.warn( + `[TaskHistoryStore] Preserved explicit-resume ownership across restart: child ${child.id} → interrupted, task ${item.id} still awaits it`, + ) + } else { + await this.repairActiveDelegation(item, child) + console.warn( + `[TaskHistoryStore] Reconciled orphaned active child: child ${child.id} → interrupted, task ${item.id} → active`, + ) + } repairsInThisPass++ } else if (child.status === "completed") { await this.upsertCore( diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts index 2df6de4ef4..cf6d426a54 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.realConcurrency.spec.ts @@ -263,6 +263,54 @@ describe("TaskHistoryStore real cross-host locking", () => { } }) + it("preserves explicit-resume ownership across a restart", async () => { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-restart-ownership-")) + const storeA = new TaskHistoryStore(storagePath) + const parent = { + ...item("parent"), + status: "delegated" as const, + awaitingChildId: "child", + delegatedToId: "child", + childIds: ["child"], + } + const child = { ...item("child"), status: "interrupted" as const, parentTaskId: "parent", rootTaskId: "parent" } + + try { + await storeA.initialize() + await storeA.upsert(parent) + await storeA.upsert(child) + + // The user accepts the resume prompt; the activated record persists. + await storeA.resumeInterruptedTask("child", "parent") + expect(storeA.get("child")).toMatchObject({ status: "active", explicitResume: true }) + } finally { + storeA.dispose() + } + + // A restart replays startup recovery on the same storage. The resumed + // child's session is gone, but the user-approved awaiting link survives so + // the next linked resume still succeeds. + const storeB = new TaskHistoryStore(storagePath) + try { + await storeB.initialize() + expect(storeB.get("child")).toMatchObject({ status: "interrupted", explicitResume: true }) + expect(storeB.get("parent")).toMatchObject({ + status: "delegated", + awaitingChildId: "child", + delegatedToId: "child", + }) + + await expect(storeB.resumeInterruptedTask("child", "parent")).resolves.toMatchObject({ + id: "child", + status: "active", + explicitResume: true, + }) + } finally { + storeB.dispose() + await fs.rm(storagePath, { recursive: true, force: true }) + } + }) + it("reads the awaited child's disk record at lock time even when the cache is stale", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "redelegation-stale-cache-")) const storeA = new TaskHistoryStore(storagePath) diff --git a/src/core/task-persistence/__tests__/TaskHistoryStore.reconciliation.spec.ts b/src/core/task-persistence/__tests__/TaskHistoryStore.reconciliation.spec.ts index e37fd1a25e..7293eaba8d 100644 --- a/src/core/task-persistence/__tests__/TaskHistoryStore.reconciliation.spec.ts +++ b/src/core/task-persistence/__tests__/TaskHistoryStore.reconciliation.spec.ts @@ -275,6 +275,50 @@ describe("TaskHistoryStore reconcileDelegationState", () => { expect(persistedParent.delegatedToId).toBeUndefined() }) + it("preserves explicit-resume ownership across restart", async () => { + const child = makeItem({ + id: "child-4-resumed", + status: "active", + parentTaskId: "parent-4-resumed", + rootTaskId: "parent-4-resumed", + explicitResume: true, + }) + const parent = makeItem({ + id: "parent-4-resumed", + status: "delegated", + awaitingChildId: "child-4-resumed", + delegatedToId: "child-4-resumed", + childIds: ["child-4-resumed"], + }) + await seedItems([parent, child]) + + await store.initialize() + + // The lost session ends, but the user-approved awaiting link survives so + // the next linked resume succeeds. + expect(store.get("child-4-resumed")).toMatchObject({ + id: "child-4-resumed", + status: "interrupted", + explicitResume: true, + }) + expect(store.get("parent-4-resumed")).toMatchObject({ + id: "parent-4-resumed", + status: "delegated", + awaitingChildId: "child-4-resumed", + delegatedToId: "child-4-resumed", + }) + + const tasksDir = path.join(tmpDir, "tasks") + const persistedChild = JSON.parse( + await fs.readFile(path.join(tasksDir, "child-4-resumed", "history_item.json"), "utf8"), + ) as HistoryItem + const persistedParent = JSON.parse( + await fs.readFile(path.join(tasksDir, "parent-4-resumed", "history_item.json"), "utf8"), + ) as HistoryItem + expect(persistedChild).toMatchObject({ status: "interrupted", explicitResume: true }) + expect(persistedParent).toMatchObject({ status: "delegated", awaitingChildId: "child-4-resumed" }) + }) + it("repairs a delegated child with an omitted status as implicit active", async () => { const child = makeItem({ id: "child-implicit-active", diff --git a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts index 2c717a7d42..ba946bd636 100644 --- a/src/core/task-persistence/__tests__/taskLifecycle.spec.ts +++ b/src/core/task-persistence/__tests__/taskLifecycle.spec.ts @@ -79,6 +79,14 @@ describe("task lifecycle transitions", () => { expect(() => resumeInterruptedTask(item("child"))).toThrow(LifecycleTransitionError) }) + it("marks an explicit resume so startup recovery can preserve ownership", () => { + const child = item("child", { status: "interrupted", parentTaskId: "parent" }) + const rootTask = item("root-task", { status: "interrupted" }) + + expect(resumeInterruptedTask(child, "parent")).toMatchObject({ status: "active", explicitResume: true }) + expect(resumeInterruptedTask(rootTask)).toMatchObject({ status: "active", explicitResume: true }) + }) + it("resumes an interrupted child only while its parent still awaits it", () => { const parent = item("parent", { status: "delegated", @@ -88,7 +96,11 @@ describe("task lifecycle transitions", () => { }) const child = item("child", { status: "interrupted", parentTaskId: "parent" }) - expect(resumeInterruptedChild(parent, child)).toMatchObject({ status: "active", parentTaskId: "parent" }) + expect(resumeInterruptedChild(parent, child)).toMatchObject({ + status: "active", + parentTaskId: "parent", + explicitResume: true, + }) }) it("rejects the parent-locked resume after the parent re-delegates elsewhere", () => { diff --git a/src/core/task-persistence/taskLifecycle.ts b/src/core/task-persistence/taskLifecycle.ts index cfb57bd68d..b24abcb008 100644 --- a/src/core/task-persistence/taskLifecycle.ts +++ b/src/core/task-persistence/taskLifecycle.ts @@ -27,7 +27,9 @@ export function assertValidTransition(from: HistoryItemStatus | undefined, to: H /** * Reactivates a task only through the explicit user-resume path. This is kept * outside VALID_TASK_STATUS_TRANSITIONS so generic stale writes cannot revive - * an interrupted task. + * an interrupted task. The persisted explicitResume marker lets startup + * recovery recognize a resumed child after a restart: the session is gone, but + * the parent keeps its awaiting link so the next linked resume succeeds. */ export function resumeInterruptedTask(task: HistoryItem, expectedParentTaskId?: string): HistoryItem { if (task.status !== "interrupted") { @@ -36,7 +38,7 @@ export function resumeInterruptedTask(task: HistoryItem, expectedParentTaskId?: if (task.parentTaskId !== expectedParentTaskId) { throw new LifecycleTransitionError(`Cannot resume task ${task.id}: parent linkage changed`) } - return { ...task, status: "active" } + return { ...task, status: "active", explicitResume: true } } /** From 58d9f0eefe7838675265a578c4ce2198096494ee Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 21 Oct 2026 02:27:53 +0000 Subject: [PATCH 12/14] fix: re-interrupt abandoned child tasks after resume commits --- .../ClineProvider.delegation.spec.ts | 16 +++++ src/core/task/Task.ts | 30 +++++++++ .../__tests__/Task.pending-action.spec.ts | 63 +++++++++++++++++++ .../Task.resumeApiHistoryPersistence.spec.ts | 43 +++++++++++++ src/core/webview/ClineProvider.ts | 21 +++++++ 5 files changed, 173 insertions(+) diff --git a/src/__tests__/ClineProvider.delegation.spec.ts b/src/__tests__/ClineProvider.delegation.spec.ts index 60d530104a..d7c799cf85 100644 --- a/src/__tests__/ClineProvider.delegation.spec.ts +++ b/src/__tests__/ClineProvider.delegation.spec.ts @@ -74,6 +74,22 @@ describe("ClineProvider.resumeInterruptedTask()", () => { expect(taskHistoryStore.resumeInterruptedTask).toHaveBeenCalledWith("child-1", "parent-1") }) + it("reinterruptAbandonedChild delegates to markDelegatedChildInterrupted", async () => { + const markDelegatedChildInterrupted = vi.fn().mockResolvedValue(undefined) + const log = vi.fn() + const provider = makeProviderStub({ markDelegatedChildInterrupted, log }) + + await ClineProvider.prototype.reinterruptAbandonedChild.call(provider, { + childTaskId: "child-1", + parentTaskId: "parent-1", + }) + + expect(markDelegatedChildInterrupted).toHaveBeenCalledWith({ + childTaskId: "child-1", + parentTaskId: "parent-1", + }) + }) + it("rejects abandonment committed by another host after the parent precheck", async () => { const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), "resume-abandon-")) const storeA = new TaskHistoryStore(storagePath) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 58387c46c0..fb95070bb8 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3457,6 +3457,25 @@ export class Task extends EventEmitter implements TaskLike { ) throw error } + + if ((this.abort || this.abandoned) && this.parentTaskId) { + // The commit activated a child whose runner was aborted or + // abandoned while the commit was in flight, so the cancel or + // eviction skipped its own interruption marking. Re-interrupt + // first, then restore the pre-resume history while the record + // status still matches this resume's starting status. The + // restore removes the recovery copy only when it succeeds. + await provider.reinterruptAbandonedChild({ + childTaskId: this.taskId, + parentTaskId: this.parentTaskId, + }) + await this.restorePreparedResumeHistory( + recoveryCopyPath, + existingApiConversationHistory, + modifiedApiConversationHistory, + ) + return + } } // The resume committed, so the prepared history is the accepted @@ -3648,6 +3667,17 @@ export class Task extends EventEmitter implements TaskLike { // No API-history restore runs: the denial is a committed user // decision, not an unaccepted prepared slice. await provider.resumeInterruptedTask(this.taskId, this.parentTaskId) + if ((this.abort || this.abandoned) && this.parentTaskId) { + // The commit activated a child whose runner was aborted or + // abandoned while the commit was in flight. The denial result + // is a committed decision, so only the record needs + // re-interruption. + await provider.reinterruptAbandonedChild({ + childTaskId: this.taskId, + parentTaskId: this.parentTaskId, + }) + return + } // The denial result is already persisted, so the loop must not // append it again. await this.initiateTaskLoop(denialUserContent, { userContentAlreadyPersisted: true }) diff --git a/src/core/task/__tests__/Task.pending-action.spec.ts b/src/core/task/__tests__/Task.pending-action.spec.ts index 4e6aedc5b0..aca304f9bc 100644 --- a/src/core/task/__tests__/Task.pending-action.spec.ts +++ b/src/core/task/__tests__/Task.pending-action.spec.ts @@ -249,6 +249,69 @@ describe("Task pending action replay", () => { expect(resume.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(initiateTaskLoop).mock.invocationCallOrder[0]) }) + it("re-interrupts an abandoned delegated child after the denial activation commits", async () => { + const provider = { + resumeInterruptedTask: vi.fn().mockResolvedValue(undefined), + reinterruptAbandonedChild: vi.fn().mockResolvedValue(undefined), + } + const task = createTask(provider) + const state = task as unknown as { + initialStatus?: string + parentTaskId?: string + abandoned?: boolean + apiConversationHistory: unknown[] + addToApiConversationHistory: ReturnType + } + state.initialStatus = "interrupted" + state.parentTaskId = "parent-1" + state.abandoned = true + state.apiConversationHistory = [] + state.addToApiConversationHistory = vi.fn().mockResolvedValue(true) + task.ask = vi.fn().mockResolvedValue({ response: "noButtonClicked" }) + const initiateTaskLoop = (task as unknown as { initiateTaskLoop: ReturnType }).initiateTaskLoop + + await getPendingActionAccess(task).resumePendingTaskAction(createAction) + + expect(provider.resumeInterruptedTask).toHaveBeenCalledWith("task-1", "parent-1") + expect(provider.reinterruptAbandonedChild).toHaveBeenCalledWith({ + childTaskId: "task-1", + parentTaskId: "parent-1", + }) + const resume = vi.mocked(provider.resumeInterruptedTask) + expect(resume.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(provider.reinterruptAbandonedChild).mock.invocationCallOrder[0], + ) + expect(initiateTaskLoop).not.toHaveBeenCalled() + }) + + it("does not re-interrupt a standalone task when the activation lands after an abort", async () => { + const provider = { + resumeInterruptedTask: vi.fn().mockResolvedValue(undefined), + reinterruptAbandonedChild: vi.fn().mockResolvedValue(undefined), + } + const task = createTask(provider) + const state = task as unknown as { + initialStatus?: string + parentTaskId?: string + abort?: boolean + apiConversationHistory: unknown[] + addToApiConversationHistory: ReturnType + } + state.initialStatus = "interrupted" + state.parentTaskId = undefined + state.abort = true + state.apiConversationHistory = [] + state.addToApiConversationHistory = vi.fn().mockResolvedValue(true) + task.ask = vi.fn().mockResolvedValue({ response: "noButtonClicked" }) + const initiateTaskLoop = (task as unknown as { initiateTaskLoop: ReturnType }).initiateTaskLoop + + await getPendingActionAccess(task).resumePendingTaskAction(createAction) + + expect(provider.resumeInterruptedTask).toHaveBeenCalledWith("task-1", undefined) + expect(provider.reinterruptAbandonedChild).not.toHaveBeenCalled() + expect(initiateTaskLoop).toHaveBeenCalledTimes(1) + }) + it("stops before activation when the denial result fails to persist", async () => { const provider = { resumeInterruptedTask: vi.fn().mockResolvedValue(undefined) } const task = createTask(provider) diff --git a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts index 0b2e277859..ad572831e2 100644 --- a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts +++ b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts @@ -331,6 +331,49 @@ describe("Task resume api-history persistence on real files", () => { expect(await findRecoveryCopyName(taskDir)).toBeUndefined() }) + it("re-interrupts an aborted child and restores the pre-resume history after the commit lands", async () => { + await provider.taskHistoryStore.upsert({ + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: childId, + delegatedToId: childId, + childIds: [childId], + } as HistoryItem) + await provider.taskHistoryStore.upsert(makeChildRecord()) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + const { task, loop, resumeActivation } = prepareResumedTask(makeChildRecord()) + const reinterrupt = vi.spyOn(provider, "reinterruptAbandonedChild") + // A cancel racing the commit flips the abort flag while the locked + // activation is in flight, after the cancel's own interruption marking + // no-op'd on the still-interrupted record. + resumeActivation.mockImplementation(async (taskId: string, parentTaskId?: string) => { + await ClineProvider.prototype.resumeInterruptedTask.call(provider, taskId, parentTaskId) + task.abort = true + }) + + await task["resumeTaskFromHistory"]() + + expect(reinterrupt).toHaveBeenCalledWith({ childTaskId: childId, parentTaskId: parentId }) + expect(loop).not.toHaveBeenCalled() + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(authoritativeApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("interrupted") + expect(provider.taskHistoryStore.get(childId)?.status).toBe("interrupted") + const parent = provider.taskHistoryStore.get(parentId) + expect(parent?.status).toBe("delegated") + expect(parent?.awaitingChildId).toBe(childId) + // The restore succeeded, so the pre-resume copy was removed. + expect(await findRecoveryCopyName(taskDir)).toBeUndefined() + }) + it("stops before activation and keeps the persisted api history when the preparation write fails", async () => { await provider.taskHistoryStore.upsert(makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined })) const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 7aaee522f0..2eadfa3edf 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -758,6 +758,27 @@ export class ClineProvider } } + /** + * Re-interrupts a delegated child whose interrupted-resume commit landed + * after the child's runner was already aborted or abandoned, so the + * cancelling flow skipped its own interruption marking and left an active + * record with no runner. Delegates to markDelegatedChildInterrupted(), + * which holds the parent transition lock and only flips a child that is + * still active and still awaited by its parent. + */ + public async reinterruptAbandonedChild({ + childTaskId, + parentTaskId, + }: { + childTaskId: string + parentTaskId: string + }): Promise { + this.log( + `[reinterruptAbandonedChild] Resume committed after child ${childTaskId} was aborted or abandoned — re-interrupting under parent ${parentTaskId}`, + ) + await this.markDelegatedChildInterrupted({ childTaskId, parentTaskId }) + } + getTaskStackSize(): number { return this.taskRegistry.length } From 1e0e96854a2168c47c7b89b2c225c8343034ebc3 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 21 Oct 2026 03:21:25 +0000 Subject: [PATCH 13/14] fix: retain resume recovery history for aborted standalone tasks --- src/core/task/Task.ts | 24 +++++++------ .../Task.resumeApiHistoryPersistence.spec.ts | 35 +++++++++++++++++++ 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index fb95070bb8..cf0a5546d8 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3458,17 +3458,21 @@ export class Task extends EventEmitter implements TaskLike { throw error } - if ((this.abort || this.abandoned) && this.parentTaskId) { - // The commit activated a child whose runner was aborted or + if (this.abort || this.abandoned) { + // The commit activated a task whose runner was aborted or // abandoned while the commit was in flight, so the cancel or - // eviction skipped its own interruption marking. Re-interrupt - // first, then restore the pre-resume history while the record - // status still matches this resume's starting status. The - // restore removes the recovery copy only when it succeeds. - await provider.reinterruptAbandonedChild({ - childTaskId: this.taskId, - parentTaskId: this.parentTaskId, - }) + // eviction skipped its own interruption marking. A linked child + // is re-interrupted under its parent transition lock. A + // standalone record has no parent transition, so it stays + // active and the restore's committed_elsewhere gate keeps the + // recovery copy available. Either way the restore runs, and it + // removes the recovery copy only when it succeeds. + if (this.parentTaskId) { + await provider.reinterruptAbandonedChild({ + childTaskId: this.taskId, + parentTaskId: this.parentTaskId, + }) + } await this.restorePreparedResumeHistory( recoveryCopyPath, existingApiConversationHistory, diff --git a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts index ad572831e2..b07c00229c 100644 --- a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts +++ b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts @@ -374,6 +374,41 @@ describe("Task resume api-history persistence on real files", () => { expect(await findRecoveryCopyName(taskDir)).toBeUndefined() }) + it("keeps a standalone aborted task active and retains the recovery copy after the commit lands", async () => { + // A standalone record has no parent transition, so the committed + // active status stands. The restore must still run so its + // committed_elsewhere gate retains the recovery copy, and the task + // loop must not start on the aborted runner. + const standaloneRecord = makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined }) + await provider.taskHistoryStore.upsert(standaloneRecord) + const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + + const { task, loop, resumeActivation } = prepareResumedTask(standaloneRecord) + const reinterrupt = vi.spyOn(provider, "reinterruptAbandonedChild") + // A cancel racing the commit flips the abort flag while the + // activation is in flight. + resumeActivation.mockImplementation(async (taskId: string, parentTaskId?: string) => { + await ClineProvider.prototype.resumeInterruptedTask.call(provider, taskId, parentTaskId) + task.abort = true + }) + + await task["resumeTaskFromHistory"]() + + expect(reinterrupt).not.toHaveBeenCalled() + expect(loop).not.toHaveBeenCalled() + const persisted = await readPersistedApiHistory(storagePath, childId) + expect(persisted).toEqual(preparedApiHistory) + const persistedRecord = await readPersistedRecord(storagePath, childId) + expect(persistedRecord.status).toBe("active") + expect(provider.taskHistoryStore.get(childId)?.status).toBe("active") + // The record is active, so no restore write ran. The recovery copy + // stays on disk with the pre-resume history for recovery. + const recoveryCopyName = await findRecoveryCopyName(taskDir) + expect(recoveryCopyName).toBeDefined() + const recoveryCopy = JSON.parse(await fs.readFile(path.join(taskDir, recoveryCopyName as string), "utf8")) + expect(recoveryCopy).toEqual(authoritativeApiHistory) + }) + it("stops before activation and keeps the persisted api history when the preparation write fails", async () => { await provider.taskHistoryStore.upsert(makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined })) const taskDir = await seedTaskFiles(storagePath, childId, authoritativeApiHistory) From a17edba0adc082945f491863d472d110baa77e7a Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 21 Oct 2026 04:25:12 +0000 Subject: [PATCH 14/14] fix: prepare interrupted resume history with compare-and-set --- docs/architecture/task-lifecycle-model.md | 2 +- src/core/task/Task.ts | 41 +++++++++- .../task/__tests__/Task.persistence.spec.ts | 43 ++++++---- .../Task.resumeApiHistoryPersistence.spec.ts | 82 ++++++++++++++++++- 4 files changed, 149 insertions(+), 19 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index b5721dba71..7750649dc3 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -63,7 +63,7 @@ Production completion also accepts a recovery-compatible `active` parent that st Explicit resume validates both records under file locks. A linked child resume holds the parent file's advisory lock across the parent ownership check and the child activation, so a competing host cannot re-delegate the parent in between. The re-delegation write (`atomicUpdateParentDelegation`) re-reads the awaited child's on-disk record under the child record's own advisory lock inside the parent-file lock, so a peer write's backup-rename window cannot make a valid child record look missing, and rejects the delegation unless that child is still interrupted. The two guards share one lock order, parent file before child file, so they close the competing resume and re-delegation interleavings in both directions. The child-file merge still compares the caller's expected parent with the authoritative child backlink. The resume rejects an abandonment that committed before it, even in its partial-write window. -Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A failed preparation write also stops the resume, and the Task restores the authoritative API conversation history in memory. Before its destructive preparation write the resume first stores a durable pre-resume recovery copy of the original history in the task directory under a UUID-distinct name, so a failed preparation or restore cannot lose the original across a restart. When the activation commit rejects, the Task restores the persisted history through a conditional write whose status gate, content compare, and commit all run under the API-history file's advisory lock; the gate reads the task record under the record file's lock inside that scope, and no existing writer takes the reverse lock order. The restore commits only while the persisted record still shows the status the resume started from and the disk still holds the prepared slice, so an active record and a newer foreign history always win. An unreadable record status or an unreadable history fails the restore closed, and the restore never runs on unknown state. A record that is already active on disk keeps the prepared history, and the skipped restore is logged as a partial failure that names the recovery copy, which is retained because the active status alone does not prove this attempt's prepared history was the one accepted. A restore write that fails logs the same partial failure and names the recovery copy that still holds the original bytes. These guarantees bind writers that honor the shared advisory locks, which covers every in-repo writer of both files. The denied pending-action replay uses the same checked order: it persists the denial tool result before the activation commit, a failed append stops before activation and leaves the record interrupted with its durable pending action, and an activation rejection keeps the committed denial without an API-history restore, because the denial is a settled user decision rather than an unaccepted prepared slice. An accepted non-interrupted resume still runs without a store transition. +Representative reducer scenarios cover stale linked and standalone approvals. Two-store tests cover abandonment after the parent precheck and both resume/re-delegation orderings. A declined resume returns before any store transition for both `resume_task` and `resume_completed_task`. This acceptance gate is deliberate and replaces the previous fall-through into the task loop. An accepted resume completes its fallible history reads and writes before the activation commit, so a preparation failure leaves the record interrupted and retryable. A failed preparation write also stops the resume, and the Task restores the authoritative API conversation history in memory. The preparation write of an interrupted resume is a compare-and-set under the API-history file's advisory lock. It replaces the original history only while the disk still holds that original and the record is still interrupted, so only one of two accepted resumes can prepare and the other stops before activation without writing. Before that write the resume first stores a durable pre-resume recovery copy of the original history in the task directory under a UUID-distinct name, so a failed preparation or restore cannot lose the original across a restart. When the activation commit rejects, the Task restores the persisted history through a conditional write whose status gate, content compare, and commit all run under the API-history file's advisory lock; the gate reads the task record under the record file's lock inside that scope, and no existing writer takes the reverse lock order. The restore commits only while the persisted record still shows the status the resume started from and the disk still holds the prepared slice, so an active record and a newer foreign history always win. An unreadable record status or an unreadable history fails the restore closed, and the restore never runs on unknown state. A record that is already active on disk keeps the prepared history, and the skipped restore is logged as a partial failure that names the recovery copy, which is retained because the active status alone does not prove this attempt's prepared history was the one accepted. A restore write that fails logs the same partial failure and names the recovery copy that still holds the original bytes. These guarantees bind writers that honor the shared advisory locks, which covers every in-repo writer of both files. The denied pending-action replay uses the same checked order: it persists the denial tool result before the activation commit, a failed append stops before activation and leaves the record interrupted with its durable pending action, and an activation rejection keeps the committed denial without an API-history restore, because the denial is a settled user decision rather than an unaccepted prepared slice. An accepted non-interrupted resume still runs without a store transition. Rehydrated interrupted tasks omit lineage from message metadata saves, so their stale construction snapshots cannot restore severed links after rejection. Reverse-order stale abandonment and other live-task writers remain within the ownership/generation gaps below. diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index cf0a5546d8..95ac6f3e8c 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3409,7 +3409,18 @@ export class Task extends EventEmitter implements TaskLike { messages: existingApiConversationHistory, }) - const persisted = await this.overwriteApiConversationHistory(modifiedApiConversationHistory) + let persisted: boolean + if (this.initialStatus === "interrupted") { + // Compare-and-set under the API-history file lock. Only one of two + // accepted resumes can replace the original history, so a rejected + // attempt never leaves its own prepared history on disk. + persisted = await this.prepareInterruptedResumeHistory( + existingApiConversationHistory, + modifiedApiConversationHistory, + ) + } else { + persisted = await this.overwriteApiConversationHistory(modifiedApiConversationHistory) + } if (!persisted) { // The failed write leaves the disk file with the authoritative // history or its safeWriteJson backup. Keep the recovery copy so @@ -3498,6 +3509,34 @@ export class Task extends EventEmitter implements TaskLike { } } + private async prepareInterruptedResumeHistory(original: ApiMessage[], prepared: ApiMessage[]): Promise { + const provider = this.providerRef.deref() + if (!provider) { + return false + } + let result: ResumeHistoryRestoreResult + try { + result = await restoreApiMessagesIfUnchanged({ + taskId: this.taskId, + globalStoragePath: this.globalStoragePath, + expected: original, + replacement: prepared, + verifyCanRestore: async () => + (await provider.taskHistoryStore.readValidatedDiskStatus(this.taskId)) === "interrupted" + ? "proceed" + : "abort", + }) + } catch (error) { + console.error(`[Task#${this.taskId}] prepareInterruptedResumeHistory: write failed`, error) + return false + } + if (result.outcome !== "restored") { + return false + } + this.hydrateApiConversationHistory(prepared) + return true + } + /** * Restores the authoritative API history after the resume persisted * prepared history but never activated the record. The interrupted record diff --git a/src/core/task/__tests__/Task.persistence.spec.ts b/src/core/task/__tests__/Task.persistence.spec.ts index 6a01b58fbe..a37b09bd76 100644 --- a/src/core/task/__tests__/Task.persistence.spec.ts +++ b/src/core/task/__tests__/Task.persistence.spec.ts @@ -2509,10 +2509,16 @@ describe("Task persistence", () => { ) } + /** The interrupted resume prepares through the conditional write. The first call lands. */ + function prepareLands(): void { + mockRestoreApiMessagesIfUnchanged.mockResolvedValueOnce({ outcome: "restored" }) + } + it("stops before activation when the prepared api history fails to persist", async () => { mockReadTaskMessages.mockResolvedValue([]) mockReadApiMessages.mockResolvedValue([{ role: "user", content: "Continue" }]) - mockSaveApiMessages.mockRejectedValueOnce(new Error("disk full")) + mockRestoreApiMessagesIfUnchanged.mockRejectedValueOnce(new Error("disk full")) + vi.spyOn(console, "error").mockImplementation(() => {}) const task = makeResumeTask() const loop = vi.spyOn(getTaskPersistenceAccess(task), "initiateTaskLoop") @@ -2523,8 +2529,9 @@ describe("Task persistence", () => { expect(mockProvider.resumeInterruptedTask).not.toHaveBeenCalled() expect(loop).not.toHaveBeenCalled() - // Only the failed preparation write ran. The disk file never changed. - expect(mockSaveApiMessages).toHaveBeenCalledTimes(1) + // Only the failed conditional preparation write ran. + expect(mockSaveApiMessages).not.toHaveBeenCalled() + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) // The durable pre-resume copy exists and stays, because the // safeWriteJson rollback that restored the original on disk may // itself have failed. @@ -2547,6 +2554,7 @@ describe("Task persistence", () => { mockProvider.resumeInterruptedTask = vi .fn() .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + prepareLands() runGateInsideRestore() const task = makeResumeTask() @@ -2557,12 +2565,16 @@ describe("Task persistence", () => { expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledTimes(1) expect(loop).not.toHaveBeenCalled() // The preparation slice is the only content that reached the disk layer. - const savedMessages = vi.mocked(mockSaveApiMessages).mock.calls.map((call) => call[0].messages) - expect(savedMessages).toEqual([[]]) + expect(mockSaveApiMessages).not.toHaveBeenCalled() + const prepareOptions = vi.mocked(mockRestoreApiMessagesIfUnchanged).mock.calls[0][0] + expect(prepareOptions.expected).toEqual([ + { role: "user", content: "Continue", messageId: expect.any(String) }, + ]) + expect(prepareOptions.replacement).toEqual([]) // The restore compared against the prepared slice and carried the // authoritative history as its replacement. - expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) - const restoreOptions = vi.mocked(mockRestoreApiMessagesIfUnchanged).mock.calls[0][0] + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(2) + const restoreOptions = vi.mocked(mockRestoreApiMessagesIfUnchanged).mock.calls[1][0] expect(restoreOptions.expected).toEqual([]) expect(restoreOptions.replacement).toEqual([ { role: "user", content: "Continue", messageId: expect.any(String) }, @@ -2577,7 +2589,7 @@ describe("Task persistence", () => { .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) .mockResolvedValueOnce([{ role: "user", content: "Continue" }]) // The preparation write lands. The locked restore write then fails. - mockSaveApiMessages.mockResolvedValueOnce(undefined) + prepareLands() mockRestoreApiMessagesIfUnchanged.mockRejectedValueOnce(new Error("restore disk full")) mockProvider.resumeInterruptedTask = vi .fn() @@ -2597,8 +2609,7 @@ describe("Task persistence", () => { // through the Task save helper. expect(retry).not.toHaveBeenCalled() // The prepared slice is the last content that reached the disk layer. - const savedMessages = vi.mocked(mockSaveApiMessages).mock.calls.map((call) => call[0].messages) - expect(savedMessages).toEqual([[]]) + expect(mockSaveApiMessages).not.toHaveBeenCalled() const recoveryCopyPath = await mockCreateResumeHistoryRecoveryCopy.mock.results[0].value // The partial-failure log names the durable recovery copy, so the // original history stays identifiable on disk after a restart. @@ -2619,6 +2630,7 @@ describe("Task persistence", () => { mockProvider.taskHistoryStore.readValidatedDiskStatus = vi .fn() .mockRejectedValue(new LifecycleTransitionError("no valid disk record")) + prepareLands() runGateInsideRestore() const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) @@ -2626,10 +2638,10 @@ describe("Task persistence", () => { await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") - expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(2) // The gate rejected inside the lock scope, so the restore never // compared or wrote, and the recovery copy stays on disk. - expect(mockSaveApiMessages).toHaveBeenCalledTimes(1) + expect(mockSaveApiMessages).not.toHaveBeenCalled() expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("failed closed"), expect.anything()) consoleError.mockRestore() @@ -2644,6 +2656,7 @@ describe("Task persistence", () => { .fn() .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) mockProvider.taskHistoryStore.readValidatedDiskStatus = vi.fn().mockResolvedValue("active") + prepareLands() runGateInsideRestore() const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}) @@ -2670,6 +2683,7 @@ describe("Task persistence", () => { mockProvider.resumeInterruptedTask = vi .fn() .mockRejectedValue(new LifecycleTransitionError("parent linkage changed")) + prepareLands() mockRestoreApiMessagesIfUnchanged.mockResolvedValueOnce({ outcome: "foreign_write" }) const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}) @@ -2677,7 +2691,7 @@ describe("Task persistence", () => { await expect(task["resumeTaskFromHistory"]()).rejects.toThrow("parent linkage changed") - expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(2) // The newer disk content is never clobbered, and the original stays // recoverable in the copy. expect(mockRemoveResumeHistoryRecoveryCopy).not.toHaveBeenCalled() @@ -2699,7 +2713,8 @@ describe("Task persistence", () => { expect(mockProvider.resumeInterruptedTask).toHaveBeenCalledTimes(1) expect(loop).toHaveBeenCalledTimes(1) - expect(mockRestoreApiMessagesIfUnchanged).not.toHaveBeenCalled() + // Only the conditional preparation write ran. No restore ran. + expect(mockRestoreApiMessagesIfUnchanged).toHaveBeenCalledTimes(1) expect(mockRemoveResumeHistoryRecoveryCopy).toHaveBeenCalledTimes(1) // The copy is removed only after the activation commit landed. expect(mockRemoveResumeHistoryRecoveryCopy.mock.invocationCallOrder[0]).toBeLessThan( diff --git a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts index b07c00229c..de5cbe4d73 100644 --- a/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts +++ b/src/core/task/__tests__/Task.resumeApiHistoryPersistence.spec.ts @@ -127,6 +127,7 @@ vi.mock("../../condense", () => ({ type TaskTestAccess = { initiateTaskLoop: (userContent: unknown[]) => Promise retrySaveApiConversationHistory: () => Promise + prepareInterruptedResumeHistory: (original: ApiMessage[], prepared: ApiMessage[]) => Promise } function taskAccess(task: Task): TaskTestAccess { @@ -453,9 +454,9 @@ describe("Task resume api-history persistence on real files", () => { const { task, loop, resumeActivation } = prepareResumedTask( makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined }), ) - const realOverwrite = task.overwriteApiConversationHistory.bind(task) - vi.spyOn(task, "overwriteApiConversationHistory").mockImplementation(async (history, persist) => { - const result = await realOverwrite(history, persist) + const realPrepare = taskAccess(task).prepareInterruptedResumeHistory.bind(task) + vi.spyOn(taskAccess(task), "prepareInterruptedResumeHistory").mockImplementation(async (original, prepared) => { + const result = await realPrepare(original, prepared) await task.abortTask(true) return result }) @@ -471,6 +472,81 @@ describe("Task resume api-history persistence on real files", () => { expect(await findRecoveryCopyName(taskDir)).toBeUndefined() }) + describe.each([ + ["standalone", true], + ["linked child", false], + ])("two accepted resumes of one interrupted %s record", (_name, isStandalone) => { + const standalone = () => + isStandalone ? makeChildRecord({ parentTaskId: undefined, rootTaskId: undefined }) : makeChildRecord() + + async function raceTwoResumes() { + if (!isStandalone) { + await provider.taskHistoryStore.upsert({ + id: parentId, + number: 2, + ts: 1, + task: "Parent task", + tokensIn: 0, + tokensOut: 0, + totalCost: 0, + status: "delegated", + awaitingChildId: childId, + delegatedToId: childId, + childIds: [childId], + } as HistoryItem) + } + await provider.taskHistoryStore.upsert(standalone()) + await seedTaskFiles(storagePath, childId, authoritativeApiHistory) + const first = prepareResumedTask(standalone()) + const second = prepareResumedTask(standalone()) + // Hold both attempts at the resume prompt, so both read the same + // original history before either one prepares. + let release!: () => void + const gate = new Promise((resolve) => (release = resolve)) + let waiting = 0 + for (const { task } of [first, second]) { + vi.spyOn(task, "ask").mockImplementation(async () => { + if (++waiting === 2) { + release() + } + await gate + return { response: "yesButtonClicked" } + }) + } + const results = await Promise.allSettled([ + first.task["resumeTaskFromHistory"](), + second.task["resumeTaskFromHistory"](), + ]) + return { first, second, results } + } + + it("lets only one attempt prepare and activate", async () => { + const { first, second, results } = await raceTwoResumes() + + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1) + const rejected = results.find((r) => r.status === "rejected") as PromiseRejectedResult + expect(String(rejected.reason)).toContain("Failed to persist the prepared API conversation history") + expect(first.loop.mock.calls.length + second.loop.mock.calls.length).toBe(1) + // Both tasks share the one provider spy. + expect(first.resumeActivation).toHaveBeenCalledTimes(1) + }) + + it("leaves the winner's prepared history on disk and never the loser's", async () => { + const { first, second, results } = await raceTwoResumes() + + const winner = results[0].status === "fulfilled" ? first : second + const loser = winner === first ? second : first + expect(await readPersistedApiHistory(storagePath, childId)).toEqual(preparedApiHistory) + expect(winner.task.apiConversationHistory.map(({ role }) => role)).toEqual( + preparedApiHistory.map(({ role }) => role), + ) + // The loser wrote nothing and holds the original history in memory. + expect(loser.task.apiConversationHistory.map(({ role }) => role)).toEqual( + authoritativeApiHistory.map(({ role }) => role), + ) + }) + }) + it("reports a partial failure and proves the original bytes in the recovery copy when the restore write fails", async () => { await provider.taskHistoryStore.upsert({ id: parentId,