Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 35 additions & 26 deletions docs/architecture/task-lifecycle-model.md

Large diffs are not rendered by default.

55 changes: 29 additions & 26 deletions docs/architecture/task-lifecycle-persisted-ownership-model.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions packages/types/src/history.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export const historyItemSchema = z.object({
awaitingChildId: z.string().optional(), // Child currently awaited (set when delegated)
completedByChildId: z.string().optional(), // Child that completed and resumed this parent
completionResultSummary: z.string().optional(), // Summary from completed child
explicitResume: z.boolean().optional(), // Set when the explicit user-resume path reactivated this task; startup recovery keeps the parent's awaiting link
pendingAction: pendingTaskActionSchema.optional(),
})

Expand Down
107 changes: 101 additions & 6 deletions scripts/check-task-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,26 @@ import {
completeDelegatedChild,
delegateTaskToChild,
interruptDelegatedChild,
LifecycleTransitionError,
resumeInterruptedChild,
resumeInterruptedTask,
settleRejectedCreateSubtaskAction,
} from "../src/core/task-persistence/taskLifecycle"

const taskIds = ["parent", "child-a", "child-b"] as const
type TaskId = (typeof taskIds)[number]
type ModelState = Record<TaskId, HistoryItem | undefined>
interface ModelTask extends HistoryItem {
modelWasResumed?: boolean
}
type ModelState = Record<TaskId, ModelTask | undefined>

interface Transition {
name: string
next: ModelState
delegation?: { parentId: TaskId }
completion?: { childId: TaskId; pendingActionId?: string }
settlement?: { taskId: TaskId; actionId: string }
deniedReplay?: { taskId: TaskId; actionId: string }
}

interface TraceStep {
Expand All @@ -33,10 +40,20 @@ interface WitnessContext {
transition: Transition
}

const MAX_DEPTH = 12
const MAX_DEPTH = 13
const MAX_STATES = 10_000
const actionIds = ["action-1", "action-2"] as const
const expectedActions = ["delegate", "interrupt", "complete", "abandon", "stage", "settle-rejected"] as const
const expectedActions = [
"delegate",
"interrupt",
"resume",
"resume-stale",
"resume-denied",
"complete",
"abandon",
"stage",
"settle-rejected",
] as const
const semanticLandmarks = {
"interrupted-child-redelegation": (state: ModelState) =>
state.parent?.status === "delegated" &&
Expand All @@ -47,6 +64,12 @@ const semanticLandmarks = {
state.parent.awaitingChildId === "child-a" &&
state["child-a"]?.status === "delegated" &&
state["child-a"].awaitingChildId === "child-b",
"resumed-interrupted-child-nested-delegation": (state: ModelState) =>
state.parent?.status === "delegated" &&
state.parent.awaitingChildId === "child-a" &&
state["child-a"]?.status === "delegated" &&
state["child-a"].awaitingChildId === "child-b" &&
state["child-a"].modelWasResumed === true,
} satisfies Record<string, (state: ModelState) => boolean>
const semanticWitnesses = {
"interrupted-pending-delegation-settled": ({ prev, next, transition }: WitnessContext) =>
Expand Down Expand Up @@ -99,9 +122,23 @@ const semanticWitnesses = {
canonicalTask(after.pendingAction) === canonicalTask(beforeAction)
)
},
// The denied pending-action replay persists its denial result before the
// activation commit, so the durable denial settles exactly its matching
// action and the record activates without it.
"denied-replay-settles-matching-action": ({ prev, next, transition }: WitnessContext) => {
if (transition.deniedReplay === undefined) return false
const before = prev[transition.deniedReplay.taskId]
const after = next[transition.deniedReplay.taskId]
return (
before?.status === "interrupted" &&
before.pendingAction?.actionId === transition.deniedReplay.actionId &&
after?.status === "active" &&
after.pendingAction === undefined
)
},
} satisfies Record<string, (context: WitnessContext) => boolean>

function task(id: TaskId, parentTaskId?: TaskId): HistoryItem {
function task(id: TaskId, parentTaskId?: TaskId): ModelTask {
return {
id,
number: taskIds.indexOf(id),
Expand Down Expand Up @@ -132,7 +169,7 @@ function initialState(): ModelState {
return { parent: task("parent"), "child-a": undefined, "child-b": undefined }
}

function replace(state: ModelState, ...updates: HistoryItem[]): ModelState {
function replace(state: ModelState, ...updates: ModelTask[]): ModelState {
const next = { ...state }
for (const update of updates) next[update.id as TaskId] = update
return next
Expand Down Expand Up @@ -176,6 +213,42 @@ function transitions(state: ModelState): Transition[] {
})
}
}

const lineageParent = parent.parentTaskId ? state[parent.parentTaskId as TaskId] : undefined
const resumeValid =
parent.status === "interrupted" &&
(!parent.parentTaskId ||
(lineageParent?.status === "delegated" && lineageParent.awaitingChildId === parent.id))
if (resumeValid) {
const resumed =
lineageParent !== undefined
? resumeInterruptedChild(lineageParent, parent)
: resumeInterruptedTask(parent, undefined)
result.push({
name: `resume(${parentId})`,
next: replace(state, { ...resumed, modelWasResumed: true }),
})
// The denied replay uses the same production resume reducers, and
// its denial result is durable before the activation commit, so
// the matching pending action is settled in the same step.
for (const actionId of actionIds) {
if (parent.pendingAction?.actionId !== actionId) continue
result.push({
name: `resume-denied(${parentId}, ${actionId})`,
next: replace(state, { ...resumed, pendingAction: undefined }),
deniedReplay: { taskId: parentId, actionId },
})
}
} else if (parent.status === "interrupted" && parent.parentTaskId && lineageParent !== undefined) {
// Representative rejected event: the parent re-delegated elsewhere,
// so the store's parent-locked ownership check must reject the
// resume and leave every record unchanged.
assert.throws(() => resumeInterruptedChild(lineageParent, parent), LifecycleTransitionError)
result.push({
name: `resume-stale(${parentId})`,
next: state,
})
}
}

for (const childId of taskIds) {
Expand Down Expand Up @@ -268,7 +341,17 @@ function invariantViolations(state: ModelState): string[] {
}

function canonical(state: ModelState): string {
return JSON.stringify(taskIds.map((id) => state[id] ?? null))
return JSON.stringify(
taskIds.map((id) => {
const task = state[id]
if (!task) return null
// explicitResume only steers store-level startup recovery after a
// restart, which this reducer model does not represent. No modeled
// transition reads it, so it is not part of the canonical state.
const { explicitResume: _explicitResume, ...modeled } = task
return modeled
}),
)
}

function formatCounterexample(message: string, trace: TraceStep[]): string {
Expand Down Expand Up @@ -416,11 +499,23 @@ function runRepresentativeScenarios(): void {
assert.throws(() => delegateTaskToChild(delegated, "child-b", "active"), /not interrupted/)

const interruptedA = interruptDelegatedChild(delegated, childA)
const resumedA = resumeInterruptedTask(interruptedA, parent.id)
const resumedNested = delegateTaskToChild(resumedA, "child-b")
assert.equal(resumedNested.status, "delegated")
assert.equal(resumedNested.awaitingChildId, "child-b")
const redelegated = delegateTaskToChild(delegated, "child-b", interruptedA.status)
assert.throws(() => completeDelegatedChild(redelegated, interruptedA, "stale"), /not delegated to child/)
// The parent-locked resume rejects a child whose parent re-delegated, and
// accepts the same child while the parent still awaits it.
assert.throws(() => resumeInterruptedChild(redelegated, interruptedA), /no longer awaits/)
assert.equal(resumeInterruptedChild(delegated, interruptedA).status, "active")

const abandoned = abandonDelegatedChild(delegated, interruptedA)
assert.throws(() => completeDelegatedChild(abandoned.parent, abandoned.child, "late"), /not delegated to child/)
// A resume approved against the old linkage cannot revive an abandoned child.
assert.throws(() => resumeInterruptedTask(abandoned.child, parent.id), /parent linkage changed/)
assert.throws(() => resumeInterruptedTask(interruptedA), /parent linkage changed/)
assert.equal(resumeInterruptedTask(abandoned.child).status, "active")

const childB = task("child-b", "child-a")
const nestedParent = delegateTaskToChild(childA, childB.id)
Expand Down
74 changes: 72 additions & 2 deletions scripts/check-task-store-concurrency.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type OperationId =
| "complete-a"
| "redelegate-b"
| "stale-save-a"
| "preserve-save-a"
| "abandon-b"
| "reject-a"
type RecordMap = Partial<Record<TaskId, HistoryItem>>
Expand Down Expand Up @@ -113,6 +114,22 @@ const semanticLandmarks = {
state.commits.filter((entry) => entry.operationId === "complete-a").length === 1 &&
state.caches.A["child-a"]?.status === "completed" &&
state.caches.A.parent?.status === "delegated",
"preserve-save-commits-after-detachment": (state: ModelState) => {
const detachIndex = state.commits.findIndex(
(entry) =>
entry.operationId === "abandon-b" &&
entry.taskId === "child-a" &&
entry.next.parentTaskId === undefined &&
entry.next.rootTaskId === undefined,
)
if (detachIndex < 0) return false
const preserveIndex = state.commits.findIndex(
(entry) => entry.operationId === "preserve-save-a" && entry.taskId === "child-a",
)
if (preserveIndex <= detachIndex) return false
const disk = state.disk["child-a"]
return Boolean(disk && disk.parentTaskId === undefined && disk.rootTaskId === undefined)
},
} satisfies Record<string, (state: ModelState) => boolean>

function item(id: TaskId, overrides: Partial<HistoryItem> = {}): HistoryItem {
Expand Down Expand Up @@ -233,6 +250,23 @@ const operationSpecs: Record<OperationId, OperationSpec> = {
return [{ ...stale, tokensOut: stale.tokensOut + 1 }]
},
},
"preserve-save-a": {
id: "preserve-save-a",
host: "A",
externalSnapshot: true,
allowRefreshAfterRead: true,
buildWrites: (snapshot) => {
const stale = getRequired(snapshot, "child-a")
// A rehydrated interrupted task's message save declares its
// in-memory lineage non-authoritative. The preserve-lineage
// write owner strips parent and root links from the payload
// and delta before anything is written.
const stripped: HistoryItem = { ...stale, tokensOut: stale.tokensOut + 1 }
delete stripped.parentTaskId
delete stripped.rootTaskId
return [stripped]
Comment thread
coderabbitai[bot] marked this conversation as resolved.
},
},
"abandon-b": {
id: "abandon-b",
host: "B",
Expand Down Expand Up @@ -263,8 +297,11 @@ function prepareWrites(state: ModelState, spec: OperationSpec, operation: Operat
const cached = state.caches[spec.host][taskId]
// Task.saveClineMessages rebuilds lineage from the live Task but preserves the
// store's current status before upsert, so stale lineage is not accompanied by
// a stale status transition.
const incoming = spec.id === "stale-save-a" && cached?.status ? { ...built, status: cached.status } : built
// a stale status transition. The preserve-lineage save follows the same rule,
// so a save that commits after a peer's detachment never models a stale
// interrupted → active delta that the real store would never send.
const preservesStoreStatus = spec.id === "stale-save-a" || spec.id === "preserve-save-a"
const incoming = preservesStoreStatus && cached?.status ? { ...built, status: cached.status } : built
return {
taskId,
incoming,
Expand Down Expand Up @@ -681,6 +718,39 @@ const scenarios: Scenario[] = [
},
check: () => [],
},
{
name: "preserve save keeps detachment",
operations: [operationSpecs["preserve-save-a"], operationSpecs["abandon-b"]],
check: (state) => {
// Monotonic detachment for the preserve-lineage write owner:
// once the detach commit lands, no preserve-save commit may
// reintroduce the child's parent or root links.
for (const commit of state.commits) {
if (commit.operationId !== "preserve-save-a" || commit.taskId !== "child-a" || !commit.previous) {
continue
}
if (commit.previous.parentTaskId === undefined && commit.next.parentTaskId !== undefined) {
return ["preserve-lineage save restored a detached parent link"]
}
if (commit.previous.rootTaskId === undefined && commit.next.rootTaskId !== undefined) {
return ["preserve-lineage save restored a detached root link"]
}
}
if (state.operations["abandon-b"]?.phase === "done") {
const disk = state.disk["child-a"]
if (!disk) return ["preserve save dropped the detached child record"]
if (disk.parentTaskId !== undefined || disk.rootTaskId !== undefined) {
return ["preserve-lineage save left reattached lineage after abandonment"]
}
}
// The production save carries the store's current status, so it
// never models a status delta and is never rejected.
if (state.operations["preserve-save-a"]?.phase === "rejected") {
return ["preserve-lineage save was rejected for a stale status delta"]
}
return []
},
},
]

let totalStates = 0
Expand Down
Loading
Loading