From c92b29b1f4c2a25eafc84335c900bfe29cd7fb4c Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 16 Oct 2026 15:16:20 +0800 Subject: [PATCH 1/7] refactor(runtime): remove unused tool reconciliation writer Keep unknown-outcome sealing and model projection, retain historical recovery readers, and limit current tools to unknown-reporting policies. Generated-by: Codex --- .../desktop/src/main/client-settings-tools.ts | 1 - .../runtime-recovery-resolver-adr.zh-CN.md | 3 + .../runtime-resume-architecture.md | 146 +++-------- .../runtime-resume-architecture.zh-CN.md | 155 +++--------- .../runtime-resume-extraction-ledger.zh-CN.md | 3 + ...hase4-workspace-checkpoint-design.zh-CN.md | 3 + packages/core/src/runtime-event-store.ts | 13 - packages/core/src/runtime-event.ts | 5 +- packages/core/src/tool-ledger-scanner.ts | 16 +- packages/core/src/tool-recovery-bundle.ts | 21 +- .../execution-model-composition.test.ts | 2 +- .../src/server/agent-settings-tools.ts | 1 - .../recovery-authority-equivalence.test.ts | 49 ++-- .../src/__tests__/recovery-resolver.test.ts | 10 +- .../tool-runtime-durable-boundary.test.ts | 40 ++- .../tool-runtime-sqlite-boundary.test.ts | 6 +- .../runtime/src/agent-swarm-status-tool.ts | 1 - packages/runtime/src/mcp-tools.ts | 4 +- packages/runtime/src/memory-extraction.ts | 2 - packages/runtime/src/plan-tools.ts | 3 - packages/runtime/src/recovery-resolver.ts | 5 +- .../src/stream-graph-supervisor-tools.ts | 3 - packages/runtime/src/tool-runtime.ts | 15 +- .../sqlite-recovery-concurrency-child.ts | 106 -------- .../recovery-persistence-authority.test.ts | 233 ++++++------------ .../sqlite-recovery-concurrency.test.ts | 67 ----- .../__tests__/sqlite-runtime-crash.test.ts | 125 ---------- ...pace-version-authority-persistence.test.ts | 11 - .../storage/src/agent-run-store-contract.ts | 2 +- packages/storage/src/sqlite-runtime-store.ts | 212 +--------------- .../storage/src/tool-commit-validation.ts | 2 +- 31 files changed, 256 insertions(+), 1009 deletions(-) diff --git a/apps/desktop/src/main/client-settings-tools.ts b/apps/desktop/src/main/client-settings-tools.ts index f733c5f6da7..c3e8b43a0bd 100644 --- a/apps/desktop/src/main/client-settings-tools.ts +++ b/apps/desktop/src/main/client-settings-tools.ts @@ -70,7 +70,6 @@ export function buildClientSettingsTools( 'Runtime behavior, credentials, model configuration, network settings, and Bot settings are excluded.', parameters: z.object({}).strict(), categoryHint: 'read', - recoveryMode: 'replay_safe', impl: async () => project(await authority.read()), }; const updateTool: MakaTool = { diff --git a/docs/architecture/runtime-recovery-resolver-adr.zh-CN.md b/docs/architecture/runtime-recovery-resolver-adr.zh-CN.md index 58940c287a3..a98cb1311fb 100644 --- a/docs/architecture/runtime-recovery-resolver-adr.zh-CN.md +++ b/docs/architecture/runtime-recovery-resolver-adr.zh-CN.md @@ -19,6 +19,9 @@ # ADR:RuntimeEvent 唯一事实源与 RecoveryResolver 唯一判定权威 +> **2026-10-16 退役说明:** 本文涉及工具专属 reconciliation、recovery bundle 写入及其后续 PR 的内容保留为历史设计,不再是当前实施要求。没有生产工具提供完整的 reconciliation 合同,公开 bundle writer 和预设工具恢复能力已清理。已有 recovery facts 仍保留读取、复制与投影重建兼容。当前合同是结束旧执行、保留未知事实、禁止自动重放,由新的明确用户轮检查现状;详见[当前恢复架构](./runtime-resume-architecture.zh-CN.md#未知结果与历史-recovery-bundle)。 + + - 状态:Accepted,Phase 2.5 开始实施 - 日期:2026-07-17 - 范围:Runtime Resume Phase 2.5–3 diff --git a/docs/architecture/runtime-resume-architecture.md b/docs/architecture/runtime-resume-architecture.md index b77d94afc4e..7e31b4f9afc 100644 --- a/docs/architecture/runtime-resume-architecture.md +++ b/docs/architecture/runtime-resume-architecture.md @@ -4,10 +4,10 @@ title: "Chapter 8: Resume Is Not Retry—How Maka Continues Safely from Crash Fa language: en source_language: zh-CN counterpart: ./runtime-resume-architecture.zh-CN.md -implementation_status: phase_0_2_and_phase_3a_authority_current +implementation_status: safe_boundary_and_unknown_outcome_current document_status: current translation_status: synced -last_verified: 2026-10-10 +last_verified: 2026-10-16 owners: - maka-backend --- @@ -32,17 +32,17 @@ owners: # Chapter 8: Resume Is Not Retry—How Maka Continues Safely from Crash Facts -Tracking: [Production Write/Edit recovery #4319](https://github.com/apache/maka/issues/4319), [safe-boundary continuation hardening #4324](https://github.com/apache/maka/issues/4324), [sandbox boundary negotiation #3731](https://github.com/apache/maka/issues/3731) +Tracking: [safe-boundary continuation hardening #4324](https://github.com/apache/maka/issues/4324), [sandbox boundary negotiation #3731](https://github.com/apache/maka/issues/3731) > This chapter answers a deceptively dangerous question: when Maka crashes while a model is calling a tool, how can a restart tell what happened, what may continue, and what must stop for human attention? The answer is: **recover facts from immutable RuntimeEvents, let one RecoveryResolver classify tool state, and create a new Run only when history, execution, and workspace boundaries are all provably safe. Resume never resurrects the old process or disguises “try again” as recovery.** This chapter is for engineers entering Maka Runtime for the first time. The first half builds intuition with an interrupted file write. The second half explains Phases 0–4, Desktop and CLI integration, T1/T2, recovery decisions, workspace checkpoints, and the recommended implementation sequence. -It describes `main` as verified on 2026-10-10: +It describes the current recovery contract as verified on 2026-10-16: - Phases 0–2 are implemented. -- Phase 3A recovery-fact atomic write authority and the Resolver are implemented. -- The production Phase 3 reconciler, file evidence, and complete host-owner lifecycle remain future work. +- The Resolver classifies immutable tool facts; T1 without T2 remains unknown. +- Tool-specific reconciliation and the unused recovery-bundle writer have been retired. Existing recovery bundles remain readable, copyable, and rebuildable as historical facts. - Phase 4 Git checkpoints, isolated restore, and durable rebaseline are not implemented. Roadmap documents describe targets. Code and contract tests remain the authority for current behavior. This chapter always separates implemented behavior from planned work. @@ -120,7 +120,7 @@ The rest of the chapter repeatedly uses these terms: | high-water | The exact immutable-log position covered by a plan | | park | Preserve state and stop automatic execution pending stronger evidence or a human | | fail closed | Refuse to continue under uncertainty instead of guessing that it is safe | -| reconcile | Observe the outside world again to determine whether an earlier side effect completed | +| reconcile (historical design) | Tool-specific external-state inspection; no current producer | | continuation | A new execution built from trusted history, not an old execution revived in place | So “create a continuation fail-closed from a canonical high-water” simply means: @@ -135,30 +135,30 @@ These three words are easy to mix up: |---|---|---| | Repair | Durable state of an old Run | Give an interrupted Run its terminal RuntimeEvent and align Turn state | | Resume / Continuation | A history boundary already proved safe | Create fresh identities and continue the provider loop | -| Reconcile | A tool operation with T1 but no T2 outcome | Observe the external world and commit either completed or parked | +| Reconcile (historical) | An already-persisted recovery bundle | Preserve its interpretation; no current reconciliation writer | -The usual order is repair, then resolve or reconcile, and only then resume. Repair alone is already useful: the UI no longer shows “running” forever, and the user can inspect an explicit failed or interrupted Turn. +The current order is repair, classify immutable facts, and then either admit a safe continuation or keep the old execution sealed as unknown. Repair alone is already useful: the UI no longer shows “running” forever, and the user can inspect an explicit failed or interrupted Turn. ## What Phases 0–4 actually mean -The phases are not five separate implementations. Each one adds a kind of fact the system can prove. +These phase names record the historical design. The current contract retains safe continuation and unknown-outcome handling; tool-specific reconciliation has been retired. | Phase | Question it answers | Current status | |---|---|---| | Phase 0 | Given only committed RuntimeEvents, is this prefix safe to replay? | Implemented | | Phase 1 | At a complete safe boundary, may the Runtime create a new Run? | Implemented, feature-flagged | | Phase 2 | Can T1 be guaranteed before tool execution and T2 before returning the result? | Implemented in SQLite canonical mode | -| Phase 2.5 / 3A PR A | Who owns recovery facts, how do conflicts fail closed, and how is a bundle atomic? | Implemented | -| Later Phase 3 | Can tool-specific evidence settle an unknown side effect? | Designed; no production reconciler wiring | +| Phase 2.5 / 3A PR A | Who owns recovery facts and how do conflicts fail closed? | Resolver retained; bundle writing retired | +| Later Phase 3 | Can tool-specific evidence settle an unknown side effect? | Retired design; no current producer | | Phase 4 | Can a Runtime boundary bind to a workspace snapshot for restore or rebaseline? | Designed | ```mermaid flowchart LR - P0["Phase 0
interpret committed history"] --> P1["Phase 1
create execution at a complete boundary"] - P1 --> P2["Phase 2
T1/T2 bound the side-effect window"] - P2 --> P3A["Phase 3A foundation
single recovery authority + atomic bundle"] - P3A --> P3["Phase 3 recovery
tool-specific evidence / reconcile"] - P3 --> P4["Phase 4 workspace continuity
checkpoint / restore / rebaseline"] + P0["Phase 0
interpret committed history"] --> P1["Phase 1
new execution at a safe boundary"] + P1 --> P2["Phase 2
durable T1/T2 boundary"] + P2 --> Unknown["Seal unknown outcomes
fresh explicit user turn inspects state"] + P2 -. "historical read compatibility" .-> Legacy["Existing recovery bundles"] + P2 -. "independent design" .-> P4["Workspace continuity"] ``` Phase 2 does not replace Phases 0 and 1. Their replay and continuation gates remain active. Phase 2 gives those gates better evidence about whether execution crossed the tool-dispatch boundary. @@ -216,7 +216,7 @@ Safety does not come merely from putting everything in SQLite. It comes from ass The API and database enforce RuntimeEvent authority: - generic writers cannot persist dispatch, operation-linked outcome, or recovery facts; -- T1, T2, and recovery bundle each have a dedicated writer; +- T1 and T2 each have a dedicated writer; historical recovery facts are read-only; - journal rows reference their RuntimeEvent; - exact retry must match bytes and identity; - conflicting retry, orphan facts, lane smuggling, and identity drift are rejected; @@ -345,7 +345,7 @@ stateDiagram-v2 end note note right of Dispatched No response: indeterminate. - Reconcile or park. + Seal as unknown; never replay automatically. end note note right of Outcome Completed. @@ -359,11 +359,11 @@ The Resolver decision table: |---|---|---| | call + matching response, no dispatch | `completed` | Pre-T1 synthetic result or legacy completed result | | call + dispatch + matching response | `completed` | Reuse; do not rerun | -| call + dispatch, no response | `indeterminate` | Requires reconcile; continuation currently blocks | +| call + dispatch, no response | `indeterminate` | Seal as unknown; block automatic continuation; a fresh explicit user turn may inspect state | | call, no dispatch/response, first event declares new protocol | `definitely_not_dispatched` | Proven not to have crossed T1; automatic policy still needs an explicit later phase | | same gap under legacy/unknown protocol | `indeterminate` | Absence of an old dispatch fact proves nothing | -| completed recovery bundle | `completed` | Use its matching outcome | -| parked recovery bundle | `parked` | Permanent v1 stop; no second attempt | +| historical completed recovery bundle | `completed` | Use its matching outcome | +| historical parked recovery bundle | `parked` | Permanent v1 stop; no second attempt | | orphan, duplicate, identity/hash/order conflict | `corruption` | Fail closed | The legacy rule matters: only a Run whose first event declares `toolBoundary: "t1_after_preflight_v1"` may interpret missing dispatch as definitely not dispatched. @@ -599,82 +599,19 @@ retired with the file-backed store: `runtime-event-persistence` opens the SQLite store directly, all readers use SQLite, and nothing merges or falls back to stale JSONL. -## Phase 3A: atomically commit recovery facts +## Unknown outcomes and historical recovery bundles -Phase 3A implements how trustworthy recovery output is stored. It does not yet wire a production observer/reconciler that generates the output. +The current contract is deliberately limited: after a crash, end the old execution, retain uncertainty, and let a fresh explicit user turn inspect current state. Runtime does not promise to settle the old side effect automatically. T1 proves entry into the dispatch boundary, not that the tool completed. -A recovery bundle contains: +Current tool declarations accept only `never_auto_retry` and `outcome_unknown`. The former is the default. Client Capability proxies use the latter. Historical dispatches may still contain `replay_safe`, `idempotent`, `reconcile`, or `reattach`; decoding them never authorizes execution. -1. a reconcile observation; -2. an optional recovered outcome, only for proven completion; -3. a terminal `completed` or `parked` decision. +The retired Phase 3A writer could atomically append an observation, an optional successful outcome, and a completed or parked decision. No production tool declared a working reconciliation contract. Its public writer, write transitions, and failpoints have been removed. The shared scanner, bundle validator, Resolver, conversation-copy rewriting, and projection rebuild retain read compatibility with existing facts. Generic append paths continue to reject new recovery facts. -```mermaid -sequenceDiagram - participant R as Future reconciler - participant Store as SqliteRuntimeStore - participant Scan as Shared scanner / interpreter - participant Events as runtime_events - participant Proj as journal / operation projection - - R->>Store: commitToolRecoveryBundle - Store->>Scan: validate call + dispatch + observation + outcome? + decision - Scan-->>Store: valid / corruption - alt valid completed - Store->>Events: append reconcile - Store->>Events: append matching successful outcome - Store->>Events: append completed decision - Store->>Proj: update recovery_completed in same transaction - else valid parked - Store->>Events: append reconcile - Store->>Events: append parked decision - Store->>Proj: update recovery_parked in same transaction - else invalid - Store-->>R: roll back and reject - end -``` - -The writer, projection rebuild, and Resolver share one scanner/interpreter, so online, reopen, rebuild, and Resolver must agree on the same immutable ledger. - -`parked` is terminal in v1. Only an exact bundle retry may converge idempotently. Reopening recovery would require a new versioned fact. - -## Later Phase 3: file recovery only finalizes - -Write/Edit recovery first needs durable evidence bound to: - -- workspace identity and canonical target; -- operation/call/dispatch identity; -- before and expected-after identity; -- transform/algorithm version; -- production-shaped result; -- size, regular-file, symlink, and encoding boundaries. - -| Observation | Action | -|---|---| -| `matches_expected_state` | Cleanup/finalize only; synthesize outcome and commit completed bundle | -| `matches_prior_state` | Park with `reconcile_matches_prior_state` | -| `diverged` | Park; do not overwrite outside changes | -| `unreadable` | Park; do not guess | - -```mermaid -flowchart TD - T1["T1 committed, T2 missing"] --> Observe["Read durable evidence
observe current file"] - Observe --> Expected{"current == expected-after?"} - Expected -->|"Yes"| Finalize["Finalize only
do not write the file again"] - Finalize --> Completed["Commit recovered outcome
+ completed decision"] - Expected -->|"No"| Prior{"current == before?"} - Prior -->|"Yes"| ParkPrior["Park
reconcile_matches_prior_state"] - Prior -->|"No, content diverged"| ParkDiverged["Park
protect outside writes"] - Prior -->|"Unreadable"| ParkUnreadable["Park
do not guess"] -``` - -Atomic rename prevents torn files but does not provide conditional replacement. Without CAS, matching the prior state cannot authorize automatic redo. - -Generic Bash, arbitrary remote APIs, send, publish, pay, and delete continue to park without a dedicated protocol. +Ordinary T1-without-T2 recovery seals the invocation as `outcome_unknown` without writing a tool response. A subsequent explicit user turn receives a request-local unknown response. That projection retires after a later fresh, lineage-free user invocation completes; a failed turn does not retire it. Client Capability recovery instead persists an error response with `uncertainOutcome: { code: 'outcome_unknown', retrySafe: false }`. These expressions currently differ in persistence and lifetime. Neither confirms that the original effect succeeded, and this cleanup does not unify them. ## Phase 4: bind Runtime history to the workspace -Phase 1 proves history completeness. Phase 3 may settle one operation. Long tasks also need workspace-wide continuity. +Phase 1 proves history completeness. Unknown tool effects remain unknown. Long tasks may separately need workspace-wide continuity. ```ts interface WorkspaceBoundary { @@ -836,27 +773,15 @@ Eval does not resume or reconstruct Runtime execution. It asks Runtime Host to e 7. End a Run by committing exactly one terminal RuntimeEvent. 8. On restart, repair the old Run first. 9. Resolve immutable facts into completed / not-dispatched / indeterminate / parked / corruption. -10. If a production reconciler exists, commit one atomic recovery bundle; otherwise park. +10. Seal T1-without-T2 as unknown; never automatically replay the effect. A fresh explicit user turn can inspect state. 11. Check replay legality, workspace, tool catalog, and background work. 12. Revalidate and claim immediately before execution. 13. Commit continuation-start before calling the provider. 14. At any unprovable boundary, preserve facts and emit a machine-readable park reason. -## How later work should be split +## Boundaries for future work -```mermaid -flowchart TD - A["PR A
Recovery persistence authority
complete"] --> B["PR B
Immutable cursor + durable claim
merged"] - A --> C["PR C
File evidence + finalize-only recovery"] - B --> E["PR E
Checkpoint contracts"] - C --> E - E --> F["PR F
Canonical checkpoint bundle"] - F --> G["PR G
Observe-only Git carrier"] - G --> H["PR H
Capture + retention"] - H --> Restore["Isolated restore"] - H --> Rebaseline["Durable rebaseline"] - D["PR D
Host owner lifecycle"] -. "Required before default capture / auto-resume" .-> H -``` +The former PR C file-reconciliation plan is historical. Workspace continuity and Host ownership can be evaluated independently; they do not require a generic tool reconciler. Any future operation-specific recovery proposal needs a concrete production consumer and an explicit external protocol before adding Runtime authority. Each PR must name: @@ -929,9 +854,7 @@ UI copy may change; machine codes must remain stable for tests, telemetry, dashb | Phase 0 deterministic replay / unit contract | Supported | Supported | Supported | | Phase 0 process-crash committed-prefix harness | Supported | Supported | Covered, but not a power-loss claim | | Phase 1 local safe-boundary continuation | Primary target | Primary target | Limited / best-effort | -| SQLite T1/T2 and recovery-bundle semantics | Supported | Supported | Semantic support | -| Recovery-bundle SIGKILL transaction proof | Release proof platform | Release proof platform | Currently skipped as limited support | -| Phase 3 file finalize-only recovery | Not wired in production | Not wired in production | Not wired in production | +| SQLite T1/T2 and historical recovery-bundle reading | Supported | Supported | Semantic support | | Phase 4 Git workspace continuity | Not implemented | Not implemented | Not implemented | Process crash and SQLite transaction atomicity do not automatically prove power-loss durability. Filesystem and hardware behavior need separate tests. @@ -950,10 +873,7 @@ Current implementation does not promise: - model self-report as a substitute for RuntimeEvent, file evidence, or an external receipt; - green CI as a substitute for concurrency, crash, and data-safety arguments. -The two most important follow-ups are: - -1. PR B has landed: immutable event-seq boundaries, a domain-separated prefix digest, the SQLite unique claim (`runtime_continuation_claims`), and unified ancestor replay through `continuation_start_v2`; -2. PR C/D: production file evidence/reconciler and one complete host-owner lifecycle. +PR B has landed: immutable event-seq boundaries, a domain-separated prefix digest, the SQLite unique claim (`runtime_continuation_claims`), and unified ancestor replay through `continuation_start_v2`. Tool-specific reconciliation is outside the current product contract. ## Code-reading map diff --git a/docs/architecture/runtime-resume-architecture.zh-CN.md b/docs/architecture/runtime-resume-architecture.zh-CN.md index b9b52bf2260..4c1cf77c9c9 100644 --- a/docs/architecture/runtime-resume-architecture.zh-CN.md +++ b/docs/architecture/runtime-resume-architecture.zh-CN.md @@ -4,10 +4,10 @@ title: "第八章:Resume 不是重试——Maka 如何从崩溃事实安全继 language: zh-CN source_language: zh-CN counterpart: ./runtime-resume-architecture.md -implementation_status: phase_0_2_and_phase_3a_authority_current +implementation_status: safe_boundary_and_unknown_outcome_current document_status: current translation_status: synced -last_verified: 2026-10-10 +last_verified: 2026-10-16 owners: - maka-backend --- @@ -32,17 +32,17 @@ owners: # 第八章:Resume 不是重试——Maka 如何从崩溃事实安全继续 -跟踪:[生产级 Write/Edit 恢复 #4319](https://github.com/apache/maka/issues/4319)、[safe-boundary continuation 加固 #4324](https://github.com/apache/maka/issues/4324)、[sandbox boundary negotiation #3731](https://github.com/apache/maka/issues/3731) +跟踪:[safe-boundary continuation 加固 #4324](https://github.com/apache/maka/issues/4324)、[sandbox boundary negotiation #3731](https://github.com/apache/maka/issues/3731) > 本章回答一个看起来简单、实际上很危险的问题:Maka 在模型调用工具时崩溃,重启后怎样知道哪些事情已经发生、哪些事情可以继续、哪些事情必须停下来等人处理?核心答案是:**先从不可变的 RuntimeEvent 恢复事实,再由唯一的 RecoveryResolver 判定工具状态;只有历史、执行和 workspace 三条边界都能证明安全时,才创建新的 Run 继续。Resume 从不复活旧进程,也不把“再试一次”伪装成恢复。** 本文面向第一次接触 Maka Runtime 的工程师。前半部分用一个文件写入例子建立直觉,后半部分说明 Phase 0–4、Desktop/CLI 接线、T1/T2、恢复判定、workspace checkpoint 和工程实施顺序。 -本文描述截至 2026-10-10 的 `main`: +本文描述于 2026-10-16 核验的恢复合同: - Phase 0–2 已落地; -- Phase 3A 的 recovery fact 原子写入权威与 Resolver 已落地; -- Phase 3 的生产 reconciler、文件 evidence 和完整 host owner lifecycle 仍在后续路线中; +- Resolver 根据 immutable tool facts 分类;T1 无 T2 的结果保持未知; +- 工具专属 reconciliation 与无生产消费者的 recovery bundle writer 已退役;已有 bundle 仍可读取、复制和重建投影; - Phase 4 的 Git checkpoint、隔离恢复和 durable rebaseline 尚未落地。 路线文档写的是目标,代码和 contract tests 才是当前事实。本文会始终把“已经实现”和“计划实现”分开。 @@ -120,7 +120,7 @@ flowchart TD | high-water | 这次计划明确读到了不可变日志的哪个位置 | | park | 保留现状,停止自动执行,等待更强证据或人工处理 | | fail closed | 不确定时拒绝继续,而不是猜一个“可能没事”的答案 | -| reconcile | 重新观察外部世界,判断之前的副作用究竟有没有完成 | +| reconcile(历史设计) | 工具专属的外部状态检查;当前没有生产者 | | continuation | 从可信历史创建的新执行,不是旧执行原地复活 | 因此,“从 canonical high-water fail closed 地创建 continuation”并不神秘,意思只是: @@ -135,32 +135,32 @@ flowchart TD |---|---|---| | Repair | 旧 Run 的持久化状态 | 给被中断的 Run 补上 terminal RuntimeEvent,并对齐 Turn 状态 | | Resume / Continuation | 一段已经证明安全的历史边界 | 创建新身份,继续 provider loop | -| Reconcile | T1 已派发但没有 T2 outcome 的工具操作 | 观察外部世界,提交 completed 或 parked recovery decision | +| Reconcile(历史) | 已持久化的 recovery bundle | 保留原有解释;当前没有 reconciliation writer | -典型顺序是先 repair,再 resolve/reconcile,最后才可能 resume。系统启动时即使只完成了 repair,也已经有价值:UI 不会永远显示“运行中”,用户可以检查一个明确失败或中断的 Turn。 +当前顺序是先 repair,再根据 immutable facts 分类,随后安全续接,或将旧执行封为未知。系统启动时即使只完成了 repair,也已经有价值:UI 不会永远显示“运行中”,用户可以检查一个明确失败或中断的 Turn。 ## Phase 0–4 到底各做什么 -Phase 不是五套实现,而是逐层增加“可以证明的事实”。 +这些阶段名称记录历史设计演进;当前保留安全续接与未知结果处理,工具专属 reconciliation 已退役。 | 阶段 | 它回答的问题 | 当前状态 | |---|---|---| | Phase 0 | 只看已提交 RuntimeEvent 前缀,这段历史能安全 replay 吗? | 已实现 | | Phase 1 | 在完整、安全边界上,能否创建一个新 Run 继续? | 已实现,feature flag 控制 | | Phase 2 | 能否保证工具执行前有 T1、结果返回模型前有 T2? | 已实现,SQLite canonical 模式 | -| Phase 2.5 / 3A PR A | 谁拥有恢复事实?冲突怎样 fail closed?恢复 bundle 怎样原子提交? | 已实现 | -| Phase 3 后续 | 能否用专属 evidence 判断未知副作用,并完成或永久 park? | 设计中,生产 reconciler 未接线 | +| Phase 2.5 / 3A PR A | 谁拥有恢复事实?冲突怎样 fail closed? | 保留 Resolver;bundle 写入已退役 | +| Phase 3 后续 | 能否用专属 evidence 判断未知副作用? | 历史设计已退役;没有当前生产者 | | Phase 4 | 能否把 Runtime 边界绑定到 workspace snapshot,并隔离恢复或 rebaseline? | 设计中 | 从能力上看,它们形成一条递进关系: ```mermaid flowchart LR - P0["Phase 0
能解释 committed history"] --> P1["Phase 1
完整边界可新建执行"] - P1 --> P2["Phase 2
T1/T2 固化副作用窗口"] - P2 --> P3A["Phase 3A foundation
唯一恢复权威与原子 bundle"] - P3A --> P3["Phase 3 recovery
专属 evidence / reconcile"] - P3 --> P4["Phase 4 workspace continuity
checkpoint / restore / rebaseline"] + P0["Phase 0
解释 committed history"] --> P1["Phase 1
安全边界上的新执行"] + P1 --> P2["Phase 2
T1/T2 持久化边界"] + P2 --> Unknown["封口未知结果
新的明确用户轮检查"] + P2 -. "历史读取兼容" .-> Legacy["旧 recovery bundle"] + P2 -. "独立设计" .-> P4["Workspace continuity"] ``` Phase 0 和 Phase 1 并没有因为 Phase 2 出现而失效。它们仍是 replay 和 continuation 的安全闸门;Phase 2 只是让闸门获得了更精确的“是否跨过工具派发边界”证据。 @@ -218,7 +218,7 @@ Resume 安全性的核心不是“数据都写进 SQLite”,而是每类数据 “RuntimeEvent 是唯一事实源”不是一句注释,而是由 API 和数据库约束落实: - generic writer 不能写 dispatch、operation-linked outcome 或 recovery fact; -- T1/T2 和 recovery bundle 有各自专用 writer; +- T1/T2 有各自专用 writer;历史 recovery facts 只读; - journal row 必须引用对应 RuntimeEvent; - exact retry 必须字节和身份一致; - 冲突 retry、orphan event、lane smuggling 和 identity 漂移全部拒绝; @@ -349,7 +349,7 @@ stateDiagram-v2 end note note right of Dispatched 无 response:indeterminate - 需要 reconcile 或 park + 封为未知;禁止自动重放 end note note right of Outcome completed @@ -363,11 +363,11 @@ Resolver 的核心决策表是: |---|---|---| | call + matching response,无 dispatch | `completed` | T1 前 synthetic result 或 legacy completed result | | call + dispatch + matching response | `completed` | 复用结果,不重跑 | -| call + dispatch,无 response | `indeterminate` | 需要专属 reconcile;当前 continuation 会阻断 | +| call + dispatch,无 response | `indeterminate` | 封为未知,阻断自动 continuation;新的明确用户轮可以检查现状 | | call,无 dispatch、无 response,首事件声明新协议 | `definitely_not_dispatched` | 事实可证明未跨 T1;自动策略仍需由后续阶段明确 | | call,无 dispatch、无 response,legacy/unknown protocol | `indeterminate` | 不能用“没看到 dispatch”证明旧版本没执行 | -| recovery bundle 得出 completed | `completed` | 使用 bundle 中匹配的 outcome | -| recovery bundle 得出 parked | `parked` | v1 永久停车,不产生第二次 attempt | +| 历史 recovery bundle 得出 completed | `completed` | 使用 bundle 中匹配的 outcome | +| 历史 recovery bundle 得出 parked | `parked` | v1 永久停车,不产生第二次 attempt | | orphan、duplicate、identity/hash/order 冲突 | `corruption` | fail closed | 这里最值得记住的是 legacy 规则:新协议的首事件带有 `toolBoundary: "t1_after_preflight_v1"`,系统才有资格把“缺少 dispatch”解释成“确定没跨 T1”。旧日志没有这个承诺,所以同样的缺口只能保守判成未知。 @@ -596,85 +596,19 @@ host 把 SQLite store 同时接成 `RuntimeEventStore` 和 `RuntimeCommitSink` `runtime-event-persistence` 直接打开 SQLite store,所有 reader 都只读 SQLite, 不会再与过期 JSONL 合并或 fallback。 -## Phase 3A:恢复事实怎样原子提交 +## 未知结果与历史 recovery bundle -Phase 3A 已经实现“恢复结果怎样写才可信”,但还没有实现生产级 observer/reconciler 去自动产生这些结果。 +当前合同明确限定为:崩溃后结束旧执行,保留未知事实,由用户发起新的明确一轮,让模型检查现状。Runtime 不承诺自动结算旧副作用。T1 只证明已进入派发边界,不能证明工具执行完成。 -一个 recovery bundle 最多包含三条 immutable facts: +当前工具声明只接受 `never_auto_retry` 和 `outcome_unknown`,前者为默认值,Client Capability 代理使用后者。历史 dispatch 仍可能带有 `replay_safe`、`idempotent`、`reconcile` 或 `reattach`;能够解码这些值不构成执行授权。 -1. reconcile observation:当前外部状态看起来是什么; -2. 可选 recovered outcome:只有确实能证明 completed 时才存在; -3. terminal recovery decision:`completed` 或 `parked`。 +已退役的 Phase 3A writer 曾支持原子追加 observation、可选的成功 outcome,以及 completed 或 parked decision。没有生产工具提供完整的 reconciliation 合同,其公开 writer、写入 transition 和 failpoint 已移除。共享 scanner、bundle validator、Resolver、conversation copy 引用重写和投影重建继续解释已有事实。普通 append 路径仍拒绝新增 recovery facts。 -```mermaid -sequenceDiagram - participant R as Future reconciler - participant Store as SqliteRuntimeStore - participant Scan as Shared scanner / interpreter - participant Events as runtime_events - participant Proj as journal / operation projection - - R->>Store: commitToolRecoveryBundle - Store->>Scan: 校验 call + dispatch + observation + outcome? + decision - Scan-->>Store: valid / corruption - alt valid completed - Store->>Events: append reconcile - Store->>Events: append matching successful outcome - Store->>Events: append completed decision - Store->>Proj: 同事务更新 recovery_completed - else valid parked - Store->>Events: append reconcile - Store->>Events: append parked decision - Store->>Proj: 同事务更新 recovery_parked - else invalid - Store-->>R: rollback,拒绝写入 - end -``` - -Writer、projection rebuild 和 `RecoveryResolver` 共享同一个 scanner/interpreter,所以 online、close/reopen、rebuild 和 Resolver 对同一 immutable ledger 必须得到同一解释。 - -`parked` 在 v1 中是 terminal:之后只允许 exact bundle retry 幂等收敛,不能悄悄发起第二次恢复 attempt。若未来需要重新打开,必须设计新的版本化事实,而不是修改旧事件。 - -## Phase 3 后续:文件恢复为什么只做 finalize - -文件 Write/Edit 的目标策略是先持久化可信 evidence: - -- workspace identity 与 canonical target; -- operation/call/dispatch identity; -- before identity; -- expected-after identity; -- transform/algorithm version; -- production-shaped result; -- size、regular-file、symlink、UTF-8 等观察边界。 - -崩溃后比较 current state: - -| observation | 动作 | -|---|---| -| `matches_expected_state` | 只做 cleanup/finalize,合成 outcome,提交 completed bundle | -| `matches_prior_state` | park,`reconcile_matches_prior_state` | -| `diverged` | park,不覆盖外部写入 | -| `unreadable` | park,不猜测 | - -```mermaid -flowchart TD - T1["T1 已提交,T2 缺失"] --> Observe["读取 durable evidence
观察 current file"] - Observe --> Expected{"current == expected-after?"} - Expected -->|"是"| Finalize["Finalize only
不再次写文件"] - Finalize --> Completed["提交 recovered outcome
+ completed decision"] - Expected -->|"否"| Prior{"current == before?"} - Prior -->|"是"| ParkPrior["Park
reconcile_matches_prior_state"] - Prior -->|"否,内容分叉"| ParkDiverged["Park
保护外部写入"] - Prior -->|"无法读取"| ParkUnreadable["Park
不猜测"] -``` - -即使 atomic rename 能避免半文件,也不能保证“检查 hash 到 rename”之间没有另一个进程修改文件。没有 conditional replace/CAS 前,`matches_prior_state` 不能自动 redo。 - -通用 Bash、任意远程 API、发送、发布、付款和删除等没有专属协议的副作用继续默认 park。恢复覆盖范围不是越大越好;错误地自动执行一次,通常比明确停车更危险。 +普通 T1 无 T2 的恢复将 invocation 封为 `outcome_unknown`,不持久化 tool response;后续明确用户轮通过请求投影获得 unknown response。只有更晚的 fresh、无 lineage 用户 invocation 成功完成后,投影才会退役;失败一轮不会触发退役。Client Capability 则持久化带 `uncertainOutcome: { code: 'outcome_unknown', retrySafe: false }` 的错误 response。这两种表达的持久性和生命周期目前不同,都不能证明原副作用成功;本次清理不统一二者。 ## Phase 4:Runtime history 还要绑定 workspace -Phase 1 证明模型历史完整,Phase 3 证明单个 operation 可以收敛,但长任务还需要 workspace-wide continuity。 +Phase 1 证明模型历史完整,未知工具结果继续保持未知。长任务可以另行需要 workspace-wide continuity。 Phase 4 把 checkpoint 建模为: @@ -845,29 +779,15 @@ Eval 不恢复或重建 Runtime execution,只请求 Runtime Host 执行 Maka s 7. 一次执行只以提交唯一一个 terminal RuntimeEvent 来结束。 8. 崩溃重启后先 repair 旧 Run。 9. Resolver 只读 immutable facts,判定 completed / not-dispatched / indeterminate / parked / corruption。 -10. 有 production reconciler 时,对 indeterminate 提交一个原子 recovery bundle;没有时 park。 +10. 将 T1 无 T2 封为未知,禁止自动重放副作用;新的明确用户轮可以检查现状。 11. Planner 检查合法 replay、workspace、工具和后台操作。 12. Kernel 在执行前重新验证并建立 continuation claim。 13. 新 Run 提交 continuation-start 后才调用 provider。 14. 任意一步不能证明安全,就保留事实并输出 machine-readable park reason。 -## 后续实现应该怎样拆 +## 后续工作的边界 -Phase 3–4 不适合做成一个横跨 schema、runtime protocol、host lifecycle 和 platform I/O 的大 PR。推荐依赖顺序是: - -```mermaid -flowchart TD - A["PR A
Recovery persistence authority
已完成"] --> B["PR B
Immutable cursor + durable claim
已合并"] - A --> C["PR C
File evidence + finalize-only recovery"] - B --> E["PR E
Checkpoint contracts"] - C --> E - E --> F["PR F
Canonical checkpoint bundle"] - F --> G["PR G
Observe-only Git carrier"] - G --> H["PR H
Capture + retention"] - H --> Restore["Isolated restore"] - H --> Rebaseline["Durable rebaseline"] - D["PR D
Host owner lifecycle"] -. "默认启用 capture / auto-resume 前必须完成" .-> H -``` +旧 PR C 文件 reconciliation 路线只作为历史设计保留。Workspace continuity 和 Host ownership 可以独立评估,不需要以通用 tool reconciler 为前置。未来如果重新提出某个操作的专属恢复能力,应先提供具体生产消费者与明确外部协议,再增加 Runtime 权威。 每个 PR 都应该写清: @@ -937,9 +857,7 @@ UI 文案可以演进,但 code 必须稳定,才能支持 crash tests、telem | Phase 0 deterministic replay / unit contract | 支持 | 支持 | 支持 | | Phase 0 process-crash committed-prefix harness | 支持 | 支持 | 有覆盖,但不代表断电 durability | | Phase 1 local safe-boundary continuation | 主要支持目标 | 主要支持目标 | 有限支持 / best-effort | -| SQLite T1/T2 与 recovery bundle 语义 | 支持 | 支持 | 语义支持 | -| Recovery bundle SIGKILL transaction proof | 发布证明平台 | 发布证明平台 | 当前按有限支持跳过 | -| Phase 3 file finalize-only recovery | 尚未生产接线 | 尚未生产接线 | 尚未生产接线 | +| SQLite T1/T2 与历史 recovery bundle 读取 | 支持 | 支持 | 语义支持 | | Phase 4 Git workspace continuity | 尚未实现 | 尚未实现 | 尚未实现 | Process crash、SQLite transaction atomicity 和应用级 `fsync` 不能自动升级成“机器断电后绝不丢数据”的承诺。具体文件系统、磁盘缓存和平台行为必须由单独 durability test 证明。 @@ -958,10 +876,7 @@ Process crash、SQLite transaction atomicity 和应用级 `fsync` 不能自动 - 用模型自述替代 RuntimeEvent、文件 evidence 或外部 receipt; - 用 CI 全绿替代并发、崩溃和数据安全论证。 -当前最重要的两个后续缺口是: - -1. PR B 已落地:immutable event-seq 边界、domain-separated prefix digest、SQLite 唯一 claim(`runtime_continuation_claims`),祖先 replay 通过 `continuation_start_v2` 统一; -2. PR C/D:production file evidence/reconciler 与唯一 host owner lifecycle。 +PR B 已落地:immutable event-seq 边界、domain-separated prefix digest、SQLite 唯一 claim(`runtime_continuation_claims`),祖先 replay 通过 `continuation_start_v2` 统一。工具专属 reconciliation 已退出当前产品合同。 ## 代码阅读地图 @@ -972,8 +887,8 @@ Process crash、SQLite transaction atomicity 和应用级 `fsync` 不能自动 1. `packages/core/src/runtime-event.ts`:RuntimeEvent、T1 marker、dispatch 与 recovery fact envelope。 2. `packages/core/src/canonical-runtime-event.ts`:唯一 canonical codec 与稳定 JSON bytes。 3. `packages/core/src/tool-ledger-scanner.ts`:call/dispatch/outcome/reconcile/decision lane 和 corruption 检测。 -4. `packages/core/src/tool-recovery-bundle.ts`:recovery bundle 因果校验与解释。 -5. `packages/core/src/runtime-event-store.ts`:普通 append、T1/T2 与 recovery bundle 的 API 权限边界。 +4. `packages/core/src/tool-recovery-bundle.ts`:历史 recovery bundle 因果校验与解释。 +5. `packages/core/src/runtime-event-store.ts`:普通 append、T1/T2 的 API 权限边界。 ### Storage diff --git a/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md b/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md index 88874d0c0b1..d887ff3b380 100644 --- a/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md +++ b/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md @@ -19,6 +19,9 @@ # Runtime Resume #1346 拆分与提取账本 +> **2026-10-16 退役说明:** 本文涉及工具专属 reconciliation、recovery bundle 写入及其后续 PR 的内容保留为历史设计,不再是当前实施要求。没有生产工具提供完整的 reconciliation 合同,公开 bundle writer 和预设工具恢复能力已清理。已有 recovery facts 仍保留读取、复制与投影重建兼容。当前合同是结束旧执行、保留未知事实、禁止自动重放,由新的明确用户轮检查现状;详见[当前恢复架构](./runtime-resume-architecture.zh-CN.md#未知结果与历史-recovery-bundle)。 + + - 状态:Active - 更新日期:2026-07-28 - 集成实验来源:`origin/codex/runtime-resume-phase3a@24bb5f33` diff --git a/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md b/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md index 21b00c7b4b5..53e3627de6b 100644 --- a/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md +++ b/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md @@ -19,6 +19,9 @@ # Runtime Resume Phase 3–4 实施路线 +> **2026-10-16 退役说明:** 本文涉及工具专属 reconciliation、recovery bundle 写入及其后续 PR 的内容保留为历史设计,不再是当前实施要求。没有生产工具提供完整的 reconciliation 合同,公开 bundle writer 和预设工具恢复能力已清理。已有 recovery facts 仍保留读取、复制与投影重建兼容。当前合同是结束旧执行、保留未知事实、禁止自动重放,由新的明确用户轮检查现状;详见[当前恢复架构](./runtime-resume-architecture.zh-CN.md#未知结果与历史-recovery-bundle)。 + + > 路线更新(2026-08-29):旧 Git executable managed-workspace path 已删除,后续 workspace 主线从 > **Gitoxide data plane** 重新建立 production admission。本文保留 Recovery/Continuation 已落地协议与旧路线的历史论证; > 新实现不再以通用 per-file checkpoint、native manifest、CAS object store 或旧 Git CLI owner 为前置。 diff --git a/packages/core/src/runtime-event-store.ts b/packages/core/src/runtime-event-store.ts index 39f4e724697..8bd8d29e3a2 100644 --- a/packages/core/src/runtime-event-store.ts +++ b/packages/core/src/runtime-event-store.ts @@ -36,16 +36,8 @@ import type { } from './workspace-version-authority.js'; import { WORKSPACE_VERSION_AUTHORITY_CAPABILITY_V1 } from './workspace-version-authority.js'; -export const TOOL_RECOVERY_BUNDLE_CAPABILITY_V1 = 'tool_recovery_bundle_v1' as const; export const RUNTIME_CONTINUATION_AUTHORITY_V1 = 'runtime_continuation_authority_v1' as const; -export interface RuntimeRecoveryBundleCommit { - operationId: string; - reconcileRuntimeEvent: RuntimeEvent; - outcomeRuntimeEvent?: RuntimeEvent; - decisionRuntimeEvent: RuntimeEvent; -} - export interface RuntimeInvocationRecoveryInventoryEntry { readonly sessionId: string; readonly invocationId: string; @@ -277,11 +269,6 @@ export async function readRunInvocation( ); } -export interface RuntimeRecoveryBundleStore extends RuntimeEventStore { - readonly recoveryBundleCapability: typeof TOOL_RECOVERY_BUNDLE_CAPABILITY_V1; - commitToolRecoveryBundle(input: RuntimeRecoveryBundleCommit): Promise; -} - export type ContinuationClaimResult = | { kind: 'acquired'; claim: ContinuationClaimV1 } | { kind: 'existing'; claim: ContinuationClaimV1 } diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index 3e5ade5b7b0..45beebdbcdf 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -633,7 +633,10 @@ export interface RuntimeEventRefs { sourceRuntimeEventHighWater?: number; } -/** Tool-owned contract for deciding what a later recovery phase may do. */ +/** Current tools only choose how to report an unknown result; neither mode retries an effect. */ +export type ToolOutcomeRecoveryMode = 'never_auto_retry' | 'outcome_unknown'; + +/** Persisted dispatch metadata, including legacy modes; never authorizes recovery by itself. */ export type ToolRecoveryMode = | 'replay_safe' | 'idempotent' diff --git a/packages/core/src/tool-ledger-scanner.ts b/packages/core/src/tool-ledger-scanner.ts index 86c5b6cf465..4bcc80b6c4c 100644 --- a/packages/core/src/tool-ledger-scanner.ts +++ b/packages/core/src/tool-ledger-scanner.ts @@ -141,11 +141,7 @@ export type GenericToolLedgerAppendValidation = eventId: string; }; -export type ToolLedgerTransitionKind = - | 'generic_append' - | 't1_prepare' - | 't2_outcome' - | 'recovery_bundle'; +export type ToolLedgerTransitionKind = 'generic_append' | 't1_prepare' | 't2_outcome'; /** Every reason the ledger has for refusing a candidate event. */ export type ToolLedgerRejectionCode = @@ -708,15 +704,7 @@ function validateTransitionShape( (expectedTransition === 't2_outcome' && actual.length === 1 && actual[0] === 'function_response' && - events[0]?.refs?.operationId !== undefined) || - (expectedTransition === 'recovery_bundle' && - ((actual.length === 2 && - actual[0] === 'reconcile_result' && - actual[1] === 'recovery_decision') || - (actual.length === 3 && - actual[0] === 'reconcile_result' && - actual[1] === 'function_response' && - actual[2] === 'recovery_decision'))); + events[0]?.refs?.operationId !== undefined); if (valid) return { ok: true }; return { ok: false, diff --git a/packages/core/src/tool-recovery-bundle.ts b/packages/core/src/tool-recovery-bundle.ts index 5c4fca66e8a..20bf29153f5 100644 --- a/packages/core/src/tool-recovery-bundle.ts +++ b/packages/core/src/tool-recovery-bundle.ts @@ -91,17 +91,6 @@ export type ScannedToolRecoveryInterpretation = message: string; }; -export class ToolRecoveryBundleValidationError extends Error { - readonly name = 'ToolRecoveryBundleValidationError'; - - constructor( - readonly code: ToolRecoveryBundleValidationCode, - message: string, - ) { - super(message); - } -} - export function validateToolRecoveryEventBundle( input: ToolRecoveryEventBundle, ): ToolRecoveryBundleValidationResult { @@ -275,16 +264,8 @@ export function validateToolRecoveryEventBundle( return { ok: true, reconcile, decision }; } -export function assertToolRecoveryEventBundle(input: ToolRecoveryEventBundle): { - reconcile: ToolReconcileResultFact; - decision: ToolRecoveryDecisionFact; -} { - const result = validateToolRecoveryEventBundle(input); - if (!result.ok) throw new ToolRecoveryBundleValidationError(result.code, result.message); - return { reconcile: result.reconcile, decision: result.decision }; -} - /** + * Read compatibility for historical recovery bundles. Current tools never produce these facts. * Interprets the recovery tail of one scanner-owned operation. Resolver and * projection rebuild must consume this result instead of independently * assembling facts or defining their own causal-order rules. diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index d829aac87ae..5a68ec5652f 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -1016,7 +1016,7 @@ test('Host reopens one projected image from its ArtifactStore authority', async name: 'ProjectedImage', description: 'Return one inline image.', parameters: z.object({}), - recoveryMode: 'replay_safe', + recoveryMode: 'never_auto_retry', impl: async () => ({ private: 'raw execution fact' }), toModelOutput: () => ({ type: 'content', diff --git a/packages/runtime-host/src/server/agent-settings-tools.ts b/packages/runtime-host/src/server/agent-settings-tools.ts index e09cdfed1ec..67ff0050675 100644 --- a/packages/runtime-host/src/server/agent-settings-tools.ts +++ b/packages/runtime-host/src/server/agent-settings-tools.ts @@ -101,7 +101,6 @@ export function buildHostAgentSettingsTools( 'This never returns credentials, network proxy details, native client settings, or Bot settings.', parameters: z.object({}).strict(), categoryHint: 'read', - recoveryMode: 'replay_safe', impl: async () => projectSettings((await authority.read()).policy), }; const updateTool: MakaTool = { diff --git a/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts b/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts index fb7401779c0..d536a577e92 100644 --- a/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts +++ b/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts @@ -22,6 +22,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { createSqliteRuntimeStore } from '@maka/storage/sqlite-runtime-store'; @@ -36,7 +37,7 @@ const OBSERVATION_DIGEST = 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' as const; describe('recovery authority equivalence', () => { - it('gives online projection, reopen, rebuild, and Resolver one terminal interpretation', async () => { + it('gives historical projection, reopen, rebuild, and Resolver one terminal interpretation', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-recovery-equivalence-')); const dbPath = join(root, 'runtime.sqlite'); const store = createSqliteRuntimeStore(dbPath); @@ -52,12 +53,8 @@ describe('recovery authority equivalence', () => { recoveryMode: 'reconcile', committedAt: 10, }); - await store.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decisionEvent(), - }); + seedHistoricalRecoveryTail(dbPath, [reconcileEvent(), outcomeEvent(), decisionEvent()]); + await store.rebuildToolProjectionsFromRuntimeEvents(); const immutable = await store.readImmutableRuntimeEvents('session-1', 'run-1'); const online = await store.readToolOperation('operation-1'); @@ -77,7 +74,6 @@ describe('recovery authority equivalence', () => { responseIsError: false, }, ]); - assert.equal(resolution.requiresReconciliation, false); store.close(); const reopened = createSqliteRuntimeStore(dbPath); @@ -149,11 +145,8 @@ describe('recovery authority equivalence', () => { committedAt: 20, }); } else if ('parked' in scenario) { - await store.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent('diverged'), - decisionRuntimeEvent: parkedDecisionEvent(), - }); + seedHistoricalRecoveryTail(dbPath, [reconcileEvent('diverged'), parkedDecisionEvent()]); + await store.rebuildToolProjectionsFromRuntimeEvents(); } const immutable = await store.readImmutableRuntimeEvents('session-1', 'run-1'); @@ -195,7 +188,6 @@ describe('recovery authority equivalence', () => { ['duplicate_event_id', 'semantic_lane_conflict'], ); assert.equal(resolution.hasCorruption, true); - assert.equal(resolution.requiresReconciliation, false); }); it('treats a valid parked recovery bundle as terminal', () => { @@ -220,7 +212,6 @@ describe('recovery authority equivalence', () => { }, ]); assert.equal(resolution.hasCorruption, false); - assert.equal(resolution.requiresReconciliation, false); const plan = buildResumePlanFromRuntimeEvents(events); assert.equal(plan.disposition, 'blocked'); @@ -394,3 +385,31 @@ function decisionEvent(): RuntimeEvent { refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, }); } + +/** Read-compatibility fixture: these facts were persisted by the retired writer. */ +function seedHistoricalRecoveryTail(dbPath: string, events: readonly RuntimeEvent[]): void { + const db = new DatabaseSync(dbPath); + try { + const insert = db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, + event_seq, event_kind, payload_json, committed_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + for (const [index, event] of events.entries()) { + insert.run( + event.id, + event.sessionId, + event.invocationId, + event.runId, + event.turnId, + index + 3, + event.content?.kind ?? 'runtime_fact', + JSON.stringify(event), + event.ts, + ); + } + } finally { + db.close(); + } +} diff --git a/packages/runtime/src/__tests__/recovery-resolver.test.ts b/packages/runtime/src/__tests__/recovery-resolver.test.ts index 3b027d7d73a..732858fc40f 100644 --- a/packages/runtime/src/__tests__/recovery-resolver.test.ts +++ b/packages/runtime/src/__tests__/recovery-resolver.test.ts @@ -40,10 +40,9 @@ describe('RecoveryResolver', () => { }, ]); assert.equal(resolution.hasCorruption, false); - assert.equal(resolution.requiresReconciliation, false); }); - it('requires reconciliation after dispatch when no response was committed', () => { + it('keeps the outcome unknown after dispatch when no response was committed', () => { const resolution = resolveRuntimeRecovery([ initialEvent('t1_after_preflight_v1'), functionCallEvent(), @@ -62,7 +61,6 @@ describe('RecoveryResolver', () => { }, ]); assert.equal(resolution.hasCorruption, false); - assert.equal(resolution.requiresReconciliation, true); }); it('keeps dispatched operations unresolved instead of inventing durable tool results', () => { @@ -144,7 +142,6 @@ describe('RecoveryResolver', () => { assert.equal(resolution.decisions[0]?.status, 'indeterminate'); assert.equal(resolution.decisions[0]?.reason, 'dispatch_without_response'); - assert.equal(resolution.requiresReconciliation, true); }); it('treats a matching response without dispatch as a completed pre-T1 result', () => { @@ -166,7 +163,6 @@ describe('RecoveryResolver', () => { settlementOrigin: 'pre_t1_synthetic', }, ]); - assert.equal(resolution.requiresReconciliation, false); }); it('classifies a dispatch without its canonical function call as corruption', () => { @@ -186,7 +182,6 @@ describe('RecoveryResolver', () => { }, ]); assert.equal(resolution.hasCorruption, true); - assert.equal(resolution.requiresReconciliation, false); }); it('classifies a response without its canonical function call as corruption', () => { @@ -249,7 +244,6 @@ describe('RecoveryResolver', () => { }, ]); assert.equal(resolution.hasCorruption, true); - assert.equal(resolution.requiresReconciliation, false); }); it('classifies repeated response for one operation as corruption', () => { @@ -275,7 +269,6 @@ describe('RecoveryResolver', () => { }, ]); assert.equal(resolution.hasCorruption, true); - assert.equal(resolution.requiresReconciliation, false); }); it('rejects a protocol marker added after the first canonical event', () => { @@ -313,7 +306,6 @@ describe('RecoveryResolver', () => { assert.equal(resolution.decisions[0]?.status, 'indeterminate'); assert.equal(resolution.decisions[0]?.reason, 'legacy_dispatch_unknown'); assert.equal(resolution.hasCorruption, true); - assert.equal(resolution.requiresReconciliation, false); }); it('classifies a response linked to a different operation as corruption', () => { diff --git a/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts b/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts index 01bd89f044f..39c8a0a23b9 100644 --- a/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-durable-boundary.test.ts @@ -121,6 +121,44 @@ describe('ToolRuntime durable boundary', () => { assert.deepEqual(telemetry, ['aborted', 'error']); }); + it('records only unknown-reporting policies even for historical tool declarations', async () => { + for (const mode of [ + undefined, + 'never_auto_retry', + 'outcome_unknown', + 'replay_safe', + 'idempotent', + 'reconcile', + 'reattach', + ]) { + const prepared: ToolPreparedCommit[] = []; + let executions = 0; + const harness = makeHarness({ + commitToolPrepared: async (input) => { + prepared.push(input); + return { created: true, runtimeEventSeq: 1 }; + }, + commitToolOutcome: async () => ({ created: true, runtimeEventSeq: 2 }), + }); + const target = { + ...tool(() => { + executions += 1; + return 'ok'; + }), + recoveryMode: mode, + } as unknown as MakaTool; + await harness.execute(target); + const expected = mode === 'outcome_unknown' ? 'outcome_unknown' : 'never_auto_retry'; + assert.equal(prepared[0]?.recoveryMode, expected, String(mode)); + assert.equal( + prepared[0]?.dispatchRuntimeEvent.actions?.toolDispatch?.recoveryMode, + expected, + String(mode), + ); + assert.equal(executions, 1); + } + }); + it('does not invoke the tool or publish a result when T1 fails', async () => { let implementationCalls = 0; const harness = makeHarness({ @@ -882,7 +920,7 @@ function tool(impl: MakaTool['impl']): MakaTool { name: 'Read', description: 'read', parameters: {}, - recoveryMode: 'replay_safe', + recoveryMode: 'never_auto_retry', impl, }; } diff --git a/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts b/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts index 1998048d03a..2938a0e5113 100644 --- a/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts +++ b/packages/runtime/src/__tests__/tool-runtime-sqlite-boundary.test.ts @@ -433,7 +433,7 @@ describe('ToolRuntime with real SQLite boundary', () => { name: 'Read', description: 'read', parameters: {}, - recoveryMode: 'replay_safe', + recoveryMode: 'never_auto_retry', impl: async () => { implementationCalls += 1; return { ok: true, text: 'contents' }; @@ -563,7 +563,7 @@ describe('ToolRuntime with real SQLite boundary', () => { name: 'Read', description: 'read', parameters: {}, - recoveryMode: 'replay_safe', + recoveryMode: 'never_auto_retry', impl: async () => { implementationCalls += 1; return { private: 'completed execution fact' }; @@ -641,7 +641,7 @@ describe('ToolRuntime with real SQLite boundary', () => { name: 'Read', description: 'read', parameters: {}, - recoveryMode: 'replay_safe', + recoveryMode: 'never_auto_retry', impl: async () => { throw new Error('disk read failed'); }, diff --git a/packages/runtime/src/agent-swarm-status-tool.ts b/packages/runtime/src/agent-swarm-status-tool.ts index f4d2427c39c..f8c3c970318 100644 --- a/packages/runtime/src/agent-swarm-status-tool.ts +++ b/packages/runtime/src/agent-swarm-status-tool.ts @@ -162,7 +162,6 @@ export function buildAgentSwarmStatusTool(input: { 'Read compact status-only progress for the asynchronous swarm. This omits child logs, tool activity, reasoning, and partial output.', parameters: z.object({}).strip(), categoryHint: 'read', - recoveryMode: 'replay_safe', impl: async () => projectAgentSwarmStatus(await input.readSnapshot()), }; } diff --git a/packages/runtime/src/mcp-tools.ts b/packages/runtime/src/mcp-tools.ts index 11ae03ca2a6..7cbd2b8cf0c 100644 --- a/packages/runtime/src/mcp-tools.ts +++ b/packages/runtime/src/mcp-tools.ts @@ -31,7 +31,7 @@ import type { PermissionMode, ToolCategory } from '@maka/core/permission'; import { REQUEST_COMPOSITION_MAX_TOOL_DESCRIPTION_LENGTH } from '@maka/core/run-composition'; import { truncateUtf16Safe } from '@maka/core/text-sanitize'; import type { ExecutionBoundary } from '@maka/core/sandbox-boundary'; -import type { ToolRecoveryMode } from '@maka/core/runtime-event'; +import type { ToolOutcomeRecoveryMode } from '@maka/core/runtime-event'; import type { ToolResultContentPart, ToolResultOutput } from './model-protocol.js'; import type { MakaTool } from './tool-runtime.js'; @@ -98,7 +98,7 @@ export interface BuildMcpToolsOptions { callTimeoutMs?: number; categoryHint?: ToolCategory; hostAdmission?: MakaTool['hostAdmission']; - recoveryMode?: ToolRecoveryMode; + recoveryMode?: ToolOutcomeRecoveryMode; executionLocation?: 'host' | 'remote'; activityKindForDescriptor?: (descriptor: McpToolDescriptor) => ToolActivityKind | undefined; } diff --git a/packages/runtime/src/memory-extraction.ts b/packages/runtime/src/memory-extraction.ts index 5cbf2d32a77..1ccc6fe1a09 100644 --- a/packages/runtime/src/memory-extraction.ts +++ b/packages/runtime/src/memory-extraction.ts @@ -304,7 +304,6 @@ export function buildMemoryExtractionTriggerTools(input: { 'Use only when the user explicitly asks you to remember long-term information. It stores the requested memory and returns exactly what was saved.', parameters: noArguments, executionSemantics: 'exclusive_step', - recoveryMode: 'idempotent', impl: async (_args: Record, context: MakaToolContext) => { const gate = await input.capabilities.gate(); if (!gate.allowed) return { status: 'unavailable', requestedItems: [] }; @@ -325,7 +324,6 @@ export function buildMemoryExtractionTriggerTools(input: { description: 'Use when the conversation contains durable long-term information worth preserving and the user did not explicitly ask to remember it. The extraction runs after this turn.', parameters: noArguments, - recoveryMode: 'idempotent', impl: async (_args: Record, context: MakaToolContext) => { const gate = await input.capabilities.gate(); if (!gate.allowed) return { status: 'unavailable' }; diff --git a/packages/runtime/src/plan-tools.ts b/packages/runtime/src/plan-tools.ts index daba995c5ee..199819ac3ec 100644 --- a/packages/runtime/src/plan-tools.ts +++ b/packages/runtime/src/plan-tools.ts @@ -129,7 +129,6 @@ export function buildSubmitPlanTool( isPlanProposalLifecycleAdmissible, 'Plan proposal cannot fit its execution lifecycle projection', ), - recoveryMode: 'idempotent', impl: async (input, context) => { const result = await planStore.submitProposal({ operationId: planToolOperationId('submit', context), @@ -168,7 +167,6 @@ export function buildUpdatePlanTool( steps: z.array(executionStepSchema).min(1).max(PLAN_MAX_STEPS), explanation: boundedTextSchema('Plan progress explanation').optional(), }), - recoveryMode: 'idempotent', impl: async (input, context) => { const result = await planStore.updateExecution({ operationId: planToolOperationId('update', context), @@ -192,7 +190,6 @@ export function buildCancelPlanTool( parameters: z.object({ reason: boundedTextSchema('Plan cancellation reason', PLAN_LIFECYCLE_REASON_MAX_BYTES), }), - recoveryMode: 'idempotent', impl: async ({ reason }, context) => { const result = await planStore.cancelExecution({ operationId: planToolOperationId('cancel', context), diff --git a/packages/runtime/src/recovery-resolver.ts b/packages/runtime/src/recovery-resolver.ts index 23b8c4ea3de..a72b58fbbb3 100644 --- a/packages/runtime/src/recovery-resolver.ts +++ b/packages/runtime/src/recovery-resolver.ts @@ -82,7 +82,6 @@ export interface RuntimeRecoveryResolution { eventId: string; }>; hasCorruption: boolean; - requiresReconciliation: boolean; } export function resolveRuntimeRecovery(events: readonly RuntimeEvent[]): RuntimeRecoveryResolution { @@ -116,8 +115,6 @@ export function resolveRuntimeRecovery(events: readonly RuntimeEvent[]): Runtime decisions, issues, hasCorruption, - requiresReconciliation: - !hasCorruption && decisions.some((decision) => decision.status === 'indeterminate'), }; } @@ -174,6 +171,8 @@ function decisionFromOperation( if (operation.reconcileEvents.length === 0 && operation.decisionEvents.length === 0) { return decision; } + // Historical bundles remain authoritative when reading an existing ledger. + // New uncertain calls are sealed as outcome_unknown, never reconciled. const recovery = interpretScannedToolRecovery(operation, eventOrder); if (recovery.kind !== 'valid') { decision.status = 'corruption'; diff --git a/packages/runtime/src/stream-graph-supervisor-tools.ts b/packages/runtime/src/stream-graph-supervisor-tools.ts index b93fba2ece0..2183aa04b8c 100644 --- a/packages/runtime/src/stream-graph-supervisor-tools.ts +++ b/packages/runtime/src/stream-graph-supervisor-tools.ts @@ -561,7 +561,6 @@ export function buildAgentGraphSupervisorTools( 'Inspect the durable graph. Use mode=latest without a cursor for the current view; use mode=page only with a nextCursor returned by an earlier view.', parameters: viewSchema, categoryHint: 'read', - recoveryMode: 'replay_safe', impl: async (toolInput) => { const view = await readToolGraphView( input.scheduleStore, @@ -584,7 +583,6 @@ export function buildAgentGraphSupervisorTools( 'Adjust the graph durably. Always set operation. Prefer target_kind=new_preset with a user-approved subagent_id from agent_list; legacy agent_id remains supported. Unrelated provider-filled optional fields are ignored.', parameters: updateSchema, categoryHint: 'subagent', - recoveryMode: 'idempotent', impl: async (toolInput, context) => { const request = compileAgentGraphScheduleUpdate({ graphId, @@ -618,7 +616,6 @@ export function buildAgentGraphSupervisorTools( 'End this supervisor turn successfully while scheduled graph work continues. Call this after the current scheduling wave has no immediate decision; do not poll, sleep, or emit a waiting message. The host will start a new supervisor turn at the next durable graph checkpoint. This does not finish or close the graph.', parameters: yieldSchema, categoryHint: 'subagent', - recoveryMode: 'replay_safe', executionSemantics: 'exclusive_step', impl: async ({ reason }) => { const permit = input.prepareYieldPermit?.(); diff --git a/packages/runtime/src/tool-runtime.ts b/packages/runtime/src/tool-runtime.ts index 191d711d178..52187d7f30f 100644 --- a/packages/runtime/src/tool-runtime.ts +++ b/packages/runtime/src/tool-runtime.ts @@ -76,7 +76,11 @@ import { buildToolCallArgs, snapshotToolArgs } from './tool-call-snapshot.js'; import type { SessionHeader } from '@maka/core/session'; import type { ToolInvocationRecord } from '@maka/core/usage-stats/types'; import { redactSecrets } from '@maka/core/redaction'; -import { TOOL_BOUNDARY_PROTOCOL_V1, type RuntimeEvent } from '@maka/core/runtime-event'; +import { + TOOL_BOUNDARY_PROTOCOL_V1, + type RuntimeEvent, + type ToolOutcomeRecoveryMode, +} from '@maka/core/runtime-event'; import { recordToolArtifactsSafely, type ToolArtifactRecorder } from './tool-artifacts.js'; import { computerActionFields, describeComputerUseArgsViolation } from './computer-use-codec.js'; @@ -102,7 +106,6 @@ import { buildToolOperationId, canonicalToolArgsHash, type RuntimeCommitSink, - type ToolRecoveryMode, } from './runtime-commit-sink.js'; import { AdmissionLimiter } from './admission-limiter.js'; import type { AgentProfile } from './agent-catalog.js'; @@ -204,8 +207,8 @@ export interface MakaTool

{ readonly searchContextSize?: 'low' | 'medium' | 'high'; readonly maxUses?: number; }; - /** Crash-recovery contract used by the durable tool boundary. */ - recoveryMode?: ToolRecoveryMode; + /** How the durable boundary reports an unknown result; never authorizes automatic retry. */ + recoveryMode?: ToolOutcomeRecoveryMode; /** Step-level admission contract. Exclusive tools cannot share an assistant step. */ executionSemantics?: 'parallel' | 'exclusive_step'; /** Nested CodeMode admission. Ordinary tools are nestable by default. */ @@ -1977,7 +1980,9 @@ export class ToolRuntime { ...(Object.keys(stateDelta).length > 0 ? { actions: { stateDelta } } : {}), }; const canonicalArgsHash = canonicalToolArgsHash(input.tool.name, input.persistedArgs); - const recoveryMode = input.tool.recoveryMode ?? 'never_auto_retry'; + // Historical tool declarations cannot opt a current dispatch into reconciliation or replay. + const recoveryMode = + input.tool.recoveryMode === 'outcome_unknown' ? 'outcome_unknown' : 'never_auto_retry'; const dispatchEvent: RuntimeEvent = { id: `${operationId}_dispatch`, invocationId, diff --git a/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts b/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts index f481536b5f5..4f789ff362a 100644 --- a/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts +++ b/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts @@ -21,7 +21,6 @@ import { existsSync, writeSync } from 'node:fs'; import { dirname } from 'node:path'; import { type RuntimeEvent } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import { type ToolRecoveryFactEnvelope } from '@maka/core/tool-recovery-fact'; import { type WorkspaceBaselineAuthorityInput } from '@maka/core/workspace-version-authority'; import { createRuntimeBoundaryCursor, runtimePrefixSegment } from '@maka/core/runtime-boundary'; import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; @@ -56,12 +55,6 @@ try { while (!existsSync(stopPath)) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); } - } else if (mode === 'completed') { - await store!.commitToolRecoveryBundle(completedBundle()); - } else if (mode === 'parked') { - await store!.commitToolRecoveryBundle(parkedBundle()); - } else if (mode === 'rebuild') { - await store!.rebuildToolProjectionsFromRuntimeEvents(); } else if (mode === 'workspace_baseline_a' || mode === 'workspace_baseline_b') { bindWorkspaceBaselineAuthorityStoreRootInternal(store!, 'a'.repeat(64)); const result = await commitWorkspaceBaselineInternal( @@ -178,105 +171,6 @@ try { operationalLease?.close(); } -function completedBundle() { - return { - operationId: 'operation-1', - reconcileRuntimeEvent: recoveryFact( - 'completed-reconcile', - 'maka.tool.reconcile_result', - { - protocol: 'tool_reconcile_v1', - operationId: 'operation-1', - observation: 'matches_expected_state', - observationSchema: 'state_identity_v1', - observationDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - }, - 3, - ), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: recoveryFact( - 'completed-decision', - 'maka.tool.recovery_decision', - { - protocol: 'tool_recovery_v1', - operationId: 'operation-1', - disposition: 'completed', - reasonCode: 'reconcile_matches_expected_state', - outcomeEventId: 'completed-outcome', - evidenceEventIds: [ - 'call-event-1', - 'dispatch-event-1', - 'completed-reconcile', - 'completed-outcome', - ], - }, - 5, - ), - } as const; -} - -function parkedBundle() { - return { - operationId: 'operation-1', - reconcileRuntimeEvent: recoveryFact( - 'parked-reconcile', - 'maka.tool.reconcile_result', - { - protocol: 'tool_reconcile_v1', - operationId: 'operation-1', - observation: 'diverged', - observationSchema: 'state_identity_v1', - observationDigest: - 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - }, - 3, - ), - decisionRuntimeEvent: recoveryFact( - 'parked-decision', - 'maka.tool.recovery_decision', - { - protocol: 'tool_recovery_v1', - operationId: 'operation-1', - disposition: 'parked', - reasonCode: 'reconcile_diverged', - evidenceEventIds: ['call-event-1', 'dispatch-event-1', 'parked-reconcile'], - }, - 4, - ), - } as const; -} - -function outcomeEvent(): RuntimeEvent { - return { - ...baseEvent('completed-outcome', 4), - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: 'provider-call-1', - name: 'Write', - result: 'ok', - }, - refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, - }; -} - -function recoveryFact( - id: string, - kind: 'maka.tool.reconcile_result' | 'maka.tool.recovery_decision', - payload: Record, - ts: number, -): RuntimeEvent { - return { - ...baseEvent(id, ts), - actions: { - toolRecovery: { kind, version: 1, payload } as unknown as ToolRecoveryFactEnvelope, - }, - refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, - }; -} - function baseEvent(id: string, ts: number): RuntimeEvent { return { id, diff --git a/packages/storage/src/__tests__/recovery-persistence-authority.test.ts b/packages/storage/src/__tests__/recovery-persistence-authority.test.ts index a1462df67f1..3e0e5098894 100644 --- a/packages/storage/src/__tests__/recovery-persistence-authority.test.ts +++ b/packages/storage/src/__tests__/recovery-persistence-authority.test.ts @@ -23,11 +23,9 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { describe, it } from 'node:test'; -import { TOOL_RECOVERY_BUNDLE_CAPABILITY_V1 } from '@maka/core/runtime-event-store'; import { type RuntimeEvent } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js'; -import type { SqliteRuntimeStoreFailpoint } from '../sqlite-runtime-store.js'; const ARGS_HASH = canonicalToolArgsHash('Write', { path: 'notes.txt', @@ -232,7 +230,6 @@ describe('SQLite recovery persistence authority', () => { (await upgraded.readToolOperation('legacy-completed-operation'))?.currentState, 'outcome_committed', ); - assert.equal(upgraded.recoveryBundleCapability, TOOL_RECOVERY_BUNDLE_CAPABILITY_V1); } finally { upgraded.close(); } @@ -267,7 +264,7 @@ describe('SQLite recovery persistence authority', () => { await withStore(async (store) => { await assert.rejects( store.appendRuntimeEvent('session-1', 'run-1', reconcileEvent()), - /atomic recovery bundle writer/, + /Historical tool recovery facts are read-only/, ); await assert.rejects( store.importRuntimeEventsBatch({ @@ -275,7 +272,7 @@ describe('SQLite recovery persistence authority', () => { runId: 'run-1', events: [reconcileEvent()], }), - /atomic recovery bundle writer/, + /Historical tool recovery facts are read-only/, ); await assert.rejects( store.appendRuntimeEvent('session-1', 'run-1', dispatchEvent()), @@ -460,16 +457,16 @@ describe('SQLite recovery persistence authority', () => { }); }); - it('atomically settles completed recovery and rebuilds the same projection after reopen', async () => { + it('reads and rebuilds historical completed recovery after reopen', async () => { await withStore(async (store, dbPath) => { - assert.equal(store.recoveryBundleCapability, TOOL_RECOVERY_BUNDLE_CAPABILITY_V1); - await prepare(store); - await store.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decisionEvent(), - }); + seedHistoricalRecovery(dbPath, [ + callEvent(), + dispatchEvent(), + reconcileEvent(), + outcomeEvent(), + decisionEvent(), + ]); + await store.rebuildToolProjectionsFromRuntimeEvents(); const onlineOperation = await store.readToolOperation('operation-1'); const onlineJournal = await store.readToolJournal('operation-1'); @@ -497,6 +494,39 @@ describe('SQLite recovery persistence authority', () => { }); }); + it('preserves historical parked recovery without synthesizing an outcome', async () => { + await withStore(async (store, dbPath) => { + const events = [ + callEvent(), + dispatchEvent(), + reconcileEvent('diverged'), + decisionEvent('parked'), + ]; + seedHistoricalRecovery(dbPath, events); + await store.rebuildToolProjectionsFromRuntimeEvents(); + assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'recovery_parked'); + assert.deepEqual(await store.listUnsettledToolOperations(), []); + assert.deepEqual(await store.readImmutableRuntimeEvents('session-1', 'run-1'), events); + const journal = await store.readToolJournal('operation-1'); + assert.deepEqual( + journal.map(({ state }) => state), + ['prepared', 'reconcile_observed', 'recovery_parked'], + ); + store.close(); + const reopened = createSqliteRuntimeStore(dbPath); + try { + await reopened.rebuildToolProjectionsFromRuntimeEvents(); + assert.equal( + (await reopened.readToolOperation('operation-1'))?.currentState, + 'recovery_parked', + ); + assert.deepEqual(await reopened.readToolJournal('operation-1'), journal); + } finally { + reopened.close(); + } + }); + }); + it('rebuilds interleaved operations to the same projections and journal identities', async () => { await withStore(async (store) => { await prepare(store); @@ -582,77 +612,6 @@ describe('SQLite recovery persistence authority', () => { }); }); - it('rolls the whole recovery bundle back at every internal crash boundary', async () => { - for (const failpoint of [ - 'after_recovery_reconcile', - 'after_recovery_outcome', - 'after_recovery_decision', - ] as const satisfies readonly SqliteRuntimeStoreFailpoint[]) { - const root = await mkdtemp(join(tmpdir(), 'maka-recovery-failpoint-')); - const dbPath = join(root, 'runtime.sqlite'); - let active: SqliteRuntimeStoreFailpoint | undefined; - const store = createSqliteRuntimeStore(dbPath, { - failpoint: (point) => { - if (point === active) throw new Error(`recovery failpoint: ${point}`); - }, - }); - try { - await prepare(store); - active = failpoint; - await assert.rejects( - store.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decisionEvent(), - }), - new RegExp(failpoint), - ); - active = undefined; - assert.deepEqual( - (await store.readImmutableRuntimeEvents('session-1', 'run-1')).map(({ id }) => id), - ['call-event-1', 'dispatch-event-1'], - ); - assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared'); - assert.deepEqual( - (await store.readToolJournal('operation-1')).map(({ state }) => state), - ['prepared'], - ); - } finally { - store.close(); - await rm(root, { recursive: true, force: true }); - } - } - }); - - it('makes one parked bundle terminal and only permits its exact retry', async () => { - await withStore(async (store) => { - await prepare(store); - const parked = { - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent('diverged'), - decisionRuntimeEvent: decisionEvent('parked'), - } as const; - await store.commitToolRecoveryBundle(parked); - await store.commitToolRecoveryBundle(parked); - - assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'recovery_parked'); - assert.deepEqual( - (await store.readToolJournal('operation-1')).map(({ state }) => state), - ['prepared', 'reconcile_observed', 'recovery_parked'], - ); - await assert.rejects( - store.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decisionEvent(), - }), - /duplicate_event_id/, - ); - }); - }); - it('fails immutable row/payload identity mismatches closed', async () => { await withStore(async (store, dbPath) => { await store.appendRuntimeEvent('session-1', 'run-1', userEvent()); @@ -748,58 +707,6 @@ describe('SQLite recovery persistence authority', () => { }); }); - it('rejects recovery evidence whose JSON bytes can rewrite validated event ids', async () => { - await withStore(async (store) => { - await prepare(store); - const decision = decisionEvent(); - const fact = decision.actions?.toolRecovery; - if (fact?.kind !== 'maka.tool.recovery_decision') throw new Error('invalid fixture'); - Object.defineProperty(fact.payload.evidenceEventIds, 'toJSON', { - value: () => ['call-event-1', 'dispatch-event-1', 'reconcile-event-1', 'forged-outcome'], - }); - - await assert.rejects( - store.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decision, - }), - /losslessly serializable/, - ); - assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared'); - }); - }); - - it('rejects row/payload identity corruption through the online bundle writer', async () => { - await withStore(async (store, dbPath) => { - await prepare(store); - store.close(); - - const db = new DatabaseSync(dbPath); - db.prepare(`UPDATE runtime_events SET run_id = 'run-other' WHERE event_id = ?`).run( - 'call-event-1', - ); - db.close(); - - const reopened = createSqliteRuntimeStore(dbPath); - try { - await assert.rejects( - reopened.commitToolRecoveryBundle({ - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decisionEvent(), - }), - /row\/payload identity mismatch/, - ); - assert.equal((await reopened.readToolOperation('operation-1'))?.currentState, 'prepared'); - } finally { - reopened.close(); - } - }); - }); - it('fails an exact T1 retry closed when the immutable ledger is already corrupt', async () => { await withStore(async (store, dbPath) => { await prepare(store); @@ -844,28 +751,6 @@ describe('SQLite recovery persistence authority', () => { }); }); - it('fails an exact recovery retry closed when the immutable ledger is already corrupt', async () => { - await withStore(async (store, dbPath) => { - await prepare(store); - const bundle = { - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: outcomeEvent(), - decisionRuntimeEvent: decisionEvent(), - } as const; - await store.commitToolRecoveryBundle(bundle); - store.close(); - injectDuplicateCall(dbPath, 6); - - const reopened = createSqliteRuntimeStore(dbPath); - try { - await assert.rejects(reopened.commitToolRecoveryBundle(bundle), /duplicate_call/); - } finally { - reopened.close(); - } - }); - }); - it('does not fail-stop a new session tool boundary for unrelated ledger corruption', async () => { await withStore(async (store, dbPath) => { await prepare(store); @@ -1208,3 +1093,33 @@ function decisionEvent(disposition: 'completed' | 'parked' = 'completed'): Runti refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, }); } + +/** Seed already-persisted events without exposing a production recovery writer. */ +function seedHistoricalRecovery(dbPath: string, events: readonly RuntimeEvent[]): void { + const db = new DatabaseSync(dbPath); + try { + const insert = db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, + event_seq, event_kind, payload_json, committed_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + for (const [index, event] of events.entries()) { + const kind = + event.content?.kind ?? (event.actions?.toolDispatch ? 'tool_dispatch' : 'runtime_fact'); + insert.run( + event.id, + event.sessionId, + event.invocationId, + event.runId, + event.turnId, + index + 1, + kind, + JSON.stringify(event), + event.ts, + ); + } + } finally { + db.close(); + } +} diff --git a/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts b/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts index 35de68bff55..8c1e42b32a3 100644 --- a/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts +++ b/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts @@ -29,7 +29,6 @@ import { fileURLToPath } from 'node:url'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { WorkspaceBaselineAuthorityInput } from '@maka/core/workspace-version-authority'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import { scanToolLedger } from '@maka/core/tool-ledger-scanner'; import { SQLITE_RUNTIME_SCHEMA_VERSION, createSqliteRuntimeStore, @@ -69,72 +68,6 @@ interface WorkerHandle { const RUN_RACE_AMPLIFICATION = process.env.MAKA_STORAGE_STRESS === '1'; describe('SQLite recovery authority multi-process races', () => { - it('makes an exact concurrent recovery bundle idempotent', async () => { - await withPreparedDatabase(async ({ dbPath, startPath }) => { - const results = await runWorkers(dbPath, startPath, ['completed', 'completed']); - assert.deepEqual( - results.map(({ code }) => code), - [0, 0], - ); - - const store = createSqliteRuntimeStore(dbPath); - try { - assert.equal( - (await store.readToolOperation('operation-1'))?.currentState, - 'recovery_completed', - ); - assert.equal((await store.readImmutableRuntimeEvents('session-1', 'run-1')).length, 5); - } finally { - store.close(); - } - }); - }); - - it('serializes conflicting completed and parked bundles to one terminal decision', async () => { - await withPreparedDatabase(async ({ dbPath, startPath }) => { - const results = await runWorkers(dbPath, startPath, ['completed', 'parked']); - assert.deepEqual(results.map(({ code }) => code).sort(), [0, 2]); - - const store = createSqliteRuntimeStore(dbPath); - try { - const operation = await store.readToolOperation('operation-1'); - assert.ok( - operation?.currentState === 'recovery_completed' || - operation?.currentState === 'recovery_parked', - ); - const events = await store.readImmutableRuntimeEvents('session-1', 'run-1'); - assert.equal(scanToolLedger(events).hasCorruption, false); - } finally { - store.close(); - } - }); - }); - - it('serializes projection rebuild against recovery commit', async () => { - await withPreparedDatabase(async ({ dbPath, startPath }) => { - const results = await runWorkers(dbPath, startPath, ['completed', 'rebuild']); - assert.deepEqual( - results.map(({ code }) => code), - [0, 0], - ); - - const store = createSqliteRuntimeStore(dbPath); - try { - assert.equal( - (await store.readToolOperation('operation-1'))?.currentState, - 'recovery_completed', - ); - assert.equal( - scanToolLedger(await store.readImmutableRuntimeEvents('session-1', 'run-1')) - .hasCorruption, - false, - ); - } finally { - store.close(); - } - }); - }); - it('grants provider authority to exactly one process for one continuation boundary', async () => { await withPreparedDatabase(async ({ dbPath, startPath }) => { const results = await runWorkers(dbPath, startPath, ['claim', 'claim']); diff --git a/packages/storage/src/__tests__/sqlite-runtime-crash.test.ts b/packages/storage/src/__tests__/sqlite-runtime-crash.test.ts index cc1cb637db5..0e076cd34e4 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-crash.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-crash.test.ts @@ -124,45 +124,6 @@ if (childMode) { }); }); - for (const mode of [ - 'inside_recovery_reconcile', - 'inside_recovery_outcome', - 'inside_recovery_decision', - ]) { - it(`rolls back the whole recovery bundle when killed at ${mode}`, { - timeout: 30_000, - }, async () => { - await withKilledChild(mode, async (store) => { - assert.deepEqual( - (await store.readRuntimeEvents('session-1', 'run-1')).map((event) => event.id), - ['call-event-1', 'dispatch-event-1'], - ); - assert.equal((await store.readToolOperation('operation-1'))?.currentState, 'prepared'); - }); - }); - } - - it('retains the whole recovery bundle when killed after COMMIT', { - timeout: 30_000, - }, async () => { - await withKilledChild('after_recovery_commit', async (store) => { - assert.deepEqual( - (await store.readRuntimeEvents('session-1', 'run-1')).map((event) => event.id), - [ - 'call-event-1', - 'dispatch-event-1', - 'reconcile-event-1', - 'response-event-1', - 'decision-event-1', - ], - ); - assert.equal( - (await store.readToolOperation('operation-1'))?.currentState, - 'recovery_completed', - ); - }); - }); - it('rolls back a workspace baseline when killed inside its authority transaction', { timeout: 30_000, }, async () => { @@ -340,15 +301,6 @@ async function runCrashChild(mode: string): Promise { if (mode === 'inside_t2' && runtimeInsertCount === 2) blockUntilKilled(); if (mode === 'inside_workspace_terminal' && runtimeInsertCount === 2) blockUntilKilled(); } - if (point === 'after_recovery_reconcile' && mode === 'inside_recovery_reconcile') { - blockUntilKilled(); - } - if (point === 'after_recovery_outcome' && mode === 'inside_recovery_outcome') { - blockUntilKilled(); - } - if (point === 'after_recovery_decision' && mode === 'inside_recovery_decision') { - blockUntilKilled(); - } if (point === 'after_workspace_version_event_insert' && mode === 'inside_workspace_baseline') { blockUntilKilled(); } @@ -394,11 +346,6 @@ async function runCrashChild(mode: string): Promise { writeFileSync(markerPath, 'effect-happened'); blockUntilKilled(); } - if (mode.startsWith('inside_recovery_') || mode === 'after_recovery_commit') { - await store.commitToolRecoveryBundle(recoveryCommit()); - if (mode === 'after_recovery_commit') blockUntilKilled(); - throw new Error(`Recovery crash mode ${mode} missed its failpoint`); - } await store.commitToolOutcome(outcomeCommit()); if (mode === 'after_t2') blockUntilKilled(); throw new Error(`Unknown crash child mode ${mode}`); @@ -724,75 +671,3 @@ function functionResponseEvent(): RuntimeEvent { refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, }; } - -function recoveryCommit() { - return { - operationId: 'operation-1', - reconcileRuntimeEvent: reconcileEvent(), - outcomeRuntimeEvent: functionResponseEvent(), - decisionRuntimeEvent: decisionEvent(), - }; -} - -function reconcileEvent(): RuntimeEvent { - return { - id: 'reconcile-event-1', - invocationId: 'invocation-1', - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - ts: 2, - partial: false, - role: 'system', - author: 'system', - actions: { - toolRecovery: { - kind: 'maka.tool.reconcile_result', - version: 1, - payload: { - protocol: 'tool_reconcile_v1', - operationId: 'operation-1', - observation: 'matches_expected_state', - observationSchema: 'state_identity_v1', - observationDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - }, - }, - }, - refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, - }; -} - -function decisionEvent(): RuntimeEvent { - return { - id: 'decision-event-1', - invocationId: 'invocation-1', - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - ts: 3, - partial: false, - role: 'system', - author: 'system', - actions: { - toolRecovery: { - kind: 'maka.tool.recovery_decision', - version: 1, - payload: { - protocol: 'tool_recovery_v1', - operationId: 'operation-1', - disposition: 'completed', - reasonCode: 'reconcile_matches_expected_state', - outcomeEventId: 'response-event-1', - evidenceEventIds: [ - 'call-event-1', - 'dispatch-event-1', - 'reconcile-event-1', - 'response-event-1', - ], - }, - }, - }, - refs: { operationId: 'operation-1', toolCallId: 'provider-call-1' }, - }; -} diff --git a/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts b/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts index 5ced11195c3..193bb26050f 100644 --- a/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts +++ b/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts @@ -962,17 +962,6 @@ describe('workspace version persistence authority', () => { }), /workspace version authority writer/i, ); - await assert.rejects( - store.commitToolRecoveryBundle({ - operationId: 'workspace-bypass-operation', - reconcileRuntimeEvent: epochOpenedEvent, - decisionRuntimeEvent: { - ...epochOpenedEvent, - id: 'workspace-bypass-decision', - }, - }), - /workspace version authority writer/i, - ); }); }); diff --git a/packages/storage/src/agent-run-store-contract.ts b/packages/storage/src/agent-run-store-contract.ts index 70e23b5339e..1360cddd407 100644 --- a/packages/storage/src/agent-run-store-contract.ts +++ b/packages/storage/src/agent-run-store-contract.ts @@ -503,7 +503,7 @@ export function assertNoReservedToolLedgerFact(event: RuntimeEvent): void { const validation = validateGenericToolLedgerAppend(event); if (validation.ok) return; if (validation.code === 'reserved_recovery_fact') { - throw new Error('Tool recovery facts require the atomic recovery bundle writer'); + throw new Error('Historical tool recovery facts are read-only'); } if (validation.code === 'reserved_tool_boundary_fact') { throw new Error('Durable tool facts require the atomic tool boundary writer'); diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index ceb3cf9b188..5551af0c4a4 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -73,13 +73,10 @@ import { import { RunSealedError, RUNTIME_CONTINUATION_AUTHORITY_V1, - TOOL_RECOVERY_BUNDLE_CAPABILITY_V1, type ContinuationClaimResult, type ContinuationClaimStateV1, type RuntimeContinuationAuthorityStore, type RuntimeInvocationRecoveryInventoryEntry, - type RuntimeRecoveryBundleCommit, - type RuntimeRecoveryBundleStore, type RuntimeSessionEventSnapshot, type RuntimeWorkspaceVersionAuthorityStore, } from '@maka/core/runtime-event-store'; @@ -91,7 +88,6 @@ import { type RuntimeInvocationRecord, type RuntimeInvocationSearchResult, } from '@maka/core/runtime-invocation'; -import type { ToolRecoveryDecisionFact } from '@maka/core/tool-recovery-fact'; import { stableJsonStringify } from '@maka/core/tool-args-identity'; import { type RuntimePartialSnapshot, @@ -127,10 +123,7 @@ import { type ImmutableRuntimePrefixProofV1, type RuntimeBoundaryDigest, } from '@maka/core/runtime-boundary'; -import { - assertToolRecoveryEventBundle, - interpretScannedToolRecovery, -} from '@maka/core/tool-recovery-bundle'; +import { interpretScannedToolRecovery } from '@maka/core/tool-recovery-bundle'; import { configureSqliteRuntimeDatabase, configureSqliteRuntimeAutoVacuumForNewDatabase, @@ -253,9 +246,6 @@ export type ToolJournalState = export type SqliteRuntimeStoreFailpoint = | 'after_runtime_event_insert' | 'after_journal_event_insert' - | 'after_recovery_reconcile' - | 'after_recovery_outcome' - | 'after_recovery_decision' | 'after_recovery_terminal_insert' | 'after_recovery_terminal_projection' | 'after_continuation_claim_insert' @@ -309,14 +299,10 @@ export function createSqliteRuntimeStore( } export class SqliteRuntimeStore - implements - RuntimeRecoveryBundleStore, - RuntimeContinuationAuthorityStore, - RuntimeWorkspaceVersionAuthorityStore + implements RuntimeContinuationAuthorityStore, RuntimeWorkspaceVersionAuthorityStore { readonly durability = 'canonical' as const; readonly toolBoundaryProtocol = 't1_after_preflight_v1' as const; - readonly recoveryBundleCapability = TOOL_RECOVERY_BUNDLE_CAPABILITY_V1; readonly continuationAuthorityCapability = RUNTIME_CONTINUATION_AUTHORITY_V1; readonly workspaceVersionAuthorityCapability = WORKSPACE_VERSION_AUTHORITY_CAPABILITY_V1; private readonly db: DatabaseSync; @@ -3565,84 +3551,6 @@ export class SqliteRuntimeStore return this.transaction(() => this.commitToolOutcomeSync(canonicalInput)); } - async commitToolRecoveryBundle(input: RuntimeRecoveryBundleCommit): Promise { - const canonicalInput: RuntimeRecoveryBundleCommit = { - ...input, - reconcileRuntimeEvent: canonicalizeRuntimeEventForStorage(input.reconcileRuntimeEvent), - ...(input.outcomeRuntimeEvent - ? { outcomeRuntimeEvent: canonicalizeRuntimeEventForStorage(input.outcomeRuntimeEvent) } - : {}), - decisionRuntimeEvent: canonicalizeRuntimeEventForStorage(input.decisionRuntimeEvent), - }; - assertNoReservedWorkspaceAuthorityAppend(canonicalInput.reconcileRuntimeEvent); - assertNoReservedWorkspaceAuthorityAppend(canonicalInput.decisionRuntimeEvent); - if (canonicalInput.outcomeRuntimeEvent) { - assertNoReservedWorkspaceAuthorityAppend(canonicalInput.outcomeRuntimeEvent); - assertNoReservedRecoveryFact(canonicalInput.outcomeRuntimeEvent); - } - this.transaction(() => { - const operation = this.readToolOperationSync(canonicalInput.operationId); - if (!operation) throw new Error(`Unknown tool operation ${canonicalInput.operationId}`); - if (!operation.dispatchEventId) { - throw new Error('Recovery bundle requires a durable dispatch RuntimeEvent'); - } - assertToolRecoveryEventBundle({ - operation: recoveryOperationIdentity(operation), - callEvent: this.readRequiredRuntimeEvent(operation.callEventId), - dispatchEvent: this.readRequiredRuntimeEvent(operation.dispatchEventId), - reconcileEvent: canonicalInput.reconcileRuntimeEvent, - outcomeEvent: canonicalInput.outcomeRuntimeEvent, - decisionEvent: canonicalInput.decisionRuntimeEvent, - }); - assertStrictRuntimeEventOrder([ - this.runtimeEventSeq(operation.callEventId), - this.runtimeEventSeq(operation.dispatchEventId), - ]); - this.assertToolLedgerTransition( - [ - canonicalInput.reconcileRuntimeEvent, - ...(canonicalInput.outcomeRuntimeEvent ? [canonicalInput.outcomeRuntimeEvent] : []), - canonicalInput.decisionRuntimeEvent, - ], - 'recovery_bundle', - ); - if (operation.currentState !== 'prepared' || operation.resultEventId !== undefined) { - this.assertExactRecoveryBundleAlreadyCommitted(canonicalInput, operation); - return; - } - - this.commitRecoveryFactSync( - operation, - canonicalInput.reconcileRuntimeEvent, - 'reconcile_observed', - ); - this.options.failpoint?.('after_recovery_reconcile'); - if (canonicalInput.outcomeRuntimeEvent) { - this.commitToolOutcomeSync({ - operationId: canonicalInput.operationId, - journalEventId: `${canonicalInput.operationId}_outcome`, - runtimeEvent: canonicalInput.outcomeRuntimeEvent, - committedAt: canonicalInput.outcomeRuntimeEvent.ts, - }); - this.options.failpoint?.('after_recovery_outcome'); - } - - const decision = canonicalInput.decisionRuntimeEvent.actions?.toolRecovery; - if (!decision || decision.kind !== 'maka.tool.recovery_decision') { - throw new Error('Recovery bundle requires a recovery decision'); - } - const current = this.readToolOperationSync(canonicalInput.operationId); - if (!current) throw new Error(`Unknown tool operation ${canonicalInput.operationId}`); - this.commitRecoveryFactSync( - current, - canonicalInput.decisionRuntimeEvent, - decision.payload.disposition === 'completed' ? 'recovery_completed' : 'recovery_parked', - decision.payload, - ); - this.options.failpoint?.('after_recovery_decision'); - }); - } - async readToolOperation(operationId: string): Promise { return this.readToolOperationSync(operationId); } @@ -4077,57 +3985,6 @@ export class SqliteRuntimeStore return { created: true, runtimeEventSeq }; } - private commitRecoveryFactSync( - operation: ToolOperationRecord, - event: RuntimeEvent, - state: 'reconcile_observed' | 'recovery_completed' | 'recovery_parked', - decision?: ToolRecoveryDecisionFact, - ): void { - this.insertRuntimeEvent(event, event.ts, false); - this.options.failpoint?.('after_runtime_event_insert'); - this.insertToolJournalEvent(operation, event, state); - if (state === 'reconcile_observed') { - const updated = this.db - .prepare('UPDATE tool_operations SET version = version + 1 WHERE operation_id = ?') - .run(operation.operationId); - if (updated.changes !== 1) { - throw new Error(`Tool operation compare-and-set failed for ${operation.operationId}`); - } - return; - } - - if ( - state === 'recovery_completed' && - (decision?.disposition !== 'completed' || - operation.currentState !== 'outcome_committed' || - operation.resultEventId !== decision.outcomeEventId) - ) { - throw new Error('Completed recovery decision does not match the persisted outcome'); - } - if ( - state === 'recovery_parked' && - (decision?.disposition !== 'parked' || - operation.currentState !== 'prepared' || - operation.resultEventId !== undefined) - ) { - throw new Error('Parked recovery decision does not match the prepared operation'); - } - const updated = this.db - .prepare(` - UPDATE tool_operations - SET current_state = ?, version = version + 1 - WHERE operation_id = ? AND current_state = ? - `) - .run( - state, - operation.operationId, - state === 'recovery_completed' ? 'outcome_committed' : 'prepared', - ); - if (updated.changes !== 1) { - throw new Error(`Tool operation compare-and-set failed for ${operation.operationId}`); - } - } - private insertToolJournalEvent( operation: ToolOperationRecord, event: RuntimeEvent, @@ -4158,39 +4015,6 @@ export class SqliteRuntimeStore this.options.failpoint?.('after_journal_event_insert'); } - private assertExactRecoveryBundleAlreadyCommitted( - input: RuntimeRecoveryBundleCommit, - operation: ToolOperationRecord, - ): void { - const decision = input.decisionRuntimeEvent.actions?.toolRecovery; - const completed = - decision?.kind === 'maka.tool.recovery_decision' && - decision.payload.disposition === 'completed'; - if ( - (completed && - (!input.outcomeRuntimeEvent || - operation.currentState !== 'recovery_completed' || - operation.resultEventId !== input.outcomeRuntimeEvent.id)) || - (!completed && - (input.outcomeRuntimeEvent !== undefined || - operation.currentState !== 'recovery_parked' || - operation.resultEventId !== undefined)) - ) { - throw new Error(`Tool operation ${operation.operationId} is already settled`); - } - for (const event of [ - input.reconcileRuntimeEvent, - ...(input.outcomeRuntimeEvent ? [input.outcomeRuntimeEvent] : []), - input.decisionRuntimeEvent, - ]) { - const stored = this.readRuntimeEventJson(event.id); - if (stored === undefined) { - throw new Error(`Tool recovery bundle is incomplete for ${operation.operationId}`); - } - assertStoredRuntimeEventEquals(event, stored); - } - } - private transaction(operation: () => T): T { if (this.databaseLease) { return this.databaseLease.transaction('write', operation); @@ -5154,7 +4978,7 @@ function canonicalizeRuntimeEventForStorage(event: RuntimeEvent): RuntimeEvent { function assertNoReservedRecoveryFact(event: RuntimeEvent): void { if (event.actions?.toolRecovery !== undefined) { - throw new Error('Tool recovery facts require the atomic recovery bundle writer'); + throw new Error('Historical tool recovery facts are read-only'); } } @@ -5166,7 +4990,7 @@ function assertNoReservedToolLedgerFact(event: RuntimeEvent): void { const validation = validateGenericToolLedgerAppend(event); if (validation.ok) return; if (validation.code === 'reserved_recovery_fact') { - throw new Error('Tool recovery facts require the atomic recovery bundle writer'); + throw new Error('Historical tool recovery facts are read-only'); } if (validation.code === 'reserved_tool_boundary_fact') { throw new Error('Durable tool facts require the atomic tool boundary writer'); @@ -5183,34 +5007,6 @@ function isToolLedgerBearingEvent(event: RuntimeEvent): boolean { ); } -function recoveryOperationIdentity(operation: ToolOperationRecord) { - if (!operation.dispatchEventId) { - throw new Error('Recovery bundle requires a durable dispatch RuntimeEvent'); - } - return { - operationId: operation.operationId, - invocationId: operation.invocationId, - runId: operation.runId, - turnId: operation.turnId, - providerToolCallId: operation.providerToolCallId, - toolName: operation.toolName, - canonicalArgsHash: operation.canonicalArgsHash, - recoveryMode: operation.recoveryMode, - callEventId: operation.callEventId, - dispatchEventId: operation.dispatchEventId, - }; -} - -function assertStrictRuntimeEventOrder(eventSequences: readonly number[]): void { - if ( - eventSequences.some( - (eventSequence, index) => index > 0 && eventSequence <= (eventSequences[index - 1] ?? -1), - ) - ) { - throw new Error('Recovery facts violate canonical RuntimeEvent causal order'); - } -} - function requireRuntimeEventOrder( eventOrder: ReadonlyMap, eventId: string, diff --git a/packages/storage/src/tool-commit-validation.ts b/packages/storage/src/tool-commit-validation.ts index 736a5cda883..e3460db8da0 100644 --- a/packages/storage/src/tool-commit-validation.ts +++ b/packages/storage/src/tool-commit-validation.ts @@ -137,5 +137,5 @@ export function assertOutcomeIdentity(operation: ToolOperationRecord, event: Run function assertNoReservedRecoveryFact(event: RuntimeEvent): void { if (event.actions?.toolRecovery !== undefined) - throw new Error('Tool recovery facts require the atomic recovery bundle writer'); + throw new Error('Historical tool recovery facts are read-only'); } From e88431e1426c92adfec76bfabb5862bad49abeca Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 16 Oct 2026 15:22:43 +0800 Subject: [PATCH 2/7] test(runtime): drop assertions for retired recovery declarations Keep the Plan schema and Graph behavior checks while removing four assertions for declarations deleted by the reconciliation cleanup. Both affected suites pass (12 tests). Generated-by: Codex --- packages/runtime/src/__tests__/plan-mode.test.ts | 1 - .../src/__tests__/stream-graph-supervisor-tools.test.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/packages/runtime/src/__tests__/plan-mode.test.ts b/packages/runtime/src/__tests__/plan-mode.test.ts index 487bba3846c..c4b486c41b9 100644 --- a/packages/runtime/src/__tests__/plan-mode.test.ts +++ b/packages/runtime/src/__tests__/plan-mode.test.ts @@ -27,7 +27,6 @@ import type { MakaTool } from '../tool-runtime.js'; describe('Plan Mode tool surface', () => { test('requires plain-text step titles and descriptions', () => { const submitPlan = buildSubmitPlanTool({} as never); - assert.equal(submitPlan.recoveryMode, 'idempotent'); const schema = submitPlan.parameters as { safeParse(input: unknown): { success: boolean }; }; diff --git a/packages/runtime/src/__tests__/stream-graph-supervisor-tools.test.ts b/packages/runtime/src/__tests__/stream-graph-supervisor-tools.test.ts index e81b3368992..d98cb624d2d 100644 --- a/packages/runtime/src/__tests__/stream-graph-supervisor-tools.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-supervisor-tools.test.ts @@ -46,9 +46,6 @@ describe('stream graph supervisor tools', () => { assert.equal(viewTool.name, VIEW_AGENT_GRAPH_TOOL_NAME); assert.equal(updateTool.name, UPDATE_AGENT_GRAPH_TOOL_NAME); assert.equal(yieldTool.name, YIELD_AGENT_GRAPH_TOOL_NAME); - assert.equal(viewTool.recoveryMode, 'replay_safe'); - assert.equal(updateTool.recoveryMode, 'idempotent'); - assert.equal(yieldTool.recoveryMode, 'replay_safe'); await assert.rejects( async () => From 996b6b027b35e8e25d9be2397d11ce6aba17b6aa Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 16 Oct 2026 15:22:44 +0800 Subject: [PATCH 3/7] fix(ci): account for shipped marked dependency in artifact ledger The artifact-budget gate omits marked@18.0.11, already required by core image Markdown parsing. Reproduced the identical missing entry on clean upstream main after npm ci. Refresh the Linux ledger by one existing production instance; do not change dependencies or checker behavior. Generated-by: Codex --- apps/desktop/artifact-budget.json | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/desktop/artifact-budget.json b/apps/desktop/artifact-budget.json index fc70f8ca4b0..9d279a6508a 100644 --- a/apps/desktop/artifact-budget.json +++ b/apps/desktop/artifact-budget.json @@ -162,6 +162,7 @@ "node_modules/lodash.merge@4.6.2", "node_modules/lodash.pickby@4.6.0", "node_modules/long@5.3.2", + "node_modules/marked@18.0.11", "node_modules/math-intrinsics@1.1.0", "node_modules/minimatch@10.2.5", "node_modules/minipass@7.1.3", From 2e03e633892fb81929c3fe71112d71e0da53d587 Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 16 Oct 2026 15:27:37 +0800 Subject: [PATCH 4/7] fix(ci): refresh existing AppShell architecture metric Regenerate the renderer architecture ledger to match the unchanged AppShell source: nonTriviaTokens is 7280, not 7285. The existing strict-base gate derived 7280 from upstream main as well. All 168 checker fixtures and the strict comparison against main pass without changing renderer code or checker rules. Generated-by: Codex --- apps/desktop/renderer-architecture.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/renderer-architecture.json b/apps/desktop/renderer-architecture.json index 16d40288db0..f3a9c66c71b 100644 --- a/apps/desktop/renderer-architecture.json +++ b/apps/desktop/renderer-architecture.json @@ -743,7 +743,7 @@ "react": 1 }, "importSpecifiers": 43, - "nonTriviaTokens": 7285 + "nonTriviaTokens": 7280 }, "src/renderer/use-app-shell-session-list.ts": { "importDeclarations": 3, From d2c0c7dfbd14e4108ac381c18210f6d78a27c524 Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 16 Oct 2026 15:43:36 +0800 Subject: [PATCH 5/7] revert: leave upstream CI ledger corrections to main Keep this PR scoped to removing unused reconciliation and fixing its stale Plan/Graph assertions. Revert the artifact and renderer architecture ledger adjustments; upstream baseline failures are intentionally left for main to resolve. Generated-by: Codex --- apps/desktop/artifact-budget.json | 1 - apps/desktop/renderer-architecture.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/desktop/artifact-budget.json b/apps/desktop/artifact-budget.json index 9d279a6508a..fc70f8ca4b0 100644 --- a/apps/desktop/artifact-budget.json +++ b/apps/desktop/artifact-budget.json @@ -162,7 +162,6 @@ "node_modules/lodash.merge@4.6.2", "node_modules/lodash.pickby@4.6.0", "node_modules/long@5.3.2", - "node_modules/marked@18.0.11", "node_modules/math-intrinsics@1.1.0", "node_modules/minimatch@10.2.5", "node_modules/minipass@7.1.3", diff --git a/apps/desktop/renderer-architecture.json b/apps/desktop/renderer-architecture.json index f3a9c66c71b..16d40288db0 100644 --- a/apps/desktop/renderer-architecture.json +++ b/apps/desktop/renderer-architecture.json @@ -743,7 +743,7 @@ "react": 1 }, "importSpecifiers": 43, - "nonTriviaTokens": 7280 + "nonTriviaTokens": 7285 }, "src/renderer/use-app-shell-session-list.ts": { "importDeclarations": 3, From d7f30ba35e16f7e2cdc5214f0387b90f1b3291c7 Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 16 Oct 2026 17:25:07 +0800 Subject: [PATCH 6/7] test(runtime): cover direct reads of old recovery projections --- .../fixtures/legacy-tool-recovery-database.ts | 69 +++++++++++++ .../recovery-authority-equivalence.test.ts | 96 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 packages/runtime/src/__tests__/fixtures/legacy-tool-recovery-database.ts diff --git a/packages/runtime/src/__tests__/fixtures/legacy-tool-recovery-database.ts b/packages/runtime/src/__tests__/fixtures/legacy-tool-recovery-database.ts new file mode 100644 index 00000000000..41ee3c43b08 --- /dev/null +++ b/packages/runtime/src/__tests__/fixtures/legacy-tool-recovery-database.ts @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * SQL row snapshots captured from the retired writer at 331340aba5c7de5fc7d53e2441123f40f03ba34b. + * Built @maka/core and @maka/storage at that revision, then used the event helpers + * in recovery-persistence-authority.test.ts with commitToolPrepared (committedAt: 10) + * and commitToolRecoveryBundle (completed / parked); prepared stops after T1. + * Dumped SELECT * from each table below after closing the writer. These are its + * actual persisted event, operation, journal and session-ordinal rows, not rebuilt + * projections. Restore into an empty store schema before opening with current code. + */ +export const LEGACY_TOOL_RECOVERY_DATABASE_SQL = { + completed: ` +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('call-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 1, 'function_call', '{"author":"agent","content":{"args":{"content":"after","path":"notes.txt"},"id":"provider-call-1","kind":"function_call","name":"Write"},"id":"call-event-1","invocationId":"invocation-1","partial":false,"role":"model","runId":"run-1","sessionId":"session-1","ts":1,"turnId":"turn-1"}', 10); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('dispatch-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 2, 'tool_dispatch', '{"actions":{"toolDispatch":{"canonicalArgsHash":"sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9","operationId":"operation-1","protocol":"t1_after_preflight_v1","providerToolCallId":"provider-call-1","recoveryMode":"reconcile","toolName":"Write"}},"author":"system","id":"dispatch-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":1,"turnId":"turn-1"}', 10); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('reconcile-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 3, 'runtime_fact', '{"actions":{"toolRecovery":{"kind":"maka.tool.reconcile_result","payload":{"observation":"matches_expected_state","observationDigest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","observationSchema":"state_identity_v1","operationId":"operation-1","protocol":"tool_reconcile_v1"},"version":1}},"author":"system","id":"reconcile-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":2,"turnId":"turn-1"}', 2); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('outcome-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 4, 'function_response', '{"author":"tool","content":{"id":"provider-call-1","isError":false,"kind":"function_response","name":"Write","result":"ok"},"id":"outcome-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"tool","runId":"run-1","sessionId":"session-1","ts":3,"turnId":"turn-1"}', 3); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('decision-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 5, 'runtime_fact', '{"actions":{"toolRecovery":{"kind":"maka.tool.recovery_decision","payload":{"disposition":"completed","evidenceEventIds":["call-event-1","dispatch-event-1","reconcile-event-1","outcome-event-1"],"operationId":"operation-1","outcomeEventId":"outcome-event-1","protocol":"tool_recovery_v1","reasonCode":"reconcile_matches_expected_state"},"version":1}},"author":"system","id":"decision-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":4,"turnId":"turn-1"}', 4); +INSERT INTO tool_operations (operation_id, invocation_id, run_id, turn_id, provider_tool_call_id, tool_name, canonical_args_hash, recovery_mode, current_state, call_event_id, result_event_id, version, dispatch_event_id) VALUES ('operation-1', 'invocation-1', 'run-1', 'turn-1', 'provider-call-1', 'Write', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', 'recovery_completed', 'call-event-1', 'outcome-event-1', 4, 'dispatch-event-1'); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (1, 'operation-1_prepared', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'prepared', 'dispatch-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, NULL, 10); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (2, 'reconcile-event-1_journal', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'reconcile_observed', 'reconcile-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, '{"kind":"maka.tool.reconcile_result","payload":{"observation":"matches_expected_state","observationDigest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","observationSchema":"state_identity_v1","operationId":"operation-1","protocol":"tool_reconcile_v1"},"version":1}', 2); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (3, 'operation-1_outcome', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'outcome_committed', 'outcome-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, NULL, 3); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (4, 'decision-event-1_journal', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'recovery_completed', 'decision-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, '{"kind":"maka.tool.recovery_decision","payload":{"disposition":"completed","evidenceEventIds":["call-event-1","dispatch-event-1","reconcile-event-1","outcome-event-1"],"operationId":"operation-1","outcomeEventId":"outcome-event-1","protocol":"tool_recovery_v1","reasonCode":"reconcile_matches_expected_state"},"version":1}', 4); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 1, 'call-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 2, 'dispatch-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 3, 'reconcile-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 4, 'outcome-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 5, 'decision-event-1'); +`, + parked: ` +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('call-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 1, 'function_call', '{"author":"agent","content":{"args":{"content":"after","path":"notes.txt"},"id":"provider-call-1","kind":"function_call","name":"Write"},"id":"call-event-1","invocationId":"invocation-1","partial":false,"role":"model","runId":"run-1","sessionId":"session-1","ts":1,"turnId":"turn-1"}', 10); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('dispatch-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 2, 'tool_dispatch', '{"actions":{"toolDispatch":{"canonicalArgsHash":"sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9","operationId":"operation-1","protocol":"t1_after_preflight_v1","providerToolCallId":"provider-call-1","recoveryMode":"reconcile","toolName":"Write"}},"author":"system","id":"dispatch-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":1,"turnId":"turn-1"}', 10); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('reconcile-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 3, 'runtime_fact', '{"actions":{"toolRecovery":{"kind":"maka.tool.reconcile_result","payload":{"observation":"diverged","observationDigest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","observationSchema":"state_identity_v1","operationId":"operation-1","protocol":"tool_reconcile_v1"},"version":1}},"author":"system","id":"reconcile-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":2,"turnId":"turn-1"}', 2); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('decision-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 4, 'runtime_fact', '{"actions":{"toolRecovery":{"kind":"maka.tool.recovery_decision","payload":{"disposition":"parked","evidenceEventIds":["call-event-1","dispatch-event-1","reconcile-event-1"],"operationId":"operation-1","protocol":"tool_recovery_v1","reasonCode":"reconcile_diverged"},"version":1}},"author":"system","id":"decision-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":4,"turnId":"turn-1"}', 4); +INSERT INTO tool_operations (operation_id, invocation_id, run_id, turn_id, provider_tool_call_id, tool_name, canonical_args_hash, recovery_mode, current_state, call_event_id, result_event_id, version, dispatch_event_id) VALUES ('operation-1', 'invocation-1', 'run-1', 'turn-1', 'provider-call-1', 'Write', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', 'recovery_parked', 'call-event-1', NULL, 3, 'dispatch-event-1'); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (1, 'operation-1_prepared', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'prepared', 'dispatch-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, NULL, 10); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (2, 'reconcile-event-1_journal', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'reconcile_observed', 'reconcile-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, '{"kind":"maka.tool.reconcile_result","payload":{"observation":"diverged","observationDigest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","observationSchema":"state_identity_v1","operationId":"operation-1","protocol":"tool_reconcile_v1"},"version":1}', 2); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (3, 'decision-event-1_journal', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'recovery_parked', 'decision-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, '{"kind":"maka.tool.recovery_decision","payload":{"disposition":"parked","evidenceEventIds":["call-event-1","dispatch-event-1","reconcile-event-1"],"operationId":"operation-1","protocol":"tool_recovery_v1","reasonCode":"reconcile_diverged"},"version":1}', 4); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 1, 'call-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 2, 'dispatch-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 3, 'reconcile-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 4, 'decision-event-1'); +`, + prepared: ` +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('call-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 1, 'function_call', '{"author":"agent","content":{"args":{"content":"after","path":"notes.txt"},"id":"provider-call-1","kind":"function_call","name":"Write"},"id":"call-event-1","invocationId":"invocation-1","partial":false,"role":"model","runId":"run-1","sessionId":"session-1","ts":1,"turnId":"turn-1"}', 10); +INSERT INTO runtime_events (event_id, session_id, invocation_id, run_id, turn_id, event_seq, event_kind, payload_json, committed_at) VALUES ('dispatch-event-1', 'session-1', 'invocation-1', 'run-1', 'turn-1', 2, 'tool_dispatch', '{"actions":{"toolDispatch":{"canonicalArgsHash":"sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9","operationId":"operation-1","protocol":"t1_after_preflight_v1","providerToolCallId":"provider-call-1","recoveryMode":"reconcile","toolName":"Write"}},"author":"system","id":"dispatch-event-1","invocationId":"invocation-1","partial":false,"refs":{"operationId":"operation-1","toolCallId":"provider-call-1"},"role":"system","runId":"run-1","sessionId":"session-1","ts":1,"turnId":"turn-1"}', 10); +INSERT INTO tool_operations (operation_id, invocation_id, run_id, turn_id, provider_tool_call_id, tool_name, canonical_args_hash, recovery_mode, current_state, call_event_id, result_event_id, version, dispatch_event_id) VALUES ('operation-1', 'invocation-1', 'run-1', 'turn-1', 'provider-call-1', 'Write', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', 'prepared', 'call-event-1', NULL, 1, 'dispatch-event-1'); +INSERT INTO tool_journal_events (journal_seq, journal_event_id, operation_id, invocation_id, run_id, turn_id, state, runtime_event_id, canonical_args_hash, recovery_mode, external_handle, metadata_json, committed_at) VALUES (1, 'operation-1_prepared', 'operation-1', 'invocation-1', 'run-1', 'turn-1', 'prepared', 'dispatch-event-1', 'sha256:763712445cbfb7feebe3bd4ba14e29425f05e40b8cd14aef0896853dca24b4d9', 'reconcile', NULL, NULL, 10); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 1, 'call-event-1'); +INSERT INTO runtime_session_event_ordinals (session_id, ordinal, event_id) VALUES ('session-1', 2, 'dispatch-event-1'); +`, +} as const; diff --git a/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts b/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts index d536a577e92..f4193c6629e 100644 --- a/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts +++ b/packages/runtime/src/__tests__/recovery-authority-equivalence.test.ts @@ -28,6 +28,7 @@ import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { createSqliteRuntimeStore } from '@maka/storage/sqlite-runtime-store'; import { resolveRuntimeRecovery } from '../recovery-resolver.js'; import { buildResumePlanFromRuntimeEvents } from '../runtime-resume.js'; +import { LEGACY_TOOL_RECOVERY_DATABASE_SQL } from './fixtures/legacy-tool-recovery-database.js'; const ARGS_HASH = canonicalToolArgsHash('Write', { path: 'notes.txt', @@ -37,6 +38,101 @@ const OBSERVATION_DIGEST = 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' as const; describe('recovery authority equivalence', () => { + for (const scenario of ['completed', 'parked', 'prepared'] as const) { + it(`opens old-writer ${scenario} rows and plans resume without rebuilding`, async () => { + const root = await mkdtemp(join(tmpdir(), `maka-old-writer-${scenario}-`)); + const dbPath = join(root, 'runtime.sqlite'); + try { + // The schema is unchanged by writer removal. Restore captured old-writer + // rows into an empty schema, including its already-materialized projections. + createSqliteRuntimeStore(dbPath).close(); + const db = new DatabaseSync(dbPath); + try { + db.exec(LEGACY_TOOL_RECOVERY_DATABASE_SQL[scenario]); + } finally { + db.close(); + } + + const expectedStates = + scenario === 'completed' + ? ['prepared', 'reconcile_observed', 'outcome_committed', 'recovery_completed'] + : scenario === 'parked' + ? ['prepared', 'reconcile_observed', 'recovery_parked'] + : ['prepared']; + const store = createSqliteRuntimeStore(dbPath); + try { + const operation = await store.readToolOperation('operation-1'); + assert.deepEqual(operation, { + operationId: 'operation-1', + invocationId: 'invocation-1', + runId: 'run-1', + turnId: 'turn-1', + providerToolCallId: 'provider-call-1', + toolName: 'Write', + canonicalArgsHash: ARGS_HASH, + recoveryMode: 'reconcile', + currentState: expectedStates.at(-1), + callEventId: 'call-event-1', + dispatchEventId: 'dispatch-event-1', + ...(scenario === 'completed' ? { resultEventId: 'outcome-event-1' } : {}), + version: expectedStates.length, + }); + const journal = await store.readToolJournal('operation-1'); + assert.deepEqual( + journal.map(({ state }) => state), + expectedStates, + ); + assert.deepEqual( + journal.map(({ journalEventId }) => journalEventId), + scenario === 'completed' + ? [ + 'operation-1_prepared', + 'reconcile-event-1_journal', + 'operation-1_outcome', + 'decision-event-1_journal', + ] + : scenario === 'parked' + ? ['operation-1_prepared', 'reconcile-event-1_journal', 'decision-event-1_journal'] + : ['operation-1_prepared'], + ); + assert.deepEqual( + await store.listUnsettledToolOperations(), + scenario === 'prepared' ? [{ sessionId: 'session-1', ...operation }] : [], + ); + assert.deepEqual( + await store.listUnsettledToolOperations('session-1'), + scenario === 'prepared' ? [{ sessionId: 'session-1', ...operation }] : [], + ); + + const events = await store.readImmutableRuntimeEvents('session-1', 'run-1'); + assert.equal(events.length, scenario === 'completed' ? 5 : scenario === 'parked' ? 4 : 2); + const resolution = resolveRuntimeRecovery(events); + assert.equal(resolution.hasCorruption, false); + assert.equal(resolution.decisions.length, 1); + assert.equal( + resolution.decisions[0]?.status, + scenario === 'prepared' ? 'indeterminate' : scenario, + ); + const plan = buildResumePlanFromRuntimeEvents(events); + assert.equal(plan.disposition, scenario === 'completed' ? 'safe_replay' : 'blocked'); + assert.equal(plan.requiresVerification, scenario === 'prepared'); + assert.deepEqual( + plan.rejectionReasons, + scenario === 'completed' ? [] : ['dangling_tool_state'], + ); + assert.equal( + events.filter((event) => event.content?.kind === 'function_response').length, + scenario === 'completed' ? 1 : 0, + ); + } finally { + store.close(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + } + it('gives historical projection, reopen, rebuild, and Resolver one terminal interpretation', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-recovery-equivalence-')); const dbPath = join(root, 'runtime.sqlite'); From 3f013181e907a03380d7505ca2f7266668f90d9e Mon Sep 17 00:00:00 2001 From: me2seeks Date: Mon, 19 Oct 2026 13:54:57 +0800 Subject: [PATCH 7/7] refactor(storage): remove orphaned recovery validator Generated-by: Codex --- packages/storage/src/sqlite-runtime-store.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index 5551af0c4a4..81d5cae42fc 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -4976,12 +4976,6 @@ function canonicalizeRuntimeEventForStorage(event: RuntimeEvent): RuntimeEvent { return encodeCanonicalRuntimeEvent(event).event; } -function assertNoReservedRecoveryFact(event: RuntimeEvent): void { - if (event.actions?.toolRecovery !== undefined) { - throw new Error('Historical tool recovery facts are read-only'); - } -} - function assertNoReservedToolLedgerFact(event: RuntimeEvent): void { assertNoReservedWorkspaceAuthorityAppend(event); if (event.actions?.continuationStart !== undefined) {