Watchdog v1 (compare mode) + release bundle versioning

.github/actions/load-toolchain-pins/action.yml

@@ -0,0 +1,9 @@
+name: "Load toolchain pins"
+description: "Export toolchain-pins.env into the job environment."
+
+runs:
+  using: composite
+  steps:
+    - name: Load toolchain-pins.env
+      shell: bash
+      run: bash scripts/load-toolchain-pins.sh "${GITHUB_ENV}"

.github/workflows/ci.yml

@@ -4,15 +4,6 @@ on:
   push:
   pull_request:
 
-env:
-  RUST_TOOLCHAIN: "1.95.0"
-  XGENEXT2FS_VERSION: v1.5.6
-  XGENEXT2FS_SHA256_AMD64: 996e4e68a638b5dc5967d3410f92ecb8d2f41e32218bbe0f8b4c4474d7eebc59
-  XGENEXT2FS_SHA256_ARM64: e5aca81164b762bbe5447bacef41e4fa9e357fd9c8f44e519c5206227d43144d
-  CARTESI_MACHINE_VERSION: v0.20.0-test2
-  CARTESI_MACHINE_SHA256_AMD64: 39bbfc96a6cc6606307294b719df65f4f2725e8d200d062bcbd8c22355b99b56
-  CARTESI_MACHINE_SHA256_ARM64: 787d823756000cdecd72da8a3494b4c08613087379035959e561bbaef7a220ba
-
 jobs:
   rust:
     runs-on: ubuntu-latest
@@ -22,6 +13,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
+      - name: Verify toolchain pin alignment
+        run: bash scripts/verify-toolchain-pins.sh
+
       - name: Install system dependencies
         run: |
           sudo apt-get update
@@ -55,6 +52,12 @@ jobs:
       - name: Clippy
         run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
 
+      - name: Watchdog Lua tests
+        run: lua watchdog/tests/run.lua
+
+      - name: Watchdog divergence drill
+        run: bash scripts/test-watchdog-divergence-drill.sh
+
       - name: Test
         timeout-minutes: 15
         run: cargo test --workspace --all-targets --all-features --locked
@@ -68,6 +71,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
       - name: Setup guest toolchain
         uses: ./.github/actions/setup-guest-toolchain
         with:
@@ -94,6 +100,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
       - name: Setup guest toolchain
         uses: ./.github/actions/setup-guest-toolchain
         with:
@@ -111,5 +120,35 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y faketime libfaketime
 
+      - name: Build watchdog Lua deps
+        run: |
+          sudo apt-get install -y libcurl4-openssl-dev build-essential pkg-config
+          just watchdog-lua-deps
+
       - name: Run rollups E2E tests
         run: just test-rollups-e2e
+
+      # Runs after the e2e step so the canonical machine image is already built;
+      # exercises the in-process machine_cartesi binding incl. store -> reload -> advance,
+      # which the Rust harness never loads (its compare passes only load the genesis image).
+      - name: Watchdog Lua CM e2e
+        run: just test-watchdog-e2e
+
+  watchdog-docker:
+    name: Watchdog Docker image smoke
+    runs-on: ubuntu-latest
+    needs: rust
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build and smoke-test watchdog image
+        run: bash scripts/ci-watchdog-docker-smoke.sh

.github/workflows/release.yml

@@ -19,15 +19,6 @@ on:
 permissions:
   contents: write
 
-env:
-  RUST_TOOLCHAIN: "1.95.0"
-  XGENEXT2FS_VERSION: v1.5.6
-  XGENEXT2FS_SHA256_AMD64: 996e4e68a638b5dc5967d3410f92ecb8d2f41e32218bbe0f8b4c4474d7eebc59
-  XGENEXT2FS_SHA256_ARM64: e5aca81164b762bbe5447bacef41e4fa9e357fd9c8f44e519c5206227d43144d
-  CARTESI_MACHINE_VERSION: v0.20.0-test2
-  CARTESI_MACHINE_SHA256_AMD64: 39bbfc96a6cc6606307294b719df65f4f2725e8d200d062bcbd8c22355b99b56
-  CARTESI_MACHINE_SHA256_ARM64: 787d823756000cdecd72da8a3494b4c08613087379035959e561bbaef7a220ba
-
 jobs:
   build-sequencer:
     name: Build sequencer (${{ matrix.arch }})
@@ -45,6 +36,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
       - name: Install system dependencies
         run: |
           sudo apt-get update
@@ -85,6 +79,10 @@ jobs:
 
           mkdir -p "package/sequencer-${TAG}-linux-${ARCH}"
           cp "target/${TARGET}/release/sequencer" "package/sequencer-${TAG}-linux-${ARCH}/sequencer"
+          bash scripts/generate-release-manifest.sh \
+            --tag "${TAG}" \
+            --git-sha "${GITHUB_SHA}" \
+            --output "package/sequencer-${TAG}-linux-${ARCH}/RELEASE.json"
 
           cat > "package/sequencer-${TAG}-linux-${ARCH}/RUNNING.md" <<'EOF'
           ## Running
@@ -122,6 +120,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
 
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
       - name: Setup guest toolchain
         uses: ./.github/actions/setup-guest-toolchain
         with:
@@ -159,36 +160,100 @@ jobs:
           name: canonical-machine-images
           path: dist/canonical-machine-image-*.tar.gz
 
+  build-watchdog-image:
+    name: Build watchdog image (${{ matrix.arch }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            platform: linux/amd64
+            deb_sha_env: CARTESI_MACHINE_SHA256_AMD64
+          - arch: arm64
+            platform: linux/arm64
+            deb_sha_env: CARTESI_MACHINE_SHA256_ARM64
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build and export watchdog image
+        env:
+          TAG: ${{ inputs.tag || github.ref_name }}
+          DEB_SHA_ENV: ${{ matrix.deb_sha_env }}
+        run: |
+          set -euo pipefail
+          DEB_SHA="${!DEB_SHA_ENV}"
+          image="sequencer-watchdog:${TAG}"
+          docker build \
+            --platform "${{ matrix.platform }}" \
+            --build-arg "RELEASE_TAG=${TAG}" \
+            --build-arg "GIT_COMMIT=${GITHUB_SHA}" \
+            --build-arg "CARTESI_MACHINE_VERSION=${CARTESI_MACHINE_VERSION}" \
+            --build-arg "CARTESI_MACHINE_DEB_SHA256=${DEB_SHA}" \
+            -f watchdog/Dockerfile \
+            -t "${image}" \
+            .
+          mkdir -p dist
+          docker save "${image}" | gzip -9 > "dist/sequencer-watchdog-${TAG}-linux-${{ matrix.arch }}.tar.gz"
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v6
+        with:
+          name: watchdog-image-linux-${{ matrix.arch }}
+          path: dist/sequencer-watchdog-*.tar.gz
+
   publish:
     name: Publish GitHub Release
     runs-on: ubuntu-latest
     needs:
       - build-sequencer
       - build-canonical-machine-image
+      - build-watchdog-image
     steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Load toolchain pins
+        uses: ./.github/actions/load-toolchain-pins
+
       - name: Download build artifacts
         uses: actions/download-artifact@v6
         with:
           path: dist
 
       - name: Flatten artifacts
+        env:
+          TAG: ${{ inputs.tag || github.ref_name }}
         run: |
           set -euo pipefail
           mkdir -p out
           find dist -type f -name '*.tar.gz' -exec cp -v '{}' out/ \;
+          bash scripts/generate-release-manifest.sh \
+            --tag "${TAG}" \
+            --git-sha "${GITHUB_SHA}" \
+            --output "out/release-manifest-${TAG}.json"
 
       - name: Generate checksums
         working-directory: out
         run: |
           set -euo pipefail
-          sha256sum *.tar.gz > SHA256SUMS
+          sha256sum *.tar.gz *.json > SHA256SUMS
 
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v2
         with:
           tag_name: ${{ inputs.tag || github.ref_name }}
-          prerelease: ${{ inputs.prerelease || true }}
+          prerelease: ${{ github.event_name == 'workflow_dispatch' && inputs.prerelease }}
           fail_on_unmatched_files: true
           files: |
             out/*.tar.gz
+            out/*.json
             out/SHA256SUMS

.gitignore

@@ -1,10 +1,24 @@
 /target
+.deps/
+watchdog-e2e-*/
 .env
 .env.fish
 sequencer.db
 sequencer.db-shm
 sequencer.db-wal
 /out/
+examples/canonical-app/out/
 /.DS_Store
+.vscode/
 soljson-latest.js
 **/states/
+__pycache__/
+/benchmarks/
+/.watchdog-cm-work/
+
+# Local CM / demo scratch (not part of the repo)
+/input-*-output-*.bin
+docs/live-demo.md
+tests/scripts/sepolia-demo.txt
+tests/scripts/sepolia_accounts.txt
+tests/scripts/tests/

AGENTS.md

@@ -343,4 +343,7 @@ Before finishing a change, ensure:
 - [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats.
 - [`docs/recovery/README.md`](docs/recovery/README.md) — recovery design, TLA+ formal verification, design history.
 - [`docs/snapshots/`](docs/snapshots/) — app snapshots: [`format.md`](docs/snapshots/format.md) (dump trait + wire format) and [`lifecycle.md`](docs/snapshots/lifecycle.md) (take/promote/GC/lease design + crash-safety).
+- [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — production-like watchdog (Sepolia / mainnet; internal snapshot API).
+- [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — local dev: watchdog + `sequencer-devnet` on Anvil.
+- [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, compare vs advance modes, test commands.
 - [`sequencer-core/`](sequencer-core/) — shared domain types and protocol contracts.

CLAUDE.md

@@ -56,3 +56,5 @@ Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ en
 - **[`docs/threat-model/README.md`](docs/threat-model/README.md)** — trust boundaries and in-scope threats.
 - **[`docs/recovery/README.md`](docs/recovery/README.md)** — preemptive recovery design + TLA+ proofs.
 - **[`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)** — snapshot lifecycle design + invariants (take/promote/GC, crash-safety). Read before touching the inclusion lane's safe-frontier/snapshot path.
+- **[`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md)** — watchdog on live L1 (Sepolia / mainnet, production-like).
+- **[`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md)** — local dev: watchdog + `sequencer-devnet` on Anvil.

README.md

@@ -189,6 +189,7 @@ released even on client disconnect.
 
 Related docs:
 - App snapshots (format + lifecycle): `docs/snapshots/`
+- Watchdog — local dev: [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md); Sepolia/mainnet: [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md)
 
 ## Prototype Limits
 
@@ -217,6 +218,9 @@ Some tests require [Foundry](https://getfoundry.sh) (`anvil` on PATH). They run
 - [`CLAUDE.md`](CLAUDE.md) — quick reference for shell setup and commands.
 - [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats.
 - [`docs/recovery/README.md`](docs/recovery/README.md) — recovery design, TLA+ formal verification, design history.
+- [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — step-by-step: run the watchdog with a local sequencer.
+- [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — watchdog on live L1 (Sepolia staging, mainnet production).
+- [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, modules, and test commands.
 - [`sequencer-core/`](sequencer-core/) — shared domain types (`Application`, `SignedUserOp`, `Batch`, `Frame`).
 - [`examples/app-core/`](examples/app-core/) — placeholder wallet app implementing the `Application` trait.