fix(sequencer): prune in failed-sync fallback so the chain can recover

yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts

@@ -11,8 +11,7 @@ import { SecretValue } from '@aztec/foundation/config';
 import { retryUntil } from '@aztec/foundation/retry';
 import { bufferToHex } from '@aztec/foundation/string';
 import { timeoutPromise } from '@aztec/foundation/timer';
-import { type L2Block, L2BlockSourceEvents } from '@aztec/stdlib/block';
-import type { L2Tips } from '@aztec/stdlib/block';
+import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block';
 import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 
 import { jest } from '@jest/globals';

yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts

@@ -10,8 +10,7 @@ import { SecretValue } from '@aztec/foundation/config';
 import { retryUntil } from '@aztec/foundation/retry';
 import { bufferToHex } from '@aztec/foundation/string';
 import { timeoutPromise } from '@aztec/foundation/timer';
-import { type L2Block, L2BlockSourceEvents } from '@aztec/stdlib/block';
-import type { L2Tips } from '@aztec/stdlib/block';
+import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block';
 import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
 
 import { jest } from '@jest/globals';

yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts

@@ -0,0 +1,100 @@
+import type { Logger } from '@aztec/aztec.js/log';
+import { RollupContract } from '@aztec/ethereum/contracts';
+import { ChainMonitor } from '@aztec/ethereum/test';
+import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import { retryUntil } from '@aztec/foundation/retry';
+
+import { jest } from '@jest/globals';
+
+import type { EndToEndContext } from '../fixtures/utils.js';
+import { EpochsTestContext } from './epochs_test.js';
+
+jest.setTimeout(1000 * 60 * 10);
+
+// Single-sequencer suite for the failed-sync prune fallback (A-1260). The proposer cannot build while
+// its sync is paused, so the only way the pending chain can be wound back to proven is the
+// `Sequencer.tryVoteAndPruneWhenCannotBuild` path. With no prover node, epoch 0 never proves, so once
+// its proof-submission window closes the chain becomes prunable and the proposer's fallback must call
+// `prune()` despite being unable to propose.
+//
+// Timing: ethSlot=8s, aztecSlot=2×8=16s, epoch=8, proofSubmissionEpochs=1.
+describe('e2e_epochs/epochs_prune_when_cannot_build', () => {
+  let context: EndToEndContext;
+  let logger: Logger;
+  let rollup: RollupContract;
+  let monitor: ChainMonitor;
+
+  let L2_SLOT_DURATION_IN_S: number;
+
+  let test: EpochsTestContext;
+
+  beforeEach(async () => {
+    test = await EpochsTestContext.setup({
+      startProverNode: false, // Nothing ever proves epoch 0, so its pending chain stays unproven and becomes prunable.
+      ethereumSlotDuration: 8,
+      aztecEpochDuration: 8, // Long enough to land a few checkpoints in epoch 0.
+      aztecSlotDurationInL1Slots: 2,
+      aztecProofSubmissionEpochs: 1, // Pending chain becomes prunable one proof window after epoch 0.
+      minTxsPerBlock: 0, // Solo proposer advances the pending chain on empty checkpoints.
+    });
+    ({ context, logger, rollup, monitor } = test);
+    ({ L2_SLOT_DURATION_IN_S } = test);
+  });
+
+  afterEach(async () => {
+    jest.restoreAllMocks();
+    await test.teardown();
+  });
+
+  it('prunes the pending chain via the fallback path when it cannot propose', async () => {
+    // Build a few checkpoints in epoch 0 so there is a pending chain to prune. Nothing proves them.
+    const targetCheckpointNumber = CheckpointNumber(3);
+    await test.waitUntilCheckpointNumber(targetCheckpointNumber, L2_SLOT_DURATION_IN_S * 12);
+
+    const pendingBeforePause = await rollup.getCheckpointNumber();
+    const provenBeforePause = await rollup.getProvenCheckpointNumber();
+    logger.info(`Built pending checkpoint ${pendingBeforePause}, proven is ${provenBeforePause}`);
+    expect(provenBeforePause).toEqual(CheckpointNumber(0));
+    expect(pendingBeforePause).toBeGreaterThanOrEqual(targetCheckpointNumber);
+
+    // Pause sync but leave the sequencer running: the proposer's checkSync now fails every slot, so it
+    // can never propose(). The normal propose-path auto-prune therefore cannot fire — the only way the
+    // chain can prune is the new tryVoteAndPruneWhenCannotBuild fallback.
+    logger.info(`Pausing node sync so the proposer can no longer build`);
+    await context.aztecNodeAdmin.pauseSync();
+
+    // Let epoch 0's proof submission window expire so canPruneAtTime becomes true. Advance one more slot
+    // past the deadline so the proposer gets a fresh slot to run its fallback in.
+    logger.info(`Waiting for the proof submission window of epoch 0 to expire`);
+    await test.waitUntilLastSlotOfProofSubmissionWindow(0);
+    const lastBlockTs = BigInt(await context.cheatCodes.eth.lastBlockTimestamp());
+    await context.cheatCodes.eth.warp(Number(lastBlockTs) + L2_SLOT_DURATION_IN_S * 2, { resetBlockInterval: true });
+
+    // The fallback path winds the pending tip back to proven. The monitor reads L1 directly (not the
+    // paused node's archiver), so it observes the prune even while sync is paused.
+    logger.info(`Waiting for the pending chain to be pruned back to proven`);
+    await retryUntil(
+      async () => (await rollup.getCheckpointNumber()) <= (await rollup.getProvenCheckpointNumber()),
+      'pending chain pruned back to proven',
+      L2_SLOT_DURATION_IN_S * 8,
+      0.2,
+    );
+
+    const pendingAfterPrune = await rollup.getCheckpointNumber();
+    const provenAfterPrune = await rollup.getProvenCheckpointNumber();
+    logger.info(`Pruned: pending ${pendingAfterPrune}, proven ${provenAfterPrune}`);
+    expect(provenAfterPrune).toEqual(CheckpointNumber(0));
+    expect(pendingAfterPrune).toEqual(provenAfterPrune);
+    expect(pendingAfterPrune).toBeLessThan(pendingBeforePause);
+
+    // Sanity: the monitor (driven off L1 on its own poll loop) eventually agrees the pending tip dropped
+    // to proven. Polled because the monitor refreshes asynchronously and may lag the direct rollup read.
+    await retryUntil(
+      () => Promise.resolve(monitor.checkpointNumber <= monitor.provenCheckpointNumber),
+      'monitor observes pruned tip',
+      L2_SLOT_DURATION_IN_S * 2,
+      0.2,
+    );
+    expect(monitor.checkpointNumber).toEqual(monitor.provenCheckpointNumber);
+  });
+});

yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts

@@ -54,6 +54,13 @@ describe('compareActions sorting', () => {
 
     expect(sorted.indexOf('propose')).toBeLessThan(sorted.indexOf('vote-offenses'));
   });
+
+  it('places prune before propose', () => {
+    const actions: Action[] = ['propose', 'prune'];
+    const sorted = [...actions].sort(compareActions);
+
+    expect(sorted.indexOf('prune')).toBeLessThan(sorted.indexOf('propose'));
+  });
 });
 
 const mockRollupAddress = EthAddress.random().toString();
@@ -1055,4 +1062,45 @@ describe('SequencerPublisher', () => {
 
     expect(governanceProposerContract.hasActiveProposalWithPayload).toHaveBeenCalledTimes(2);
   });
+
+  describe('enqueuePruneIfPrunable', () => {
+    const pruneData = encodeFunctionData({ abi: RollupAbi, functionName: 'prune', args: [] });
+
+    it('enqueues a prune and bundles it to L1 when the rollup is prunable', async () => {
+      rollup.canPruneAtTime.mockResolvedValue(true);
+
+      expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(true);
+
+      forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, stats: undefined, multicallData: '0x' });
+      await publisher.sendRequests();
+
+      expect(forwardSpy).toHaveBeenCalledTimes(1);
+      expect(forwardSpy.mock.calls[0][0]).toEqual([{ to: mockRollupAddress, data: pruneData }]);
+    });
+
+    it('does not enqueue a prune when the rollup is not prunable', async () => {
+      rollup.canPruneAtTime.mockResolvedValue(false);
+
+      expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(false);
+
+      await publisher.sendRequests();
+      expect(forwardSpy).not.toHaveBeenCalled();
+    });
+
+    it('does not enqueue a duplicate prune for the same slot', async () => {
+      rollup.canPruneAtTime.mockResolvedValue(true);
+
+      expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(true);
+      expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(false);
+    });
+
+    it('fails closed (skips prune) when canPruneAtTime rejects', async () => {
+      rollup.canPruneAtTime.mockRejectedValue(new Error('rpc error'));
+
+      expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(false);
+
+      await publisher.sendRequests();
+      expect(forwardSpy).not.toHaveBeenCalled();
+    });
+  });
 });

yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts

@@ -117,6 +117,7 @@ type L1ProcessArgs = {
 export const Actions = [
   'invalidate-by-invalid-attestation',
   'invalidate-by-insufficient-attestations',
+  'prune',
   'propose',
   'governance-signal',
   'vote-offenses',
@@ -1060,6 +1061,47 @@ export class SequencerPublisher {
     );
   }
 
+  /**
+   * Enqueues a `prune()` transaction if the rollup is prunable at the given slot's L1 timestamp.
+   * `prune()` is permissionless and idempotent — if the chain is no longer prunable by send time the
+   * bundle simulation usually drops the entry; on a node without `eth_simulateV1` the bundle is sent
+   * as-is and the prune reverts `Rollup__NothingToPrune` inside `aggregate3(allowFailure: true)`
+   * (a failed action, never a whole-tx revert). Used by the failed-sync fallback so a stuck pending
+   * chain (e.g. bad data blocking sync) can be wound back to recover.
+   * @returns true if a prune request was enqueued, false otherwise.
+   */
+  public async enqueuePruneIfPrunable(slotNumber: SlotNumber): Promise<boolean> {
+    if (this.lastActions['prune'] === slotNumber) {
+      this.log.debug(`Skipping duplicate prune for slot ${slotNumber}`, { slotNumber });
+      return false;
+    }
+    // Use the SAME timestamp the bundle simulator overrides block.timestamp with at send time
+    // (sequencer-bundle-simulator.ts) so this upfront check and the send-time sim agree. Slot-start
+    // and last-L1-slot both fall within the same L2 slot (and epoch, which is what `canPruneAtTime`
+    // derives), so they agree today; matching the simulator keeps it robust if the contract ever uses
+    // the timestamp more granularly.
+    const ts = getLastL1SlotTimestampForL2Slot(slotNumber, this.epochCache.getL1Constants());
+    const canPrune = await this.rollupContract.canPruneAtTime(ts).catch(err => {
+      this.log.error(`Failed to check canPruneAtTime for slot ${slotNumber}`, err, { slotNumber });
+      return false;
+    });
+    if (!canPrune) {
+      this.log.debug(`Rollup not prunable at slot ${slotNumber}`, { slotNumber });
+      return false;
+    }
+    const request: L1TxRequest = {
+      to: this.rollupContract.address,
+      data: encodeFunctionData({ abi: RollupAbi, functionName: 'prune', args: [] }),
+    };
+    this.log.info(`Enqueuing rollup prune for slot ${slotNumber}`, { slotNumber });
+    return this.enqueueRequest(
+      'prune',
+      request,
+      { address: this.rollupContract.address, abi: RollupAbi, eventName: 'PrunedPending' },
+      slotNumber,
+    );
+  }
+
   /** Enqueues all slashing actions as returned by the slasher client. */
   public async enqueueSlashingActions(
     actions: ProposerSlashAction[],

yarn-project/sequencer-client/src/sequencer/sequencer.test.ts

@@ -232,6 +232,7 @@ describe('sequencer', () => {
     publisher.enqueueProposeCheckpoint.mockResolvedValue(undefined);
     publisher.enqueueGovernanceCastSignal.mockResolvedValue(true);
     publisher.enqueueSlashingActions.mockResolvedValue(true);
+    publisher.enqueuePruneIfPrunable.mockResolvedValue(false);
     publisher.sendRequestsAt.mockResolvedValue({
       result: { receipt: { status: 'success' } as any },
       successfulActions: ['propose'],
@@ -628,6 +629,7 @@ describe('sequencer', () => {
         pub.enqueueProposeCheckpoint.mockResolvedValue(undefined);
         pub.enqueueGovernanceCastSignal.mockResolvedValue(true);
         pub.enqueueSlashingActions.mockResolvedValue(true);
+        pub.enqueuePruneIfPrunable.mockResolvedValue(false);
         pub.sendRequestsAt.mockResolvedValue({
           result: { receipt: { status: 'success' } as any },
           successfulActions: ['propose'],
@@ -825,8 +827,8 @@ describe('sequencer', () => {
     const mockSlashActions = [{ type: 'vote-offenses' as const, round: 1n, votes: [], committees: [] }];
 
     it('should vote on slashing and governance when sync fails and past the start deadline', async () => {
-      // Past start_deadline for the target slot: tryVoteWhenCannotBuild should vote instead of waiting to
-      // build (sync has failed, so building is impossible anyway).
+      // Past start_deadline for the target slot: tryVoteAndPruneWhenCannotBuild should vote instead of waiting
+      // to build (sync has failed, so building is impossible anyway).
       const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber));
       dateProvider.setTime((startDeadline + 1) * 1000);
 
@@ -932,6 +934,72 @@ describe('sequencer', () => {
       expect(publisher.enqueueSlashingActions).not.toHaveBeenCalled();
       expect(publisher.sendRequestsAt).not.toHaveBeenCalled();
     });
+
+    it('should prune when prunable even if there are no votes to cast', async () => {
+      const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber));
+      dateProvider.setTime((startDeadline + 1) * 1000);
+
+      // No slashing actions and no governance payload, so all votes are falsy.
+      slasherClient.getProposerActions.mockResolvedValue([]);
+      publisher.enqueueSlashingActions.mockResolvedValue(false);
+      publisher.enqueueGovernanceCastSignal.mockResolvedValue(false);
+
+      // Set us as the proposer
+      validatorClient.getValidatorAddresses.mockReturnValue([signer.address]);
+      epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(signer.address);
+
+      // Rollup is prunable, so the fallback should enqueue a prune and still send.
+      publisher.enqueuePruneIfPrunable.mockResolvedValue(true);
+
+      await sequencer.work();
+
+      expect(publisher.enqueuePruneIfPrunable).toHaveBeenCalledWith(SlotNumber(newSlotNumber));
+      // A send fires even though only prune (and no votes) was enqueued.
+      expect(publisher.sendRequestsAt).toHaveBeenCalledWith(SlotNumber(newSlotNumber));
+    });
+
+    it('should not send anything when there are no votes and the rollup is not prunable', async () => {
+      const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber));
+      dateProvider.setTime((startDeadline + 1) * 1000);
+
+      // No slashing actions and no governance payload, so all votes are falsy.
+      slasherClient.getProposerActions.mockResolvedValue([]);
+      publisher.enqueueSlashingActions.mockResolvedValue(false);
+      publisher.enqueueGovernanceCastSignal.mockResolvedValue(false);
+
+      // Set us as the proposer
+      validatorClient.getValidatorAddresses.mockReturnValue([signer.address]);
+      epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(signer.address);
+
+      // Rollup is not prunable.
+      publisher.enqueuePruneIfPrunable.mockResolvedValue(false);
+
+      await sequencer.work();
+
+      expect(publisher.enqueuePruneIfPrunable).toHaveBeenCalledWith(SlotNumber(newSlotNumber));
+      expect(publisher.sendRequestsAt).not.toHaveBeenCalled();
+    });
+
+    it('should enqueue prune alongside votes and send a single request', async () => {
+      const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber));
+      dateProvider.setTime((startDeadline + 1) * 1000);
+
+      // Both votes and prune succeed.
+      slasherClient.getProposerActions.mockResolvedValue(mockSlashActions);
+      publisher.enqueueSlashingActions.mockResolvedValue(true);
+      publisher.enqueuePruneIfPrunable.mockResolvedValue(true);
+
+      // Set us as the proposer
+      validatorClient.getValidatorAddresses.mockReturnValue([signer.address]);
+      epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(signer.address);
+
+      await sequencer.work();
+
+      expect(publisher.enqueueSlashingActions).toHaveBeenCalled();
+      expect(publisher.enqueuePruneIfPrunable).toHaveBeenCalledWith(SlotNumber(newSlotNumber));
+      expect(publisher.sendRequestsAt).toHaveBeenCalledTimes(1);
+      expect(publisher.sendRequestsAt).toHaveBeenCalledWith(SlotNumber(newSlotNumber));
+    });
   });
 
   describe('consider invalidating checkpoint', () => {