diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts index 76c0ed86819f..de2e68d376c7 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts @@ -11,8 +11,7 @@ import { SecretValue } from '@aztec/foundation/config'; import { retryUntil } from '@aztec/foundation/retry'; import { bufferToHex } from '@aztec/foundation/string'; import { timeoutPromise } from '@aztec/foundation/timer'; -import { type L2Block, L2BlockSourceEvents } from '@aztec/stdlib/block'; -import type { L2Tips } from '@aztec/stdlib/block'; +import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block'; import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts index 7259eca70177..f90a4c3f9092 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts @@ -10,8 +10,7 @@ import { SecretValue } from '@aztec/foundation/config'; import { retryUntil } from '@aztec/foundation/retry'; import { bufferToHex } from '@aztec/foundation/string'; import { timeoutPromise } from '@aztec/foundation/timer'; -import { type L2Block, L2BlockSourceEvents } from '@aztec/stdlib/block'; -import type { L2Tips } from '@aztec/stdlib/block'; +import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block'; import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts new file mode 100644 index 000000000000..c9c03a4ce2de --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts @@ -0,0 +1,100 @@ +import type { Logger } from '@aztec/aztec.js/log'; +import { RollupContract } from '@aztec/ethereum/contracts'; +import { ChainMonitor } from '@aztec/ethereum/test'; +import { CheckpointNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; + +import { jest } from '@jest/globals'; + +import type { EndToEndContext } from '../fixtures/utils.js'; +import { EpochsTestContext } from './epochs_test.js'; + +jest.setTimeout(1000 * 60 * 10); + +// Single-sequencer suite for the failed-sync prune fallback (A-1260). The proposer cannot build while +// its sync is paused, so the only way the pending chain can be wound back to proven is the +// `Sequencer.tryVoteAndPruneWhenCannotBuild` path. With no prover node, epoch 0 never proves, so once +// its proof-submission window closes the chain becomes prunable and the proposer's fallback must call +// `prune()` despite being unable to propose. +// +// Timing: ethSlot=8s, aztecSlot=2×8=16s, epoch=8, proofSubmissionEpochs=1. +describe('e2e_epochs/epochs_prune_when_cannot_build', () => { + let context: EndToEndContext; + let logger: Logger; + let rollup: RollupContract; + let monitor: ChainMonitor; + + let L2_SLOT_DURATION_IN_S: number; + + let test: EpochsTestContext; + + beforeEach(async () => { + test = await EpochsTestContext.setup({ + startProverNode: false, // Nothing ever proves epoch 0, so its pending chain stays unproven and becomes prunable. + ethereumSlotDuration: 8, + aztecEpochDuration: 8, // Long enough to land a few checkpoints in epoch 0. + aztecSlotDurationInL1Slots: 2, + aztecProofSubmissionEpochs: 1, // Pending chain becomes prunable one proof window after epoch 0. + minTxsPerBlock: 0, // Solo proposer advances the pending chain on empty checkpoints. + }); + ({ context, logger, rollup, monitor } = test); + ({ L2_SLOT_DURATION_IN_S } = test); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + await test.teardown(); + }); + + it('prunes the pending chain via the fallback path when it cannot propose', async () => { + // Build a few checkpoints in epoch 0 so there is a pending chain to prune. Nothing proves them. + const targetCheckpointNumber = CheckpointNumber(3); + await test.waitUntilCheckpointNumber(targetCheckpointNumber, L2_SLOT_DURATION_IN_S * 12); + + const pendingBeforePause = await rollup.getCheckpointNumber(); + const provenBeforePause = await rollup.getProvenCheckpointNumber(); + logger.info(`Built pending checkpoint ${pendingBeforePause}, proven is ${provenBeforePause}`); + expect(provenBeforePause).toEqual(CheckpointNumber(0)); + expect(pendingBeforePause).toBeGreaterThanOrEqual(targetCheckpointNumber); + + // Pause sync but leave the sequencer running: the proposer's checkSync now fails every slot, so it + // can never propose(). The normal propose-path auto-prune therefore cannot fire — the only way the + // chain can prune is the new tryVoteAndPruneWhenCannotBuild fallback. + logger.info(`Pausing node sync so the proposer can no longer build`); + await context.aztecNodeAdmin.pauseSync(); + + // Let epoch 0's proof submission window expire so canPruneAtTime becomes true. Advance one more slot + // past the deadline so the proposer gets a fresh slot to run its fallback in. + logger.info(`Waiting for the proof submission window of epoch 0 to expire`); + await test.waitUntilLastSlotOfProofSubmissionWindow(0); + const lastBlockTs = BigInt(await context.cheatCodes.eth.lastBlockTimestamp()); + await context.cheatCodes.eth.warp(Number(lastBlockTs) + L2_SLOT_DURATION_IN_S * 2, { resetBlockInterval: true }); + + // The fallback path winds the pending tip back to proven. The monitor reads L1 directly (not the + // paused node's archiver), so it observes the prune even while sync is paused. + logger.info(`Waiting for the pending chain to be pruned back to proven`); + await retryUntil( + async () => (await rollup.getCheckpointNumber()) <= (await rollup.getProvenCheckpointNumber()), + 'pending chain pruned back to proven', + L2_SLOT_DURATION_IN_S * 8, + 0.2, + ); + + const pendingAfterPrune = await rollup.getCheckpointNumber(); + const provenAfterPrune = await rollup.getProvenCheckpointNumber(); + logger.info(`Pruned: pending ${pendingAfterPrune}, proven ${provenAfterPrune}`); + expect(provenAfterPrune).toEqual(CheckpointNumber(0)); + expect(pendingAfterPrune).toEqual(provenAfterPrune); + expect(pendingAfterPrune).toBeLessThan(pendingBeforePause); + + // Sanity: the monitor (driven off L1 on its own poll loop) eventually agrees the pending tip dropped + // to proven. Polled because the monitor refreshes asynchronously and may lag the direct rollup read. + await retryUntil( + () => Promise.resolve(monitor.checkpointNumber <= monitor.provenCheckpointNumber), + 'monitor observes pruned tip', + L2_SLOT_DURATION_IN_S * 2, + 0.2, + ); + expect(monitor.checkpointNumber).toEqual(monitor.provenCheckpointNumber); + }); +}); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts index b8bc704fe5d3..a6857f87c304 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts @@ -54,6 +54,13 @@ describe('compareActions sorting', () => { expect(sorted.indexOf('propose')).toBeLessThan(sorted.indexOf('vote-offenses')); }); + + it('places prune before propose', () => { + const actions: Action[] = ['propose', 'prune']; + const sorted = [...actions].sort(compareActions); + + expect(sorted.indexOf('prune')).toBeLessThan(sorted.indexOf('propose')); + }); }); const mockRollupAddress = EthAddress.random().toString(); @@ -1055,4 +1062,45 @@ describe('SequencerPublisher', () => { expect(governanceProposerContract.hasActiveProposalWithPayload).toHaveBeenCalledTimes(2); }); + + describe('enqueuePruneIfPrunable', () => { + const pruneData = encodeFunctionData({ abi: RollupAbi, functionName: 'prune', args: [] }); + + it('enqueues a prune and bundles it to L1 when the rollup is prunable', async () => { + rollup.canPruneAtTime.mockResolvedValue(true); + + expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(true); + + forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, stats: undefined, multicallData: '0x' }); + await publisher.sendRequests(); + + expect(forwardSpy).toHaveBeenCalledTimes(1); + expect(forwardSpy.mock.calls[0][0]).toEqual([{ to: mockRollupAddress, data: pruneData }]); + }); + + it('does not enqueue a prune when the rollup is not prunable', async () => { + rollup.canPruneAtTime.mockResolvedValue(false); + + expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(false); + + await publisher.sendRequests(); + expect(forwardSpy).not.toHaveBeenCalled(); + }); + + it('does not enqueue a duplicate prune for the same slot', async () => { + rollup.canPruneAtTime.mockResolvedValue(true); + + expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(true); + expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(false); + }); + + it('fails closed (skips prune) when canPruneAtTime rejects', async () => { + rollup.canPruneAtTime.mockRejectedValue(new Error('rpc error')); + + expect(await publisher.enqueuePruneIfPrunable(SlotNumber(2))).toEqual(false); + + await publisher.sendRequests(); + expect(forwardSpy).not.toHaveBeenCalled(); + }); + }); }); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index 7652016b2aec..8770950743f4 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -117,6 +117,7 @@ type L1ProcessArgs = { export const Actions = [ 'invalidate-by-invalid-attestation', 'invalidate-by-insufficient-attestations', + 'prune', 'propose', 'governance-signal', 'vote-offenses', @@ -1060,6 +1061,47 @@ export class SequencerPublisher { ); } + /** + * Enqueues a `prune()` transaction if the rollup is prunable at the given slot's L1 timestamp. + * `prune()` is permissionless and idempotent — if the chain is no longer prunable by send time the + * bundle simulation usually drops the entry; on a node without `eth_simulateV1` the bundle is sent + * as-is and the prune reverts `Rollup__NothingToPrune` inside `aggregate3(allowFailure: true)` + * (a failed action, never a whole-tx revert). Used by the failed-sync fallback so a stuck pending + * chain (e.g. bad data blocking sync) can be wound back to recover. + * @returns true if a prune request was enqueued, false otherwise. + */ + public async enqueuePruneIfPrunable(slotNumber: SlotNumber): Promise { + if (this.lastActions['prune'] === slotNumber) { + this.log.debug(`Skipping duplicate prune for slot ${slotNumber}`, { slotNumber }); + return false; + } + // Use the SAME timestamp the bundle simulator overrides block.timestamp with at send time + // (sequencer-bundle-simulator.ts) so this upfront check and the send-time sim agree. Slot-start + // and last-L1-slot both fall within the same L2 slot (and epoch, which is what `canPruneAtTime` + // derives), so they agree today; matching the simulator keeps it robust if the contract ever uses + // the timestamp more granularly. + const ts = getLastL1SlotTimestampForL2Slot(slotNumber, this.epochCache.getL1Constants()); + const canPrune = await this.rollupContract.canPruneAtTime(ts).catch(err => { + this.log.error(`Failed to check canPruneAtTime for slot ${slotNumber}`, err, { slotNumber }); + return false; + }); + if (!canPrune) { + this.log.debug(`Rollup not prunable at slot ${slotNumber}`, { slotNumber }); + return false; + } + const request: L1TxRequest = { + to: this.rollupContract.address, + data: encodeFunctionData({ abi: RollupAbi, functionName: 'prune', args: [] }), + }; + this.log.info(`Enqueuing rollup prune for slot ${slotNumber}`, { slotNumber }); + return this.enqueueRequest( + 'prune', + request, + { address: this.rollupContract.address, abi: RollupAbi, eventName: 'PrunedPending' }, + slotNumber, + ); + } + /** Enqueues all slashing actions as returned by the slasher client. */ public async enqueueSlashingActions( actions: ProposerSlashAction[], diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 87caadf8b4c8..ffd7a9fc9ea1 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -232,6 +232,7 @@ describe('sequencer', () => { publisher.enqueueProposeCheckpoint.mockResolvedValue(undefined); publisher.enqueueGovernanceCastSignal.mockResolvedValue(true); publisher.enqueueSlashingActions.mockResolvedValue(true); + publisher.enqueuePruneIfPrunable.mockResolvedValue(false); publisher.sendRequestsAt.mockResolvedValue({ result: { receipt: { status: 'success' } as any }, successfulActions: ['propose'], @@ -628,6 +629,7 @@ describe('sequencer', () => { pub.enqueueProposeCheckpoint.mockResolvedValue(undefined); pub.enqueueGovernanceCastSignal.mockResolvedValue(true); pub.enqueueSlashingActions.mockResolvedValue(true); + pub.enqueuePruneIfPrunable.mockResolvedValue(false); pub.sendRequestsAt.mockResolvedValue({ result: { receipt: { status: 'success' } as any }, successfulActions: ['propose'], @@ -825,8 +827,8 @@ describe('sequencer', () => { const mockSlashActions = [{ type: 'vote-offenses' as const, round: 1n, votes: [], committees: [] }]; it('should vote on slashing and governance when sync fails and past the start deadline', async () => { - // Past start_deadline for the target slot: tryVoteWhenCannotBuild should vote instead of waiting to - // build (sync has failed, so building is impossible anyway). + // Past start_deadline for the target slot: tryVoteAndPruneWhenCannotBuild should vote instead of waiting + // to build (sync has failed, so building is impossible anyway). const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber)); dateProvider.setTime((startDeadline + 1) * 1000); @@ -932,6 +934,72 @@ describe('sequencer', () => { expect(publisher.enqueueSlashingActions).not.toHaveBeenCalled(); expect(publisher.sendRequestsAt).not.toHaveBeenCalled(); }); + + it('should prune when prunable even if there are no votes to cast', async () => { + const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber)); + dateProvider.setTime((startDeadline + 1) * 1000); + + // No slashing actions and no governance payload, so all votes are falsy. + slasherClient.getProposerActions.mockResolvedValue([]); + publisher.enqueueSlashingActions.mockResolvedValue(false); + publisher.enqueueGovernanceCastSignal.mockResolvedValue(false); + + // Set us as the proposer + validatorClient.getValidatorAddresses.mockReturnValue([signer.address]); + epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(signer.address); + + // Rollup is prunable, so the fallback should enqueue a prune and still send. + publisher.enqueuePruneIfPrunable.mockResolvedValue(true); + + await sequencer.work(); + + expect(publisher.enqueuePruneIfPrunable).toHaveBeenCalledWith(SlotNumber(newSlotNumber)); + // A send fires even though only prune (and no votes) was enqueued. + expect(publisher.sendRequestsAt).toHaveBeenCalledWith(SlotNumber(newSlotNumber)); + }); + + it('should not send anything when there are no votes and the rollup is not prunable', async () => { + const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber)); + dateProvider.setTime((startDeadline + 1) * 1000); + + // No slashing actions and no governance payload, so all votes are falsy. + slasherClient.getProposerActions.mockResolvedValue([]); + publisher.enqueueSlashingActions.mockResolvedValue(false); + publisher.enqueueGovernanceCastSignal.mockResolvedValue(false); + + // Set us as the proposer + validatorClient.getValidatorAddresses.mockReturnValue([signer.address]); + epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(signer.address); + + // Rollup is not prunable. + publisher.enqueuePruneIfPrunable.mockResolvedValue(false); + + await sequencer.work(); + + expect(publisher.enqueuePruneIfPrunable).toHaveBeenCalledWith(SlotNumber(newSlotNumber)); + expect(publisher.sendRequestsAt).not.toHaveBeenCalled(); + }); + + it('should enqueue prune alongside votes and send a single request', async () => { + const startDeadline = sequencer.getTimeTable().getBuildStartDeadline(SlotNumber(newSlotNumber)); + dateProvider.setTime((startDeadline + 1) * 1000); + + // Both votes and prune succeed. + slasherClient.getProposerActions.mockResolvedValue(mockSlashActions); + publisher.enqueueSlashingActions.mockResolvedValue(true); + publisher.enqueuePruneIfPrunable.mockResolvedValue(true); + + // Set us as the proposer + validatorClient.getValidatorAddresses.mockReturnValue([signer.address]); + epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(signer.address); + + await sequencer.work(); + + expect(publisher.enqueueSlashingActions).toHaveBeenCalled(); + expect(publisher.enqueuePruneIfPrunable).toHaveBeenCalledWith(SlotNumber(newSlotNumber)); + expect(publisher.sendRequestsAt).toHaveBeenCalledTimes(1); + expect(publisher.sendRequestsAt).toHaveBeenCalledWith(SlotNumber(newSlotNumber)); + }); }); describe('consider invalidating checkpoint', () => { diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 0719c5e47830..19451a6f475d 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -88,8 +88,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter ({ [Attributes.SLOT_NUMBER]: slot })) - protected async tryVoteWhenCannotBuild(args: { slot: SlotNumber; targetSlot: SlotNumber }): Promise { + @trackSpan('Sequencer.tryVoteAndPruneWhenCannotBuild', ({ slot }) => ({ [Attributes.SLOT_NUMBER]: slot })) + protected async tryVoteAndPruneWhenCannotBuild(args: { slot: SlotNumber; targetSlot: SlotNumber }): Promise { const { slot, targetSlot } = args; // Prevent duplicate attempts in the same slot - if (this.lastSlotForFallbackVote === slot) { - this.log.trace(`Already attempted to vote in slot ${slot} (skipping)`); + if (this.lastSlotForFallbackAction === slot) { + this.log.trace(`Already attempted fallback actions in slot ${slot} (skipping)`); return; } @@ -976,7 +977,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter !p)) { - this.log.debug(`No votes to enqueue for slot ${slot}`); + // Even if we cannot build, try to prune so a stuck pending chain (e.g. bad data blocking sync) can + // recover. prune() is permissionless, so it rides the same fallback multicall as the votes. + const pruneEnqueued = await this.tryEnqueuePruneIfPrunable(targetSlot, publisher); + + // Bail if nothing to do + if (votes.every(p => !p) && !pruneEnqueued) { + this.log.debug(`Nothing to enqueue for slot ${slot} (no votes, not prunable)`); return; } - this.log.info(`Voting in slot ${slot} despite sync failure`, { slot }); + const [governanceVoteEnqueued, slashingVoteEnqueued] = votes; + this.log.info(`Submitting fallback requests in slot ${slot} despite sync failure`, { + slot, + pruneEnqueued, + governanceVoteEnqueued: !!governanceVoteEnqueued, + slashingVoteEnqueued: !!slashingVoteEnqueued, + }); + // Votes are EIP-712-signed for `targetSlot` (the pipelined slot in which the multicall is // expected to mine). Delay submission to the start of `targetSlot` so the tx mines in the // slot the votes were signed for. We fire-and-forget so we don't block the sequencer's // work loop while waiting for the target slot to start. void publisher.sendRequestsAt(targetSlot).catch(err => { - this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot }); + this.log.error(`Failed to publish fallback requests despite sync failure for slot ${slot}`, err, { slot }); }); } + private async tryEnqueuePruneIfPrunable(targetSlot: SlotNumber, publisher: SequencerPublisher): Promise { + try { + return await publisher.enqueuePruneIfPrunable(targetSlot); + } catch (err) { + this.log.error(`Failed to enqueue rollup prune for slot ${targetSlot}`, err, { targetSlot }); + return false; + } + } + /** * Tries to vote on slashing actions and governance proposals when escape hatch is open. * This allows the sequencer to participate in voting without performing checkpoint proposal work. @@ -1029,13 +1051,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter { this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot }); });