Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"changes": [
{
"comment": "Add `--with-block-access-lists` and `--verify-block-access-list-hash`; Amsterdam blocks pass block-hash verification",
"type": "minor",
"packageName": "@subsquid/evm-dump"
}
],
"packageName": "@subsquid/evm-dump",
"email": "abernatskiy@toha.li"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"changes": [
{
"comment": "Carry the EIP-7928 block access list in raw blocks as `blockAccessList_`",
"type": "minor",
"packageName": "@subsquid/evm-normalization"
}
],
"packageName": "@subsquid/evm-normalization",
"email": "abernatskiy@toha.li"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"changes": [
{
"comment": "Verify Amsterdam headers (blockAccessListHash, slotNumber) and optionally fetch and verify EIP-7928 block access lists; a block that stays invalid on the finalized polling path now fails the stream instead of stalling it",
"type": "minor",
"packageName": "@subsquid/evm-rpc"
}
],
"packageName": "@subsquid/evm-rpc",
"email": "abernatskiy@toha.li"
}
6 changes: 6 additions & 0 deletions evm/evm-dump/src/dumper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ interface Options extends DumperOptions {
withReceipts?: boolean
withTraces?: boolean
withStatediffs?: boolean
withBlockAccessLists?: boolean
useTraceApi?: boolean
useDebugApiForStatediffs?: boolean
useDebugTraceBlockByNumber?: boolean
Expand All @@ -34,6 +35,7 @@ interface Options extends DumperOptions {
verifyReceiptsRoot?: boolean
verifyWithdrawalsRoot?: boolean
verifyLogsBloom?: boolean
verifyBlockAccessListHash?: boolean
callFrameValidation?: CallFrameValidationMode
skipLogIndexCheck?: boolean
skipCumulativeGasUsedCheck?: boolean
Expand All @@ -55,6 +57,7 @@ export class EvmDumper extends Dumper<RawBlock, Options> {
program.option('--with-receipts', 'Fetch transaction receipt data')
program.option('--with-traces', 'Fetch EVM call traces')
program.option('--with-statediffs', 'Fetch EVM state updates')
program.option('--with-block-access-lists', 'Fetch EIP-7928 block access lists of blocks that commit to one')
program.option('--use-trace-api', 'Use trace_* API for statediffs and call traces')
program.option('--use-debug-api-for-statediffs', 'Use debug prestateTracer to fetch statediffs (by default will use trace_* api)')
program.option('--use-debug-trace-block-by-number', 'Use debug_traceBlockByNumber instead of debug_traceBlockByHash')
Expand All @@ -65,6 +68,7 @@ export class EvmDumper extends Dumper<RawBlock, Options> {
program.option('--verify-receipts-root', 'Verify block receipts against receipts root')
program.option('--verify-withdrawals-root', 'Verify block withdrawals against withdrawals root')
program.option('--verify-logs-bloom', 'Verify block logs against logs bloom')
program.option('--verify-block-access-list-hash', 'Verify block access lists against the blockAccessListHash header commitment')
program.addOption(
new Option(
'--call-frame-validation <mode>',
Expand Down Expand Up @@ -118,6 +122,7 @@ export class EvmDumper extends Dumper<RawBlock, Options> {
verifyReceiptsRoot: this.options().verifyReceiptsRoot,
verifyWithdrawalsRoot: this.options().verifyWithdrawalsRoot,
verifyLogsBloom: this.options().verifyLogsBloom,
verifyBlockAccessListHash: this.options().verifyBlockAccessListHash,
callFrameValidation: this.options().callFrameValidation,
checkLogIndex: !this.options().skipLogIndexCheck,
checkCumulativeGasUsed: !this.options().skipCumulativeGasUsedCheck,
Expand All @@ -130,6 +135,7 @@ export class EvmDumper extends Dumper<RawBlock, Options> {
receipts: this.options().withReceipts,
traces: this.options().withTraces,
stateDiffs: this.options().withStatediffs,
blockAccessLists: this.options().withBlockAccessLists,
useDebugApiForStateDiffs: this.options().useDebugApiForStatediffs,
useDebugTraceBlockByNumber: this.options().useDebugTraceBlockByNumber,
useTraceApi: this.options().useTraceApi,
Expand Down
8 changes: 7 additions & 1 deletion evm/evm-normalization/src/raw.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import {
TraceTransactionReplay,
Block,
DebugFrameResult,
DebugStateDiffResult
DebugStateDiffResult,
BlockAccessList
} from '@subsquid/evm-rpc'
import {assertNotNull} from '@subsquid/util-internal'
import assert from 'assert'
Expand All @@ -24,6 +25,7 @@ export type RawBlock = GetBlock & {
transactions: RawTransaction[]
logs_?: Log[]
unknownTraceReplays_?: TraceTransactionReplay[]
blockAccessList_?: BlockAccessList
}


Expand All @@ -41,6 +43,10 @@ export function toRawBlock(block: Block): RawBlock {
rawBlock.logs_ = block.logs
}

if (block.blockAccessList) {
rawBlock.blockAccessList_ = block.blockAccessList
}

if (block.receipts) {
let byTx = new Map(block.receipts.map(receipt => [receipt.transactionHash, receipt]))
for (let i = 0; i < transactions.length; i++) {
Expand Down
7 changes: 6 additions & 1 deletion evm/evm-rpc/src/chain-utils.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import {assertNotNull} from '@subsquid/util-internal'
import {GetBlock, Log, Receipt, Transaction, Withdrawal} from './rpc-data'
import {BlockAccessList, GetBlock, Log, Receipt, Transaction, Withdrawal} from './rpc-data'
import {Qty} from './types'
import {
avalancheBlockHash,
blockAccessListHash,
blockHash,
extDataHash,
tempoBlockHash,
Expand Down Expand Up @@ -166,6 +167,10 @@ export class ChainUtils {
return withdrawalsRoot(withdrawals)
}

calculateBlockAccessListHash(accessList: BlockAccessList) {
return blockAccessListHash(accessList)
}

recoverTxSender(transaction: Transaction) {
if (this.isHyperliquidMainnet || this.isHyperliquidTestnet) {
if (isHyperliquidSystemTx(transaction)) return
Expand Down
88 changes: 88 additions & 0 deletions evm/evm-rpc/src/data-source/poll-stream.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import {describe, it, expect, vi} from 'vitest'
import {PollStream} from './poll-stream'
import type {Block} from '../types'
import type {Commitment, Rpc} from '../rpc'


function mkBlock(number: number, invalid?: string): Block {
let block: Block = {
number,
hash: `0x${number.toString(16).padStart(64, '0')}`,
block: {} as Block['block'],
}
if (invalid) {
block._isInvalid = true
block._errorMessage = invalid
}
return block
}


// A chain at head 104 whose block 102 is invalid for the first `invalidPolls` fetches.
function mkRpc(invalidPolls: number) {
let polls = 0
return {
endpoint: 'http://rpc.test',
getLatestBlockhash: vi.fn(async () => ({number: 104, hash: '0x'})),
getBlockBatch: vi.fn(async (numbers: number[]) => {
let blocks = numbers.map(n => {
if (n == 102 && polls < invalidPolls) return mkBlock(n, 'block access list does not match blockAccessListHash')
return mkBlock(n)
})
if (numbers.includes(102)) polls += 1
return blocks
})
} as unknown as Rpc
}


function mkStream(rpc: Rpc, commitment: Commitment) {
return new PollStream({rpc, commitment, from: 100, strideSize: 10})
}


describe('PollStream', () => {
it('stops a batch before an invalid block and polls it again', async () => {
const stream = mkStream(mkRpc(1), 'finalized')

expect((await stream.next()).map(b => b.number)).toEqual([100, 101])
expect(stream.position()).toEqual(102)
expect((await stream.next()).map(b => b.number)).toEqual([102, 103, 104])
})

it('fails the finalized stream when a block stays invalid', async () => {
const stream = mkStream(mkRpc(Number.POSITIVE_INFINITY), 'finalized')
await stream.next()

let err: any
for (let i = 0; i < 10 && err == null; i++) {
await stream.next().catch(e => {
err = e
})
}

expect(err?.message).toMatch(/blockAccessListHash/)
expect(err.failedBlocks).toEqual([102])
expect(err.rpcUrl).toEqual('http://rpc.test')
})

it('recovers when a finalized block turns valid within the retry budget', async () => {
const stream = mkStream(mkRpc(5), 'finalized')

for (let i = 0; i < 5; i++) {
await stream.next()
}
expect(stream.position()).toEqual(102)
expect((await stream.next()).map(b => b.number)).toEqual([102, 103, 104])
})

it('keeps polling an invalid block at the head of the latest stream', async () => {
const stream = mkStream(mkRpc(20), 'latest')

for (let i = 0; i < 20; i++) {
await stream.next()
}
expect(stream.position()).toEqual(102)
expect((await stream.next()).map(b => b.number)).toEqual([102, 103, 104])
})
})
47 changes: 47 additions & 0 deletions evm/evm-rpc/src/data-source/poll-stream.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
import {createLogger, Logger} from '@subsquid/logger'
import {addErrorContext, last} from '@subsquid/util-internal'
import {Commitment, Rpc} from '../rpc'
import {Block, DataRequest} from '../types'


// Same budget as getBlocks() gives an invalid block on the finalized path.
const MAX_INVALID_RETRIES = 5


export interface PollStreamOptions {
rpc: Rpc
commitment: Commitment
Expand All @@ -19,13 +25,17 @@ export class PollStream {
private head?: number
private lastRead?: number
private from: number
private invalidBlock?: number
private invalidRetries = 0
private log: Logger

constructor(options: PollStreamOptions) {
this.rpc = options.rpc
this.commitment = options.commitment
this.req = options.req ?? {}
this.strideSize = options.strideSize || 5
this.from = options.from
this.log = createLogger('sqd:evm-rpc')
}

position(): number {
Expand All @@ -36,6 +46,8 @@ export class PollStream {
this.head = undefined
this.lastRead = undefined
this.from = pos
this.invalidBlock = undefined
this.invalidRetries = 0
}

async next(): Promise<Block[]> {
Expand Down Expand Up @@ -69,14 +81,49 @@ export class PollStream {

for (let i = 0; i < blocks.length; i++) {
if (blocks[i]._isInvalid) {
this.onInvalidBlock(blocks[i])
blocks = blocks.slice(0, i)
break
}
}

if (blocks.length > 0) {
if (this.invalidBlock != null && last(blocks).number >= this.invalidBlock) {
this.invalidBlock = undefined
this.invalidRetries = 0
}
this.lastRead = blocks[blocks.length - 1].number
}
return blocks
}

/**
* An invalid block is polled again on the next call. On the finalized path
* the data should not change any more, so a block that stays invalid fails
* the stream like it does in getBlocks(), instead of stalling it. At the
* head of the chain a provider may still be catching up: keep waiting.
*/
private onInvalidBlock(block: Block): void {
if (block.number === this.invalidBlock) {
this.invalidRetries += 1
} else {
this.invalidBlock = block.number
this.invalidRetries = 0
}

if (this.commitment == 'finalized' && this.invalidRetries >= MAX_INVALID_RETRIES) {
throw addErrorContext(new Error(block._errorMessage ?? 'invalid block'), {
rpcUrl: this.rpc.endpoint,
failedBlocks: [block.number],
retries: this.invalidRetries
})
}

this.log.warn({
blockNumber: block.number,
blockHash: block.hash,
reason: block._errorMessage,
retries: this.invalidRetries
}, 'invalid block, will poll it again')
}
}
54 changes: 54 additions & 0 deletions evm/evm-rpc/src/rpc-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,56 @@ export const DebugFrameResult = object({
export type DebugFrameResult = GetSrcType<typeof DebugFrameResult>


// Block access list (EIP-7928) as returned by `eth_getBlockAccessListByBlockHash`.
// `index` is the block access index: 0 for pre-execution system calls,
// 1..n for transactions, n+1 for post-execution system operations.
// https://eips.ethereum.org/EIPS/eip-7928
export const BlockAccessListChange = object({
index: SMALL_QTY,
value: QTY
})


export type BlockAccessListChange = GetSrcType<typeof BlockAccessListChange>


export const BlockAccessListCodeChange = object({
index: SMALL_QTY,
code: BYTES
})


export type BlockAccessListCodeChange = GetSrcType<typeof BlockAccessListCodeChange>


export const BlockAccessListSlotChanges = object({
key: QTY,
changes: array(BlockAccessListChange)
})


export type BlockAccessListSlotChanges = GetSrcType<typeof BlockAccessListSlotChanges>


export const BlockAccessListAccount = object({
address: BYTES,
storageChanges: array(BlockAccessListSlotChanges),
storageReads: array(QTY),
balanceChanges: array(BlockAccessListChange),
nonceChanges: array(BlockAccessListChange),
codeChanges: array(BlockAccessListCodeChange)
})


export type BlockAccessListAccount = GetSrcType<typeof BlockAccessListAccount>


export const BlockAccessList = array(BlockAccessListAccount)


export type BlockAccessList = GetSrcType<typeof BlockAccessList>


export const GetBlock = object({
number: SMALL_QTY,
hash: BYTES,
Expand Down Expand Up @@ -651,6 +701,10 @@ export const GetBlock = object({
withdrawals: option(array(Withdrawal)),
withdrawalsRoot: option(BYTES),
requestsHash: option(BYTES),
// https://eips.ethereum.org/EIPS/eip-7928
blockAccessListHash: option(BYTES),
// https://eips.ethereum.org/EIPS/eip-7843
slotNumber: option(QTY),
l1BlockNumber: option(SMALL_QTY),
// Tempo-specific block header fields
// https://github.com/tempoxyz/tempo/blob/main/crates/primitives/src/header.rs
Expand Down
Loading
Loading