From 1e55d8b988c50a97f4df3b3a75045e406ab1751d Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Fri, 23 Oct 2026 18:04:08 +0300 Subject: [PATCH 1/2] feat(query): sets, filters on value sets too long for a list A client that filters on hundreds of thousands of addresses cannot send them as a list. It declares a fuse8 structure over them once, under `sets`, and lists a reference to it among a filter's values. The engine only looks values up, by the text the response renders, and refuses a set its canaries do not vouch for: a set built from other bytes matches almost nothing, which reads as a quiet stretch of chain (ADR-17). A set costs a few dozen bytes to declare again under another name, and each reference is a lookup for every row its table scans, so the references a request makes are capped. A catalog that names a table or an alias after a reserved key, `sets` now among them, is refused, since nothing could address it. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 11 + Cargo.toml | 3 + crates/metadata/src/loader.rs | 55 +- crates/metadata/src/types.rs | 13 + spec/01-data-model.md | 6 +- spec/02-request.md | 131 ++++- spec/04-evaluation.md | 69 +++ spec/06-errors.md | 7 +- spec/07-invariants.md | 70 ++- spec/08-conformance.md | 32 +- spec/09-parameters.md | 25 +- spec/GAPS.md | 3 + ...alue-sets-travel-as-fuse8-with-canaries.md | 111 ++++ src/error.rs | 17 +- src/lib.rs | 1 + src/output/encoder.rs | 7 +- src/query/parse.rs | 21 +- src/query/plan.rs | 302 ++++++++++- src/scan/predicate.rs | 384 ++++++++++++++ src/sets.rs | 396 +++++++++++++++ tests/conformance/ct2_request/bounds.rs | 76 +++ tests/conformance/ct2_request/mod.rs | 1 + tests/conformance/ct2_request/sets.rs | 476 ++++++++++++++++++ tests/conformance/ct3_filter_algebra/mod.rs | 1 + tests/conformance/ct3_filter_algebra/sets.rs | 328 ++++++++++++ tests/conformance/harness/fixtures.rs | 20 + tests/conformance/harness/mod.rs | 1 + tests/conformance/harness/sets.rs | 184 +++++++ 28 files changed, 2671 insertions(+), 80 deletions(-) create mode 100644 spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md create mode 100644 src/sets.rs create mode 100644 tests/conformance/ct2_request/sets.rs create mode 100644 tests/conformance/ct3_filter_algebra/sets.rs create mode 100644 tests/conformance/harness/sets.rs diff --git a/Cargo.lock b/Cargo.lock index 3cc4145..551aa83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2890,6 +2890,7 @@ version = "0.1.0" dependencies = [ "anyhow", "arrow", + "base64", "bytes", "criterion", "faster-hex", @@ -2911,6 +2912,7 @@ dependencies = [ "sqd-query", "tempfile", "tikv-jemallocator", + "xorf", "xxhash-rust", "zstd", ] @@ -3702,6 +3704,15 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "xorf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f13846c9009c0e550697ddde851e934b5dfafbce757a5ef5edfdb27ea20803e" +dependencies = [ + "libm", +] + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 3b43fa6..1bee2b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ faster-hex = "0.10" xxhash-rust = { version = "0.8.15", features = ["xxh3"] } ryu = "1.0.23" rustc-hash = "2" +base64 = "0.22" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.6" @@ -64,6 +65,8 @@ libtest-mimic = "0.8" gungraun = "=0.20.0" # Queries per second under concurrency, against saved baselines. criterion = "0.5" +# Builds the sets tests send; the engine only looks them up. +xorf = { version = "0.13", default-features = false, features = ["binary-fuse"] } [[bench]] name = "throughput" diff --git a/crates/metadata/src/loader.rs b/crates/metadata/src/loader.rs index 468493d..f7b1502 100644 --- a/crates/metadata/src/loader.rs +++ b/crates/metadata/src/loader.rs @@ -1,4 +1,7 @@ -use crate::{DatasetDescription, SpecialFilter, MAX_DISCRIMINATOR_BYTES, SCHEMA_VERSION}; +use crate::{ + DatasetDescription, SpecialFilter, MAX_DISCRIMINATOR_BYTES, RESERVED_REQUEST_KEYS, + SCHEMA_VERSION, +}; use anyhow::{Context, Result}; use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}; use std::collections::HashMap; @@ -663,8 +666,14 @@ fn check_block_table(desc: &DatasetDescription) -> Result<()> { /// request surface entirely. `output.name` has no such default — a table that /// declares none is simply not addressable in `fields` — so only declared ones /// are claimed there. +/// +/// The request's reserved keys are claimed before any table: a request is read +/// for them first, so a table holding one of them could never be addressed. fn check_names_are_unique(desc: &DatasetDescription) -> Result<()> { - let mut request_names: HashMap<&str, &str> = HashMap::new(); + let mut request_names: HashMap<&str, &str> = RESERVED_REQUEST_KEYS + .iter() + .map(|key| (*key, "a reserved key of every request")) + .collect(); let mut output_names: HashMap<&str, &str> = HashMap::new(); for (table_name, table) in &desc.tables { @@ -2669,12 +2678,13 @@ tables: } /// A duplicate name makes a client's request ambiguous, and iteration order - /// — not the catalog — decides which table answers it (INV-D10). + /// — not the catalog — decides which table answers it (INV-D10). A request + /// name that is a reserved key is held by every request already. /// /// Covers CT-1 · INV-D10 #[test] fn test_validate_rejects_duplicate_names() { - let catalog = |request_name: &str, output_name: &str| { + let catalog = |request_name: &str, output_name: &str, alias_name: &str| { format!( r#" version: v2 @@ -2710,22 +2720,34 @@ tables: block_number: {{ type: uint64 }} seq: {{ type: uint32 }} aliases: - aliased: + {alias_name}: table: items filters: [] "# ) }; - parse_dataset_description(&catalog("others", "other")).expect("distinct names must load"); + parse_dataset_description(&catalog("others", "other", "aliased")) + .expect("distinct names must load"); - for (what, request_name, output_name) in [ - ("two tables claiming one request name", "items", "other"), - ("two tables claiming one output name", "others", "item"), + let mut refused = vec![ ( - "an alias claiming a table's request name", + "two tables claiming one request name", + "items", + "other", "aliased", + ), + ( + "two tables claiming one output name", + "others", + "item", + "aliased", + ), + ( + "an alias claiming a table's request name", + "others", "other", + "items", ), // `blocks` declares no request name, so it holds its own — and a // table declaring that name takes it, leaving the block table @@ -2734,11 +2756,18 @@ aliases: "a table claiming another's undeclared request name", "blocks", "other", + "aliased", ), - ] { + ]; + for key in RESERVED_REQUEST_KEYS { + refused.push(("a table claiming a reserved key", key, "other", "aliased")); + refused.push(("an alias claiming a reserved key", "others", "other", key)); + } + + for (what, request_name, output_name, alias_name) in refused { assert!( - parse_dataset_description(&catalog(request_name, output_name)).is_err(), - "{what} must be refused" + parse_dataset_description(&catalog(request_name, output_name, alias_name)).is_err(), + "{what} ({request_name}, {output_name}, {alias_name}) must be refused" ); } } diff --git a/crates/metadata/src/types.rs b/crates/metadata/src/types.rs index 3ffb2fa..e4384b9 100644 --- a/crates/metadata/src/types.rs +++ b/crates/metadata/src/types.rs @@ -535,6 +535,19 @@ pub enum RelationKind { Parents, } +/// The top-level keys of a request that are not item requests (spec §2.1). A +/// request is read for these first, so a table or alias whose request name is +/// one of them could never be addressed. +pub const RESERVED_REQUEST_KEYS: &[&str] = &[ + "type", + "fromBlock", + "toBlock", + "includeAllBlocks", + "fields", + "parentBlockHash", + "sets", +]; + /// `P-MAX-DISCRIMINATOR-BYTES` (spec/09-parameters.md §9.1): the longest prefix /// a discriminator can dispatch on, and so the largest length a catalog may map /// to a column. diff --git a/spec/01-data-model.md b/spec/01-data-model.md index 6299ead..d46ee4b 100644 --- a/spec/01-data-model.md +++ b/spec/01-data-model.md @@ -240,7 +240,7 @@ Filter kinds: | Kind | Client supplies | Semantics | |---|---|---| -| `inList` | array of values | Row matches if the column value is in the array. §4.2 | +| `inList` | array of values, and on a column rendered as text of set references (§2.7) | Row matches if the column value is in the array, or passes the lookup of a set it names. §4.2, §4.2.11 | | `equals` | a scalar | Row matches if the column value equals it. | | `rangeGte` / `rangeLte` | a number | Inclusive bound on a named column. | | `gteConst` | a boolean | When `true`, constrains a named column to be ≥ a catalog-fixed constant. Used for "is non-zero" flags over minimal-form hex. | @@ -369,8 +369,8 @@ They are collected as [INV-D1](07-invariants.md#inv-d1) … [INV-D11](07-invaria - Every alias: its target table exists; its implicit-filter columns exist; its `columnAlias` filters target real columns and are in its filter list; its relations are valid. -- Request names are unique across tables *and* aliases; output names are - unique across tables. +- Request names are unique across tables *and* aliases, and none is a reserved + key of §2.1; output names are unique across tables. The last check is easy to overlook and easy to violate. A duplicate request name makes a client's request ambiguous, and the ambiguity is resolved by iteration diff --git a/spec/02-request.md b/spec/02-request.md index aa546ca..bf5a6b3 100644 --- a/spec/02-request.md +++ b/spec/02-request.md @@ -22,7 +22,7 @@ A query is a single JSON object. } ``` -Three kinds of key appear at the top level: the six **reserved keys** above, and +Two kinds of key appear at the top level: the **reserved keys** of §2.1, and **item request arrays** named after a table's request name (§1.2). There is no third kind. A key that is neither reserved nor a known request name is an error ([INV-Q2](07-invariants.md#inv-q2)). @@ -37,6 +37,7 @@ third kind. A key that is neither reserved nor a known request name is an error | `parentBlockHash` | string | absent | Hash the client believes precedes `fromBlock`. | | `includeAllBlocks` | boolean | `false` | Emit headers for blocks with no items. | | `fields` | object | `{}` | Output field selection, per table. | +| `sets` | object | `{}` | Named value sets that filters refer to (§2.7). | Every key but `type` is optional; the defaults above are normative ([INV-Q9](07-invariants.md#inv-q9)). @@ -194,25 +195,31 @@ An engine MUST enforce, before reading any data: | Values in one bloom filter | ≤ `P-MAX-BLOOM-VALUES` | [INV-Q10](07-invariants.md#inv-q10) | | Discriminator-family filters per item request | ≤ `P-MAX-DISCRIMINATOR-FILTERS` | [INV-Q11](07-invariants.md#inv-q11) | | Bytes in one discriminator value | ≤ `P-MAX-DISCRIMINATOR-BYTES` | [INV-Q12](07-invariants.md#inv-q12) | +| Canaries in one set | at least one, ≤ `P-MAX-SET-CANARIES` | [INV-Q17](07-invariants.md#inv-q17) | The item-request cap is counted across *all* tables and aliases uniformly ([ADR-6](decisions/ADR-6-aliases-count-toward-the-request-cap.md)). A per-table exemption would let a client aim a hundred requests at one table and a hundred more at an alias of the same table. -An engine SHOULD additionally bound the whole request to `P-MAX-REQUEST-BYTES` -and a single `inList` to `P-MAX-IN-LIST` values -([INV-Q13](07-invariants.md#inv-q13)). One hundred item requests, each with a +An engine SHOULD additionally bound the whole request to `P-MAX-REQUEST-BYTES`, +a single `inList` to `P-MAX-IN-LIST` values, and the set references a request +makes to `P-MAX-SET-REFERENCES` ([INV-Q13](07-invariants.md#inv-q13)). One hundred item requests, each with a filter list of a million addresses, is a well-formed query by the rules above and a memory-amplification attack in practice. +A set (§2.7) counts toward `P-MAX-REQUEST-BYTES` like the rest of the request. +`P-MAX-IN-LIST` does not apply to it: a set is not a list, and the bytes that +carry it are its bound. What bounds the references is their number, since each +is a lookup for every row its table scans. + Every value here is resolved in [09-parameters.md](09-parameters.md). ## 2.5 Filter value forms | Filter kind | Accepted value | Rejected | |---|---|---| -| `inList` | array of strings, or of numbers where the column is numeric | anything else | +| `inList` | array of strings, or of numbers where the column is numeric; on a `hexBytes`, `base58` or `utf8` column the array may also hold set references (§2.7) | anything else | | `equals` | string, number, or boolean matching the column's type | arrays | | `rangeGte` / `rangeLte` | unsigned integer | strings, floats, negatives | | `gteConst` | boolean | anything else | @@ -232,10 +239,122 @@ error either. It matches nothing ([INV-P14](07-invariants.md#inv-p14)). ## 2.6 Validation order -All of §2.1–§2.5 MUST be checked before any chunk data is read, and an engine +All of §2.1–§2.5 and §2.7 MUST be checked before any chunk data is read, and an engine MUST NOT emit partial output before a validation error ([INV-E2](07-invariants.md#inv-e2)). Validation is a pure function of the request and the catalog. Errors detectable only against a chunk — a missing column, a fork — are execution errors and are covered in [06-errors.md](06-errors.md). + +## 2.7 Sets + +A **set** stands in for a list too long to send. The client builds a compact +structure over its values and sends that instead, and the engine keeps every row +whose value the structure may hold. The structure has false positives and no +false negatives, so the answer contains the one the same values would give as a +list, and the client removes the extra rows itself +([ADR-17](decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md)). + +Sets are declared once, under `sets`, and filters refer to them by name: + +```jsonc +{ + "type": "evm", + "sets": { + "pairs": { + "fuse8": "", + "canaries": ["0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc", "0x0d4a11d5…"] + } + }, + "fields": { + "log": { "address": true }, + "transaction": { "from": true, "to": true } + }, + "logs": [{ "address": [{ "set": "pairs" }], "topic0": ["0x1c41…"] }], + "transactions": [{ "from": [{ "set": "pairs" }] }, { "to": [{ "set": "pairs" }] }] +} +``` + +A name is any string. It means nothing beyond tying references to their set, so +a client may make names up however it likes. + +### The set object + +| Key | Value | +|---|---| +| `fuse8` | The structure, as canonical base 64 with padding (RFC 4648). §4.2.11 says what its bytes are. | +| `canaries` | An array of strings: a few of the set's values, as the client holds them. | + +A set carries exactly one structure key, and `fuse8` is the only one. The key +names the format, so a later format can arrive under a key of its own without +changing what `fuse8` means. A set with no structure key, with two, or with an +unknown one is `InvalidSet`. So is a structure that is not canonical base 64 or +whose header does not fit its body (§4.2.11), and so is a set that no filter +refers to, which is nearly always a reference the client meant to write and did +not ([INV-Q15](07-invariants.md#inv-q15)). + +### References + +A filter refers to a set with an object that holds one key, `{"set": ""}`, +written as an element of the filter's list. A list may hold values and references +to several sets, and a row passes it when it equals one of the values or one of +the sets passes it (§4.2.11). So a client adds a few values to a set, or joins two +sets, without building one again: + +```jsonc +"logs": [{ "address": ["0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f", { "set": "pairs" }, { "set": "pools" }] }] +``` + +A reference written alone is a list of one, as a value written alone is. An +object in a list that is not a reference is `InvalidFilterValue`. + +- The name MUST be declared under `sets`. Otherwise `UnknownSet` + ([INV-Q15](07-invariants.md#inv-q15)). +- The filter MUST be an `inList` filter on a column whose encoding is + `hexBytes`, `base58` or `utf8` (§1.5). A reference in any other list is + `InvalidFilterValue` ([INV-Q16](07-invariants.md#inv-q16)). The structure is + looked up with the text the response renders, and only these encodings render a + value as the text a client holds. +- The table's selected fields MUST render the filtered column, directly or + through the virtual field that rolls it (§2.3): `address` for a set on + `logs.address`, `topics` for one on `logs.topic0`. Otherwise + `SetFieldNotSelected` ([INV-Q18](07-invariants.md#inv-q18)). A client can + remove only the extra rows it can see. +- A request SHOULD make at most `P-MAX-SET-REFERENCES` references, summed across + every list of every item request, a name counted once per list. Otherwise + `RequestTooLarge` ([INV-Q13](07-invariants.md#inv-q13)). + +Any number of filters may refer to one set, and its structure still travels once. +That is why sets are declared apart from the filters that use them: +`from ∈ S ∨ to ∈ S` is two item requests ([INV-P5](07-invariants.md#inv-p5)), and +so are the two owners of a Solana token balance. + +A set has at least one value, because each canary is one of its values. A client +whose set is empty sends `[]` instead, which matches nothing +([INV-P3](07-invariants.md#inv-p3)). + +### Canaries + +A structure is opaque. The engine cannot tell one built from the right bytes from +one built from checksummed addresses, from addresses without their `0x`, from raw +bytes, or with another hash. Each of those matches almost nothing, and an answer +that is almost empty looks like a quiet stretch of chain: a wrong answer that +looks right ([§6.1](06-errors.md#61-principles)). Canaries make the mistake an +error. + +A set carries at least one canary and at most `P-MAX-SET-CANARIES`, otherwise it +is `InvalidSet`. For every column that refers to the set, the engine folds each +canary as that column folds a list value ([INV-P8](07-invariants.md#inv-p8)), +checks that the result is a value the column can render — well-formed hex (§2.5) +on a `hexBytes` column, a non-empty run of base58 digits on a `base58` one — and +looks it up in the structure. A canary that fails either check is +`SetCanaryMismatch` ([INV-Q17](07-invariants.md#inv-q17)). + +A wrong structure passes a canary only by a false positive, so each canary a +client adds multiplies the chance that a wrong structure gets through by the +format's false-positive rate. A client SHOULD draw its canaries at random from +the set each time it builds one: a mistake that spoils only some values is caught +only by a canary among them. No canary catches a set on the wrong field, such as +a set of addresses on a topic filter, because the canaries are well-formed values +there too. diff --git a/spec/04-evaluation.md b/spec/04-evaluation.md index 5092b53..e3e68a1 100644 --- a/spec/04-evaluation.md +++ b/spec/04-evaluation.md @@ -203,6 +203,75 @@ stat it cannot interpret MUST decline to prune, not guess. The one visible consequence of pruning is where truncation lands (§5.5), and that is governed by weight, not by scan order. +### 4.2.11 Sets + +A set reference on column `c` (§2.7) is satisfied by a row whose value, as the +response renders it, passes the set's lookup +([INV-P17](07-invariants.md#inv-p17)): + +``` +satisfies(r, c ∈ set) ⇔ r[c] is not null ∧ lookup(set, text(r[c])) +``` + +A list that holds values `V` beside references to sets `S₁ … Sₙ` passes what any +of them passes: + +``` +satisfies(r, c ∈ [V, S₁ … Sₙ]) ⇔ r[c] ∈ V ∨ ∃ i . satisfies(r, c ∈ Sᵢ) +``` + +The values mean what they mean in a list without references: they are folded +(§4.2.4), and a malformed hex value matches nothing +([INV-Q12](07-invariants.md#inv-q12)). + +`text(v)` is the rendered string without its quotes: lowercase `0x` hex on a +`hexBytes` column, base58 digits on a `base58` one, the stored text on a `utf8` +one. It is the same text whatever the column's physical type, so a column stored +as bytes is looked up by the text it renders to, never by its bytes. + +Every value the client built the set from passes, and a value it did not build +it from MAY pass. The answer therefore contains the answer the same values give +as a list, and the extra rows are the client's to remove, as they are for a bloom +filter (§4.2.5). An engine cannot remove them, since it never sees the values. + +The lookup is the whole contract between a client's builder and the engine. The +engine never builds a set, so any bytes for which the lookup passes every value +are a correct set, however they were made. + +**`fuse8`.** The structure is a header followed by an array `F` of one-byte +fingerprints. The header is four unsigned little-endian integers: a 64-bit +`seed`, then the 32-bit `L`, `M` and `N`. Arithmetic is on unsigned 64-bit +integers, modulo 2⁶⁴: + +``` +sound ⇔ L is a power of two, L ≤ 2¹⁸, M = L − 1, + N > 0, N is a multiple of L, |F| ≥ N + 2L + +lookup(v): + k = XXH3-64(UTF-8 bytes of v) seed 0, default secret + h = mix(k + seed) + f = (h xor (h >> 32)) mod 2⁸ + i0 = ⌊h · N / 2⁶⁴⌋ + i1 = (i0 + L) xor ((h >> 18) and M) + i2 = (i0 + 2L) xor (h and M) + pass ⇔ f xor F[i0] xor F[i1] xor F[i2] = 0 + +mix(x): + x = x xor (x >> 33); x = x · 0xff51afd7ed558ccd + x = x xor (x >> 33); x = x · 0xc4ceb9fe1a85ec53 + x = x xor (x >> 33) +``` + +A structure that is not sound is `InvalidSet`, because its lookup could read past +`F` ([INV-Q15](07-invariants.md#inv-q15)). The share of other values that pass +is the format's false-positive rate, which the fingerprint's width sets +([ADR-17](decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md)). + +A minimum and a maximum say nothing about which values a structure holds, so no +statistic can rule a row group out for a set, nor for a list that holds one +([INV-P16](07-invariants.md#inv-p16)). Looking a set up once per dictionary value +rather than once per row is ordinary pushdown. + ## 4.3 Relations ### 4.3.1 Source scoping diff --git a/spec/06-errors.md b/spec/06-errors.md index bf6f536..9ca8217 100644 --- a/spec/06-errors.md +++ b/spec/06-errors.md @@ -33,6 +33,7 @@ body: | `parentBlockHash` mismatch | Serve the data | Fork error ([INV-E5](07-invariants.md#inv-e5)) | | `parentBlockHash` on a dataset with no parent-hash column | Accept and skip the check | Error ([INV-E5](07-invariants.md#inv-e5)) | | Join key of an unsupported type | Match nothing | Error ([INV-E7](07-invariants.md#inv-e7)) | +| Set built from other bytes than the column renders | Match only the rows it happens to | Error ([INV-Q17](07-invariants.md#inv-q17)) | **Errors are stable.** Each error has a machine-readable kind. Clients switch on the kind; only humans read the message ([INV-E6](07-invariants.md#inv-e6)). @@ -56,7 +57,11 @@ Raised from the request and catalog, before any data is read. | `ConflictingFilters` | More than one discriminator-family filter in one item request. | | `InvalidHex` | A `discriminator` value lacks `0x`, has odd length, or contains a non-hex digit. | | `DiscriminatorTooLong` | A discriminator value exceeds `P-MAX-DISCRIMINATOR-BYTES`. | -| `InvalidFilterValue` | A filter value has a form the filter kind does not accept. | +| `InvalidFilterValue` | A filter value has a form the filter kind does not accept, including a set reference on a filter that cannot take one ([INV-Q16](07-invariants.md#inv-q16)). | +| `UnknownSet` | A filter refers to a set that `sets` does not declare. | +| `InvalidSet` | A set has no structure key, two, or an unknown one; a structure that is not canonical base 64 or not sound; a number of canaries out of bounds; or no filter refers to it ([INV-Q15](07-invariants.md#inv-q15), [INV-Q17](07-invariants.md#inv-q17)). | +| `SetCanaryMismatch` | A canary is not a value the referring column can render, or the set's structure does not hold it ([INV-Q17](07-invariants.md#inv-q17)). | +| `SetFieldNotSelected` | A filter refers to a set on a column that no selected field renders ([INV-Q18](07-invariants.md#inv-q18)). | | `RequestTooLarge` | The request exceeds an engine-configured byte or list-length bound. | | `UnsupportedRequestField` | A reserved key the dataset cannot honour — `parentBlockHash` where the block table declares no parent-hash column. | diff --git a/spec/07-invariants.md b/spec/07-invariants.md index 23cfa25..09230b2 100644 --- a/spec/07-invariants.md +++ b/spec/07-invariants.md @@ -102,10 +102,11 @@ weight is unchanged by its presence. ### INV-D10 **Names are unique.** Within a dataset, the request name is unique across tables -and aliases; the output name is unique across tables. Within a table, no virtual -field shares its name with a column or a variant field. +and aliases, and none is a reserved key of §2.1; the output name is unique across +tables. Within a table, no virtual field shares its name with a column or a +variant field. -*Why:* a duplicate makes a client's request ambiguous, resolved by iteration order — arbitrarily. A field name that is two things is read as one by the writer and the weight model and as the other by the planner; a weight key column read as a roll of size-weighed columns is charged nothing. +*Why:* a duplicate makes a client's request ambiguous, resolved by iteration order — arbitrarily. A request name that is a reserved key is read as the reserved key, and its table can never be addressed. A field name that is two things is read as one by the writer and the weight model and as the other by the planner; a weight key column read as a roll of size-weighed columns is charged nothing. *Test:* static. ### INV-D11 @@ -126,8 +127,8 @@ All checked before any data is read ([INV-E2](#inv-e2)). name. Otherwise `UnknownDataset`. ### INV-Q2 -**No unknown top-level keys.** A top-level key is one of the six reserved keys -(§2.1) or a request name. Anything else is `UnknownTable`. +**No unknown top-level keys.** A top-level key is one of the reserved keys of +§2.1 or a request name. Anything else is `UnknownTable`. ### INV-Q3 **`fromBlock ≤ toBlock`.** Otherwise `InvalidBlockRange`. @@ -164,7 +165,8 @@ output name is `UnknownFieldGroup`. `fields.X` MUST be an object. ### INV-Q9 **Defaults.** `fromBlock` = 0. `toBlock` = unbounded. `includeAllBlocks` = false. -`fields` = `{}`. `parentBlockHash` = absent. Every item request array = `[]`. +`fields` = `{}`. `sets` = `{}`. `parentBlockHash` = absent. Every item request +array = `[]`. Inside an item request, every filter absent and every relation flag `false`. A default applies to an *absent* key only. A key that is present with the wrong @@ -213,10 +215,12 @@ nothing and reads nothing. ### INV-Q13 **Requests are bounded in size.** An engine SHOULD reject a request exceeding -`P-MAX-REQUEST-BYTES`, or containing an `inList` longer than `P-MAX-IN-LIST`, -with `RequestTooLarge`. +`P-MAX-REQUEST-BYTES`, containing an `inList` longer than `P-MAX-IN-LIST`, or +making more than `P-MAX-SET-REFERENCES` set references, with `RequestTooLarge`. +References are summed across every list of every item request, a name counted +once per list. -*Why:* 100 item requests, each with a million-address filter list, is well-formed under every rule above and a memory-amplification attack in practice. +*Why:* 100 item requests, each with a million-address filter list, is well-formed under every rule above and a memory-amplification attack in practice. So is one list naming ten thousand copies of a tiny set: ten thousand lookups for every value it meets. ### INV-Q14 **The field surface is closed.** A table's selectable fields are the ones the @@ -230,6 +234,43 @@ grouping, joining and rolling. *Why:* the same reason [INV-P15](#inv-p15) closes the filter surface. An open output surface makes the physical column layout part of the wire contract, and every column an archive writer adds becomes a field clients may pin on. *Test:* for every dataset and table, assert `fields` accepts exactly the §3 list. In particular `{"fields":{"log":{"blockNumber":true}}}` and `{"fields":{"log":{"topic0":true}}}` error, though `block_number` and `topic0` are real columns. +### INV-Q15 +**Sets are well-formed, declared and used.** Each value of `sets` is an object +with exactly one structure key, `fuse8`, holding canonical base 64 of a sound +structure ([§4.2.11](04-evaluation.md#4211-sets)), and a `canaries` array. A +reference `{"set": name}` names a declared set, otherwise `UnknownSet`. Any other +defect, a set that no filter refers to among them, is `InvalidSet`. + +*Why:* a reference to nothing and a set nobody uses are the same mistake seen from two sides, a filter the client meant to write and did not. Answering either drops a filter without a word. +*Test:* a reference to an undeclared name is `UnknownSet`. A declared set with no reference, a set with no structure key, with two, with `fuse16`, with bytes that are not canonical base 64, and with a header whose `N + 2L` exceeds its fingerprints are each `InvalidSet`. + +### INV-Q16 +**A set stands only in a list of text.** A set reference may be an element only +of an `inList` filter whose column's encoding is `hexBytes`, `base58` or `utf8`, +alone or beside values and other references. Anywhere else it is +`InvalidFilterValue`, and so is an object in a list that is not a reference. + +*Why:* the lookup hashes the text the response renders. A number, a quantity padded to its width, or a discriminator prefix has no text a client could have hashed the same way. +*Test:* a reference on `transactions.from` passes validation, alone and beside a value. One on a range filter (`firstNonce`) and one on a Solana `discriminator` are each `InvalidFilterValue`. + +### INV-Q17 +**A set's canaries are values it holds.** A set carries at least one canary and +at most `P-MAX-SET-CANARIES`, otherwise `InvalidSet`. For every column that refers +to the set, each canary, folded as the column folds a list value +([INV-P8](#inv-p8)), MUST be a value the column can render and MUST pass the +set's lookup. Otherwise `SetCanaryMismatch`, before any data is read. + +*Why:* a structure is opaque, and every way to build one wrong — hashing a checksummed address, dropping the `0x`, hashing raw bytes, another hash or seed — is accepted and matches almost nothing. That answer looks like a quiet stretch of chain. +*Test:* build a set from the values of a fixture chunk in each of the wrong ways above; each is `SetCanaryMismatch` with canaries drawn from those values. The right set with its canaries upper-cased passes on a `hexBytes` column. A hex canary on a `base58` column is `SetCanaryMismatch`. No canaries, and one more than `P-MAX-SET-CANARIES`, are each `InvalidSet`. + +### INV-Q18 +**A set-filtered column reaches the response.** For every filter that refers to a +set, the table's selected fields render the filtered column, directly or through +the virtual field that rolls it. Otherwise `SetFieldNotSelected`. + +*Why:* the extra rows a set returns are the client's to remove, and a client can remove only what it can see. +*Test:* `{"logs":[{"address":[{"set":"s"}]}]}` without `fields.log.address` is `SetFieldNotSelected` and with it passes. A set on `topic0` passes with `fields.log.topics` and fails without it. + --- ## P — Filter algebra @@ -343,6 +384,17 @@ guess. *Test:* run every fixture query with pruning disabled; assert byte-identical responses. +### INV-P17 +**A set passes what its lookup passes.** A row satisfies a set reference on `c` +iff `r[c]` is not null and the set's lookup passes the text the response renders +for `r[c]` ([§4.2.11](04-evaluation.md#4211-sets)). A row satisfies a list that +holds values and set references iff it equals one of the values or satisfies one +of the references. An engine MUST NOT add a row the lookup rejects, nor drop one +it passes. + +*Why:* the lookup is all a client's builder and the engine share. An engine that looks up other bytes — a binary column's stored bytes, a value it did not render, another seed — drops members of the set, and no client can see that. +*Test:* for every filter of every bundled catalog that takes a set, build one from values a fixture chunk holds. Assert the answer contains the answer to the same values as a list, and equals the rows that an independent implementation of the lookup passes. Send the same values again as a list of some of them beside two sets of the rest, and assert the answer equals the rows the listed values and either lookup pass. + --- ## R — Relations diff --git a/spec/08-conformance.md b/spec/08-conformance.md index 401fa4a..47bcd23 100644 --- a/spec/08-conformance.md +++ b/spec/08-conformance.md @@ -68,7 +68,7 @@ that returns `Ok`. Table-driven. One case per row of §6.2, asserting the error *kind*, not the message. Covers [INV-Q1](07-invariants.md#inv-q1) – -[INV-Q13](07-invariants.md#inv-q13), [INV-E2](07-invariants.md#inv-e2), +[INV-Q18](07-invariants.md#inv-q18), [INV-E2](07-invariants.md#inv-e2), [INV-E6](07-invariants.md#inv-e6). The cases that matter most are the ones an engine is likeliest to get wrong by @@ -88,6 +88,12 @@ being permissive: → ConflictingFilters {"type":"solana","instructions":[{"discriminator":["0xabc"]}]} → InvalidHex +{"type":"evm","logs":[{"address":[{"set":"s"}]}]} → UnknownSet +{"type":"evm","sets":{"unused":{"fuse8":"…","canaries":["0x…"]}}} + → InvalidSet +{"type":"evm","sets":{"s":{"fuse8":"","canaries":["0xAbC…"]}}, + "fields":{"log":{"address":true}},"logs":[{"address":[{"set":"s"}]}]} + → SetCanaryMismatch ``` Then fuzz — [CT-9](#810-ct-9--fuzz). @@ -115,6 +121,9 @@ a few that do not. Then assert the laws, over randomly composed queries: | `Q(c: [huge])` = ∅, no error | [INV-P14](07-invariants.md#inv-p14) | | `Q(c: [a, m])` = `Q(c: [a])` for a malformed hex `m` the chunk does not store | [INV-Q12](07-invariants.md#inv-q12) | | `Q` with pruning disabled = `Q` | [INV-P16](07-invariants.md#inv-p16) | +| `Q(c: set(S)) ⊇ Q(c: S)` for a set built from `S` | [INV-P17](07-invariants.md#inv-p17) | +| `Q(c: set)` = the rows an independent lookup passes | [INV-P17](07-invariants.md#inv-p17) | +| `Q(c: [V, set(A), set(B)])` = `Q(c: V ∪ lookup(A) ∪ lookup(B))` | [INV-P17](07-invariants.md#inv-p17) | Two of these are worth writing by hand even if you generate the rest, because they encode the two catastrophic misreadings: @@ -279,7 +288,7 @@ reproduce is a rumour. ## 8.11 Traceability matrix -Dated **2026-09-12**. Every invariant, the class that checks it, and an honest +Dated **2026-10-23**. Every invariant, the class that checks it, and an honest status: - **C** — covered. A test would fail if the invariant broke. @@ -325,7 +334,7 @@ renamed out from under a comment inside it. | [INV-D7](07-invariants.md#inv-d7) | CT-6, CT-8 | **C** | `physical_width_does_not_reach_the_answer` rewrites one chunk at every one of the eight integer widths, for every integer column that can hold its values, and byte-compares each; `narrowing_every_column_at_once_does_not_reach_the_answer` the combination one column at a time hides; `a_narrow_column_in_a_shuffled_chunk_does_not_reach_the_answer` the pairing with a storage order that is not the item-key order, which is what reaches the sort comparator at all. `a_list_key_answers_the_same_at_any_element_width` covers the element types inside a list on a real chunk, and `a_fixture_chunk_answers_the_same_at_any_width` the scalar widths there. The filtered columns, which the sweep above reaches only through a string filter: `an_integer_filter_reads_every_physical_width` retypes a one-byte discriminator and an item index to every width under a plain list, the discriminator filter and an index list, and `a_boolean_filter_reads_a_flag_stored_as_a_number` a flag stored as a number. Underneath: `a_narrower_column_orders_against_a_wider_one_by_value`, block-range masks, in-lists (`an_unsigned_list_reads_every_physical_width`, `a_value_the_stored_width_cannot_hold_matches_nothing`, `a_string_list_reads_every_text_type`), scalars (`a_scalar_reads_every_physical_width`, `a_boolean_reads_a_numeric_column`), semi-joins, range predicates (`a_range_reads_every_physical_width`), `test_solana_tx_version_reads_the_sentinel_at_every_physical_width`, `test_bignum_reads_a_narrowed_column` | | [INV-D8](07-invariants.md#inv-d8) | CT-6 | **C** | `storage_layout_does_not_reach_the_answer` turns one knob per case, one per mechanism the invariant names — row groups of 1, 7 and more than the table holds; data pages of 1 and 5; uncompressed and snappy; no dictionary; no statistics; and the rows stored back to front and then in no order at all, which is a permutation no sort key produces. `a_fixture_chunk_answers_the_same_under_any_layout` repeats six of them on an archiver's chunk. Storage that is not parquet at all: `a_columnar_reader_answers_what_the_parquet_reader_does` reads the same chunks through the columnar reader, with no statistics and with windows from larger than the table down to one row, and `every_fixture_query_answers_the_same_through_the_columnar_reader` repeats every fixture query of every dataset that way. Statistics windows that do not cover the table — offsets that start past row 0, step back, overshoot, or end elsewhere — prune nothing: `window_offsets_that_do_not_cover_the_table_prune_nothing` | | [INV-D9](07-invariants.md#inv-d9) | CT-1 | **C** | `test_system_columns_excluded_from_weight`, `undeclared_columns_are_not_filterable`, `test_validate_rejects_fields_backed_by_system_columns` — direct, virtual-field and variant-field exposure | -| [INV-D10](07-invariants.md#inv-d10) | CT-1 | **C** | `test_validate_rejects_duplicate_names` — two tables on one request name, two on one output name, an alias on a table's request name, and a table on the name another holds without declaring it; `test_validate_rejects_a_virtual_field_that_shares_a_name` — a virtual field named like a weight key column, an item order key, a column it rolls, any other column and a variant field | +| [INV-D10](07-invariants.md#inv-d10) | CT-1 | **C** | `test_validate_rejects_duplicate_names` — two tables on one request name, two on one output name, an alias on a table's request name, a table on the name another holds without declaring it, and a table and an alias on each reserved key; `test_validate_rejects_a_virtual_field_that_shares_a_name` — a virtual field named like a weight key column, an item order key, a column it rolls, any other column and a variant field | | [INV-D11](07-invariants.md#inv-d11) | CT-1 | **C** | `test_validate_rejects_a_weight_key_with_no_fixed_weight` — a declared key column weighed by its size or marked `system`, and each default key column weighed by its size: the item order key, the address column, the variant column, an item table's block number and the block table's number | | [INV-Q1](07-invariants.md#inv-q1) | CT-2 | **P** | reached through every fixture; no negative case for an unknown `type` | | [INV-Q2](07-invariants.md#inv-q2) | CT-2 | **C** | `test_parse_unknown_table_error`; `test_internal_table_names_are_not_request_keys` tries every table name and snake-cased alias of every bundled catalog that is not a request name | @@ -339,8 +348,12 @@ renamed out from under a comment inside it. | [INV-Q10](07-invariants.md#inv-q10) | CT-2 | **C** | `a_bloom_filter_takes_at_most_ten_values`, either side of the cap | | [INV-Q11](07-invariants.md#inv-q11) | CT-2 | **C** | `one_discriminator_filter_per_item_request`; the family is read from the catalog, so the check is not Solana-specific | | [INV-Q12](07-invariants.md#inv-q12) | CT-2, CT-3, CT-7 | **C** | `a_malformed_hex_value_is_not_refused` tries every malformation on every hex filter of every bundled catalog, tables and aliases alike, beside a well-formed value, alone and bare; `a_malformed_discriminator_is_refused` and `a_hex_filter_refuses_a_value_that_is_not_a_string` hold the two refusals the rule keeps, and `a_discriminator_is_checked_in_full_whatever_its_order` holds them beside an empty prefix on either side of it; `test_parse_hex` the parse. `a_malformed_hex_value_matches_nothing` asserts the no-op and the empty match on a text, a hex-number and a plain integer column, through a second item request and a relation; `a_malformed_hex_value_compares_as_text_where_the_column_stores_text` a stored odd-length quantity, a fixed-width binary column and a list of text; `a_list_with_no_value_left_reads_no_column` a list with nothing left on a chunk without the column. `malformed_hex_values_answer_the_same_as_the_reference` compares every hex filter of every fixture chunk with the reference, and every hex-number filter again on a copy of the chunk without its column; `a_discriminator_with_an_empty_prefix_answers_the_same_as_the_reference` an empty prefix beside every refused value, in both orders | -| [INV-Q13](07-invariants.md#inv-q13) | CT-2 | **C** | `a_request_is_bounded_in_bytes` and `an_in_list_is_bounded_in_length`, either side of each cap; the list case is written with short values so the byte cap cannot be what it is measuring | +| [INV-Q13](07-invariants.md#inv-q13) | CT-2 | **C** | `a_request_is_bounded_in_bytes`, `an_in_list_is_bounded_in_length` and `set_references_are_bounded_in_number`, either side of each cap; the list case is written with short values so the byte cap cannot be what it is measuring, and the reference case counts across items, counts one set named from many items each time, and a name repeated in one list once | | [INV-Q14](07-invariants.md#inv-q14) | CT-2 | **C** | `the_field_surface_is_exactly_the_declared_one` compares every table's declared list against a transcription of the reference's, as a set, and then parses each name. The transcription alone stayed green through the thirteen fields of gap 35, so `the_catalogs_serve_every_field_the_reference_serves` recomputes it from the reference's own `item_field_selection!` blocks wherever the fixture tree's checkout is present, both ways: a field the reference adds fails here before a review finds it. `a_column_the_catalog_carries_is_not_a_field` covers `blockNumber` and `topic0` | +| [INV-Q15](07-invariants.md#inv-q15) | CT-2 | **C** | `a_set_is_declared_used_and_well_formed`: a reference to an undeclared name, a declared set nobody refers to (alone and beside one that is used), and a set with no structure key, two, an unknown one, an unknown key, bytes that are not a string, base 64 with a line break or without its padding, and fingerprints one short of `N + 2L`; a reference beside an empty list still counts as one. `an_unsound_header_is_refused` breaks each condition of soundness alone, `L` above 2¹⁸ among them, and `a_sound_header_keeps_every_lookup_inside` hashes twenty thousand values through the smallest and largest sound headers | +| [INV-Q16](07-invariants.md#inv-q16) | CT-2 | **C** | `a_set_stands_only_in_a_list_of_text` sends a reference on every filter of every bundled catalog, tables and aliases alike, alone and in a list beside a value, with the fields that render its column selected: accepted on the filters over `hexBytes`, `base58` and `utf8` text, `InvalidFilterValue` everywhere else, `SetFieldNotSelected` where no field renders the column. The invariant's own cases are asserted by name beside the walk, and so are four objects that are not references, alone and in a list | +| [INV-Q17](07-invariants.md#inv-q17) | CT-2 | **C** | `a_sets_canaries_are_values_it_holds` builds a set from checksummed text, from text without `0x`, from the raw bytes, with another seed and with another hash, each refused by four checksummed canaries; the right set passes with them and upper-cased. One non-member among members, a hex canary with an odd digit count and a hex canary on a `base58` column are refused even where the structure holds them, the last two by form. A set on a `hexBytes` and a `utf8` column at once is refused by the column that does not fold. Zero canaries and one over `P-MAX-SET-CANARIES` are `InvalidSet`, and exactly `P-MAX-SET-CANARIES` passes | +| [INV-Q18](07-invariants.md#inv-q18) | CT-2 | **C** | `a_set_filtered_column_reaches_the_response`: `logs.address` without `fields.log.address`, with it switched off, and with only another table's field selected, each `SetFieldNotSelected`; `topic0` passes through `topics` and fails without it. `a_set_stands_only_in_a_list_of_text` covers every other filter, the extraction columns of aliases among them, which no field renders | | [INV-P1](07-invariants.md#inv-p1) | CT-3 | **C** | `an_unfiltered_item_request_is_the_whole_table_and_a_filter_only_narrows` counts an unfiltered request against the chunk's own rows in range, read off the parquet rather than from another query | | [INV-P2](07-invariants.md#inv-p2) | CT-3 | **C** | `test_in_list_predicate_strings`, `test_in_list_predicate_u64`, `test_numeric_in_list_filter`, and `a_value_list_is_the_union_of_its_values` over generated splits of a column's real values — including the split that leaves one side empty, which is the same law with [INV-P3](07-invariants.md#inv-p3) on one arm. `value_order_repetition_and_misses_do_not_reach_the_answer` pins the list as a set | | [INV-P3](07-invariants.md#inv-p3) | CT-3 | **C** | `an_empty_filter_list_matches_nothing` — the discriminator, a discriminator column, an ordinary in-list, and an empty list beside a filter that does match; `a_list_with_no_value_left_reads_no_column` — a list emptied by dropping what the column cannot hold, on a hex-number, a plain integer, a fixed-width binary, a list-of-integers and a signed column, each on a chunk without that column, and the item's other filters still validated whichever key comes first | @@ -357,6 +370,7 @@ renamed out from under a comment inside it. | [INV-P14](07-invariants.md#inv-p14) | CT-3 | **C** | `unmatchable_values_are_not_errors` and `a_hex_value_too_wide_for_the_column_matches_nothing` for values wider than the column, `a_signed_column_takes_negative_filter_values` for one below its floor, and `a_block_bound_past_the_stored_width_matches_nothing` for the half the invariant states about the block bounds — where wrapping would truncate into a block the chunk holds | | [INV-P15](07-invariants.md#inv-p15) | CT-3 | **C** | `undeclared_columns_are_not_filterable`, `an_alias_has_its_own_filter_surface`, `an_alias_has_its_own_relation_surface` | | [INV-P16](07-invariants.md#inv-p16) | CT-3 | **P** | `pruning_never_drops_a_row_the_mask_would_keep` states the decision as a law — a group may be pruned only where the mask would have emptied it — over the eight widths a writer may pick, each column holding its own extremes, against every filter kind the plan compiles on an integer column (equality, in-list, both ranges) at every value kind it compiles them with (`uint8` through `uint64`, `int16`, `int64`), across each width's boundaries. Each of the fix's three arms fails it under mutation. It caught pruning that read a statistic at the parquet physical type while evaluation narrowed to the stored one, so a `uint8` 255 pruned the group holding the `-1` an `Int8` column stores it as, and the answer turned on whether the writer had recorded statistics. `a_statistic_that_is_not_text_prunes_nothing` covers the decline half the invariant names. The equality run is `a_filter_returns_the_same_rows_with_nothing_to_prune_on`: five filters — matching, non-matching, conjunctive, empty-list, absent — against chunks written with no statistics, a row per row group, a row per data page, and no dictionary, compared as bytes; `a_nested_column_earlier_in_the_schema_does_not_reach_the_answer` adds the case that made the invariant load-bearing, a `List>` earlier in the schema putting every filter after it on another column's bounds. `a_row_group_an_items_statistics_rule_out_holds_none_of_its_rows` covers pruning per item: random items over sorted text and integer columns, with alternatives, bounds cut to a prefix and a column with no statistics, against every row evaluated by hand. The same sweep covers the other half, a filter the statistics prove and the scan leaves out: row groups holding one value of a column, with and without nulls beside it, and each item's tag checked as well as the rows; `a_proof_holds_exactly_where_the_mask_keeps_every_row` states the proof as a law over the eight widths, exact for equality and lists. What a plausible break still passes: a writer that records no nulls where there are some is believed; the law is stated over integer columns only, so text pruning rests on the byte-equality run and that sweep; filter reordering is not varied; early termination is CT-5's `scan` module; and the equality run is five filters over one query shape rather than the every-fixture-query sweep the invariant asks for | +| [INV-P17](07-invariants.md#inv-p17) | CT-3 | **C** | `a_set_passes_exactly_what_its_lookup_passes`: for every filter that takes a set, on every fixture chunk, a set built by `xorf` from up to half the strings the table's rows render is compared, row for row across every page, with a list of every string `xorf`'s own lookup passes, and holds the answer to its members as a list. The members are sent again as a list of a third of them beside two sets of the rest, and compared with a list of the listed values and every string either lookup passes. The sweep requires that some set passed a row outside its members, so the equality has seen false positives, and that most sets left rows out. `the_lookup_agrees_with_the_builders_own` compares the engine's lookup with `xorf`'s on members and on twenty thousand others at eight sizes. Dictionary columns, columns stored as bytes, the reuse of a verdict for a repeated value and a list of values beside two sets have unit tests of their own | | [INV-R1](07-invariants.md#inv-r1) | CT-4 | **C** | `a_relation_pulls_only_its_own_item_requests_matches` generates §8.5's two-item-request construction — disjoint halves of one filter's values, only the first carrying the flag. The sweep fails unless some split had a second half reaching a target the first does not, which is the only case that can tell the two behaviours apart: over a filter whose every value shares the same targets, an engine scoping the relation to the whole table answers identically, and no number of cases notices. `test_alias_relation_source_items` is the compile-side half. The scan marks each relation's rows as it evaluates the items, and `a_relation_follows_the_rows_its_own_items_matched` checks every mark against the items evaluated again, through block ranges, positions, batch sizes, null filter values and items that row-group statistics rule out; `relations_on_one_key_follow_only_their_own_items` covers two relations on one key that could share a key set. `a_relation_planned_once_follows_every_item_that_asked_for_it` covers the plan that evaluates one relation for several items: one join under two names, one name meaning two joins through an alias or through two aliases, two joins apart only in the target key, and one relation flagged twice in one item | | [INV-R2](07-invariants.md#inv-r2) | CT-4 | **C** | `a_pulled_row_does_not_pull_its_own`. The chain's relations are a cycle on purpose, so a second hop is *observable* rather than merely non-terminating: from a filtered log request it would come back through `transactions.log` carrying the logs the filter excluded, and through `transactions.trace` carrying traces nothing asked for. Both are asserted — the origin table unchanged, the table two hops out empty | | [INV-R3](07-invariants.md#inv-r3) | CT-4 | **C** | `a_row_reachable_several_ways_is_returned_once` compares emitted rows against distinct rows for every table of every generated query, over queries that ask every table directly *and* through its relations — which is what builds the overlapping paths nobody enumerates: a row matched by two item requests, a row pulled by two relations, a row both matched and pulled. `test_arrow_multisource_dedup` is the Arrow path's | @@ -403,25 +417,25 @@ renamed out from under a comment inside it. | [INV-X2](07-invariants.md#inv-x2) | CT-8 | **C** | `an_ignored_nullable_column_does_not_change_output` | | [INV-X3](07-invariants.md#inv-x3) | CT-8 | **C** | `filtering_an_absent_column_is_an_error` on both scan entry points, `filtering_a_present_column_still_works` for the other direction, `one_unanswerable_item_rejects_the_whole_request` for a filter one item request of several carries, and `an_alias_filter_on_a_column_the_chunk_lacks_is_an_error` for one reached through an alias's extraction column | -**Totals: 65 C, 21 P, 0 U** of 86. Property coverage is therefore 0.76 +**Totals: 70 C, 21 P, 0 U** of 91. Property coverage is therefore 0.77 (`P-COV-PROPERTY` in [09-parameters.md](09-parameters.md)). -The figure fell from 0.73 to 0.69 on 2026-09-12, and no test was removed to make it fall. [INV-D6](07-invariants.md#inv-d6), [INV-P13](07-invariants.md#inv-p13) and [INV-Q14](07-invariants.md#inv-q14) were read against the tests they name and did not hold at **C**; their rows now say what those tests actually do. A ratchet on a number three rows had inflated would have locked the inflation in, so the baseline moved down once, deliberately, and ratchets from there. INV-Q14 earned its **C** back the same day with a test that recomputes the transcription from the reference, and INV-P13 later with tests that send a mixed-length discriminator and an empty prefix and check the rows. +The figure fell from 0.73 to 0.69 one day in September 2026, and no test was removed to make it fall. [INV-D6](07-invariants.md#inv-d6), [INV-P13](07-invariants.md#inv-p13) and [INV-Q14](07-invariants.md#inv-q14) were read against the tests they name and did not hold at **C**; their rows now say what those tests actually do. A ratchet on a number three rows had inflated would have locked the inflation in, so the baseline moved down once, deliberately, and ratchets from there. INV-Q14 earned its **C** back the same day with a test that recomputes the transcription from the reference, and INV-P13 later with tests that send a mixed-length discriminator and an empty prefix and check the rows. -Of the 86 rows at **C** or **P**, 75 are backed by a tagged test and 11 rest on +Of the 91 rows at **C** or **P**, 80 are backed by a tagged test and 11 rest on prose alone. Those 11 are the rows whose note says "fixtures only" or describes a group of cases without naming one, and they are the ones nothing recomputes: the status is what somebody believed on the day they typed it. Shrinking that number is what turns MG-1's ratchet from an intention into an arithmetic fact, so the checker recomputes all three numbers in this paragraph rather than trusting them. -A tag is not the same as a gate. 3 of the 75 are backed only by tests marked +A tag is not the same as a gate. 3 of the 80 are backed only by tests marked `#[ignore]`, which MG-3's portable job does not run: the test would fail if the invariant broke, but only on a machine that has the chunks. The checker reports those rows on every run too, because a status that no job can falsify is a status worth seeing. -No row is left at **U**. That is a smaller claim than it sounds: 24 rows are +No row is left at **U**. That is a smaller claim than it sounds: 21 rows are partial, and a **P** is a row somebody looked at and did not finish. What it does settle is that every invariant now has somewhere for a failure to show up. diff --git a/spec/09-parameters.md b/spec/09-parameters.md index 239571c..cd963d5 100644 --- a/spec/09-parameters.md +++ b/spec/09-parameters.md @@ -29,16 +29,31 @@ Checked before any chunk data is read ([INV-E2](07-invariants.md#inv-e2)). | `P-MAX-DISCRIMINATOR-BYTES` | Bytes in one discriminator value ([INV-Q12](07-invariants.md#inv-q12)) | 16 | 16 | | `P-MAX-REQUEST-BYTES` | Whole-request size bound ([INV-Q13](07-invariants.md#inv-q13)) | 2 MiB | 2 MiB | | `P-MAX-IN-LIST` | Values in one `inList` ([INV-Q13](07-invariants.md#inv-q13)) | 100 000 | 100 000 | +| `P-MAX-SET-CANARIES` | Canaries in one set ([INV-Q17](07-invariants.md#inv-q17)) | 64 | ⚠ 64 | +| `P-MAX-SET-REFERENCES` | Set references summed across every list of every item request, a name counted once per list ([INV-Q13](07-invariants.md#inv-q13)) | 64 | ⚠ 64 | The discriminator family is read from the catalog, not named in code: it is the `discriminator` special filter plus every column it dispatches to. A dataset with a differently-named discriminator is bounded by the same rule with no code change ([INV-X1](07-invariants.md#inv-x1)). -`P-MAX-REQUEST-BYTES` bounds parsing work and total request representation. -`P-MAX-IN-LIST` independently bounds the set built for one filter, including the -case where short values allow a large list to fit below the byte cap -([ADR-13](decisions/ADR-13-request-resource-bounds.md)). +`P-MAX-REQUEST-BYTES` bounds parsing work and total request representation, and +it is the only bound on a set's size (§2.7). `P-MAX-IN-LIST` independently bounds the +set built for one filter, including the case where short values allow a large +list to fit below the byte cap ([ADR-13](decisions/ADR-13-request-resource-bounds.md)). + +`P-MAX-SET-CANARIES` bounds the work a set's canaries add to validation: each is +one lookup, for every column that refers to the set. It is a proposal of +[ADR-17](decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md), which is itself proposed. Four canaries already +let a wrong structure through less than once in a billion requests, so the bound +is far above what a client needs. + +`P-MAX-SET-REFERENCES` bounds the lookups a request adds to every row: each +reference is one, for every row its table scans. A set costs a client a few dozen +bytes to declare again under another name, so the bytes do not bound them, and +references are counted rather than sets. It is a proposal of the same ADR. A +client joins a few sets in a list and refers to each from a few filters, well +under it. ## 9.2 Response bounds @@ -80,7 +95,7 @@ its target together ([ADR-14](decisions/ADR-14-a-matrix-correction-rebaselines-t | Parameter | Role | Observed | Target | |---|---|---|---| -| `P-COV-PROPERTY` | Fraction of invariants at status **C** in the traceability matrix ([MG-1](08-conformance.md#812-merge-gates)) | 0.76 (65 of 86) | ⚠ 0.69, then ratchet | +| `P-COV-PROPERTY` | Fraction of invariants at status **C** in the traceability matrix ([MG-1](08-conformance.md#812-merge-gates)) | 0.77 (70 of 91) | ⚠ 0.69, then ratchet | | `P-COV-DIFF` | Line coverage of lines a change touches ([MG-2](08-conformance.md#812-merge-gates)) | unmeasured | ⚠ 0.80 | | `P-COV-TOTAL` | Whole-repository line coverage floor ([MG-2](08-conformance.md#812-merge-gates)) | unmeasured | ⚠ 0.70 | | `P-FLAKE-RETRIES` | Retries a test gets before it is quarantined ([MG-7](08-conformance.md#812-merge-gates)) | none | ⚠ 1 | diff --git a/spec/GAPS.md b/spec/GAPS.md index fb825d4..d85e97c 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -362,6 +362,9 @@ Permitted; must not change the meaning of anything above. [INV-O14](07-invariants.md#inv-o14). - `evm.traces.rewardValueNonZero` — present in both, listed here because it is easy to miss among the other three `*NonZero` flags. +- Sets (§2.7). The reference rejects the top-level `sets` key, so a request + that carries one goes only to an engine that serves it + ([ADR-17](decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md)). --- diff --git a/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md b/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md new file mode 100644 index 0000000..30a677a --- /dev/null +++ b/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md @@ -0,0 +1,111 @@ +# ADR-17 — Large value sets travel as fuse8 structures with canaries + +Status: Proposed + +## Context + +Some clients filter on more values than a list can carry. A DEX factory deploys a +contract for every pair it lists, and a client that indexes the pairs filters logs +on hundreds of thousands of addresses; the largest factories have millions. On +Solana the same shape is every token account a program owns. An `inList` stops at +`P-MAX-IN-LIST` values and `P-MAX-REQUEST-BYTES`, and an address costs about 45 +bytes of JSON, so one request carries a few tens of thousands at most. The +workarounds are worse than the problem: scan without the filter and drop rows on +the client, or split the list over many queries that each scan the same range +([#82](https://github.com/subsquid/sqd-query-engine/issues/82)). + +The portal sends the whole query with every chunk request, and recent chunks are +small. So whatever stands in for the list is paid for once per chunk, and its +size matters more than anything else about it. On a 224-block Ethereum chunk, a +structure over 400 000 values is four to eight times the size of the answer it +selects. The measurements behind this record are in +[#84](https://github.com/subsquid/sqd-query-engine/pull/84). + +## Decision + +A filter on a column that renders as text may list a **set** among its values +(§2.7). Sets are declared once per request, under a top-level `sets` key, and +filters refer to them by name. + +- **One format, `fuse8`**: a binary fuse filter with 8-bit fingerprints over the + XXH3-64 of each value's rendered text. The specification fixes the lookup, not + the construction (§4.2.11). The engine never builds a set, so any bytes the + lookup accepts for every value are a correct one. +- **Canaries are required.** A set carries a few of its values as the client + holds them, and the engine refuses the request when one of them is not a value + the column can render or is not in the structure. +- **The filtered field must be selected**, so that the client can see what it + has to remove. +- **False positives are the client's to remove**, as they are for the stored + bloom filters of [ADR-8](ADR-8-bloom-false-positives-are-contract.md). The + engine cannot remove them: it never sees the values. + +## Why these and not the alternatives + +**Fuse8, not a Bloom filter.** At the same rate of false positives, about one in +256, a fuse8 structure takes about 9 bits per value and a Bloom filter about 11.5. +The bound for any structure is log₂ 256 = 8 bits, so nothing else does much +better. Over 400 000 values the prototype measured 612 KB of base64 against +826 KB, and the Bloom filter's seeded hashes took two to three times as long per +row. A +Bloom filter can also be filled until it matches everything, which would need a +density rule. A fuse8 lookup cannot be made to pass every value, because part of +each fingerprint comes from hash bits that no index reads. + +**Fuse8, not Fuse16.** Fuse16 returned almost no false positives and made the +fastest requests, because a false positive costs more than its bytes: the worker +reads back the output columns of the row group it sits in. But it is twice the +size, and the size is paid on every chunk. + +**No cache of sets on the worker.** Sending a set to each worker once, and only +its digest after that, would make size cheap. It does not pay off. A factory's set +grows while the client discovers children, so the set and its digest change with +every rebuild. Each change misses on every worker, and each miss costs one more +round trip. + +**Named sets, not sets written inside the filter.** A set written inside a filter +travels once per filter that uses it. `from ∈ S ∨ to ∈ S` takes two item requests, +and both owners of a Solana token balance take two more. Inline, each would carry +its own copy; named, the bytes travel once. + +**References in the list, not in place of it.** A reference is one more element +of a filter's list, so a filter's value stays a list, and a client adds a value +to a set or joins two sets by listing them instead of building another. The +list passes what any of its parts passes. Its values can rule out no row group +that a set in the same list may still pass, so a list with a set costs what the +set costs. Each reference is a lookup for every row its table scans, and a set +costs a few dozen bytes to declare again under another name, so a request makes +at most `P-MAX-SET-REFERENCES` references. + +**Canaries, not trust.** Every way to build a set wrong — hashing a checksummed +address, dropping the `0x`, hashing the raw bytes instead of the text, another +hash or another seed — produces a structure that is accepted and matches almost +nothing. An almost empty answer looks like a quiet stretch of chain. Canaries turn +each of these mistakes into an error before any data is read. A wrong structure +passes a canary only by a false positive, so four of them let one through about +once in 4·10⁹ requests. + +## Consequences + +An engine that predates sets refuses a request carrying them, with `UnknownTable` +on the top-level key ([INV-Q2](../07-invariants.md#inv-q2)). It cannot answer as if +the filter were absent. The portal can route on the same key: a request with +`sets` goes only to an engine that supports them, and fails where none holds the +range. A real-time source that cannot evaluate a set has to refuse it in the same +way. + +Canaries miss two mistakes. A set on the wrong field, such as addresses on a topic +filter, passes them, because the canaries are well-formed values there too. And a +mistake that spoils only some values, say only the long ones, is caught only by a +canary among them, which is why a client should pick its canaries at random each +time it builds a set. + +The client builds the set, and rebuilds it whenever the set grows. A stream that +meets a new member at block B restarts at B with a set holding everything its last +batch found, so it rebuilds at most once a batch. The prototype's builder, +compiled to WASM, builds 400 000 values in about 26 ms under Node. A TypeScript +port takes about 87 ms, and is a second implementation to keep in step with the +lookup. + +`P-MAX-REQUEST-BYTES` still bounds a request, and fuse8 costs about 1.5 bytes of +base 64 per value, so one request carries a set of about 1.4 million values. diff --git a/src/error.rs b/src/error.rs index ac35423..e552286 100644 --- a/src/error.rs +++ b/src/error.rs @@ -45,8 +45,19 @@ pub enum ErrorKind { InvalidHex, /// A discriminator value exceeds `P-MAX-DISCRIMINATOR-BYTES`. DiscriminatorTooLong, - /// A filter value has a form the filter kind does not accept. + /// A filter value has a form the filter kind does not accept, such as a set + /// reference on a filter that takes no set. InvalidFilterValue, + /// A filter refers to a set that `sets` does not declare. + UnknownSet, + /// A declared set is malformed, unsound, has too few or too many canaries, + /// or no filter refers to it. + InvalidSet, + /// A set's canary is not a value the referring column can render, or the + /// set does not hold it. + SetCanaryMismatch, + /// A filter refers to a set on a column no selected field renders. + SetFieldNotSelected, /// The request exceeds `P-MAX-REQUEST-BYTES` or `P-MAX-IN-LIST`. RequestTooLarge, /// A reserved key the dataset cannot honour. @@ -83,6 +94,10 @@ impl ErrorKind { ErrorKind::InvalidHex => "InvalidHex", ErrorKind::DiscriminatorTooLong => "DiscriminatorTooLong", ErrorKind::InvalidFilterValue => "InvalidFilterValue", + ErrorKind::UnknownSet => "UnknownSet", + ErrorKind::InvalidSet => "InvalidSet", + ErrorKind::SetCanaryMismatch => "SetCanaryMismatch", + ErrorKind::SetFieldNotSelected => "SetFieldNotSelected", ErrorKind::RequestTooLarge => "RequestTooLarge", ErrorKind::UnsupportedRequestField => "UnsupportedRequestField", ErrorKind::TableNotFound => "TableNotFound", diff --git a/src/lib.rs b/src/lib.rs index 2a145c9..c3fdae7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod output; mod parallel; pub mod query; pub mod scan; +pub mod sets; pub mod text; #[cfg(test)] diff --git a/src/output/encoder.rs b/src/output/encoder.rs index 1f54b91..e9ce31d 100644 --- a/src/output/encoder.rs +++ b/src/output/encoder.rs @@ -417,7 +417,8 @@ fn encode_base58_fixed_binary(array: &dyn Array, row: usize, buf: &mut Vec) /// The Bitcoin alphabet, which is the one Solana uses. `0`, `O`, `I` and `l` are /// absent so no two characters look alike. -const BASE58_ALPHABET: &[u8; 58] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; +pub(crate) const BASE58_ALPHABET: &[u8; 58] = + b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; /// Base58 of a byte string, quoted. /// @@ -425,7 +426,7 @@ const BASE58_ALPHABET: &[u8; 58] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghij /// emitted as `1`s first — that is what makes the encoding injective, and what /// keeps a 32-byte key that starts with a zero byte 32 bytes long on the way /// back. -fn encode_base58_bytes(bytes: &[u8], buf: &mut Vec) { +pub(crate) fn encode_base58_bytes(bytes: &[u8], buf: &mut Vec) { buf.push(b'"'); let zeros = bytes.iter().take_while(|b| **b == 0).count(); @@ -801,7 +802,7 @@ fn needs_escape(bytes: &[u8]) -> bool { rest.iter().any(|&b| b < 0x20 || b == b'"' || b == b'\\') } -fn encode_hex_bytes(bytes: &[u8], buf: &mut Vec) { +pub(crate) fn encode_hex_bytes(bytes: &[u8], buf: &mut Vec) { buf.push(b'"'); buf.extend_from_slice(b"0x"); let hex_len = bytes.len() * 2; diff --git a/src/query/parse.rs b/src/query/parse.rs index 28139de..0601c23 100644 --- a/src/query/parse.rs +++ b/src/query/parse.rs @@ -1,5 +1,8 @@ use crate::error::ErrorKind; -use crate::metadata::{Alias, DatasetDescription, SpecialFilter, TableDescription}; +use crate::metadata::{ + Alias, DatasetDescription, SpecialFilter, TableDescription, RESERVED_REQUEST_KEYS, +}; +use crate::sets::{parse_sets, Sets}; use crate::{engine_bail, engine_ensure, engine_err}; use anyhow::Result; use std::collections::HashMap; @@ -18,6 +21,9 @@ pub struct Query { pub fields: HashMap>, /// Filter items per table: table_name → items. pub items: HashMap>, + /// The sets filters may refer to (§2.7), checked well-formed. That each + /// reference resolves, and each set has one, is checked by the plan. + pub sets: Sets, } /// A single filter item within a table request. @@ -92,15 +98,6 @@ const MAX_REQUEST_BYTES: usize = 2 * 1024 * 1024; /// under every other rule (INV-Q13). const MAX_IN_LIST: usize = 100_000; -const KNOWN_TOP_KEYS: &[&str] = &[ - "type", - "fromBlock", - "toBlock", - "includeAllBlocks", - "fields", - "parentBlockHash", -]; - /// A block bound is an unsigned 64-bit integer or absent. A present-but-malformed /// bound is an error rather than a default: coercing `{"fromBlock": "18000000"}` /// to genesis answers a different question than the one asked, and says nothing @@ -208,12 +205,13 @@ pub fn parse_query(json_bytes: &[u8], metadata: &DatasetDescription) -> Result> = HashMap::new(); for (key, value) in obj { - if KNOWN_TOP_KEYS.contains(&key.as_str()) { + if RESERVED_REQUEST_KEYS.contains(&key.as_str()) { continue; } @@ -274,6 +272,7 @@ pub fn parse_query(json_bytes: &[u8], metadata: &DatasetDescription) -> Result Result { .unwrap_or_default(); let mut table_plans = Vec::new(); + let mut referred = References::default(); // Catalog order, since the request's map has none: it decides which of two // failing tables a query reports, and that must not change between runs. @@ -158,11 +163,17 @@ pub fn compile(query: &Query, metadata: &DatasetDescription) -> Result { // the scan produces. let mut total_items = 0usize; + let scope = SetScope { + sets: &query.sets, + selected: query.fields.get(table_name).map_or(&[], Vec::as_slice), + }; + for item in items { - let item_predicate = match compile_item_predicates(item, table_desc)? { - CompiledItem::Unsatisfiable => continue, - CompiledItem::Predicate(predicate) => predicate, - }; + let item_predicate = + match compile_item_predicates(item, table_desc, &scope, &mut referred)? { + CompiledItem::Unsatisfiable => continue, + CompiledItem::Predicate(predicate) => predicate, + }; total_items += 1; let item_index = all_predicates.len(); all_predicates.push(item_predicate); @@ -232,6 +243,20 @@ pub fn compile(query: &Query, metadata: &DatasetDescription) -> Result { }); } + // A set nothing refers to is a reference the client meant to write and did + // not, and answering drops that filter without a word (INV-Q15). + if let Some(name) = query + .sets + .keys() + .find(|name| !referred.named.contains(*name)) + { + engine_bail!( + ErrorKind::InvalidSet, + "set '{}' is declared, and no filter refers to it", + name + ); + } + Ok(Plan { from_block: query.from_block, to_block: query.to_block, @@ -367,9 +392,51 @@ fn check_item_limits(item: &QueryItem, table: &TableDescription) -> Result<()> { Ok(()) } +/// The references a request's filters make to its sets. +#[derive(Default)] +struct References { + /// Every set a filter names. + named: HashSet, + /// The references, summed across every list, a name counted once per list. + count: usize, +} + +impl References { + /// Adds one list's references, refusing more than `P-MAX-SET-REFERENCES` in + /// all (INV-Q13). Each is a lookup for every row its table scans, however + /// little the set it names cost to send. + fn add(&mut self, key: &str, names: &IndexSet<&str>) -> Result<()> { + self.count += names.len(); + engine_ensure!( + self.count <= MAX_SET_REFERENCES, + ErrorKind::RequestTooLarge, + "filter '{}' brings the request to {} set references, at most {} allowed", + key, + self.count, + MAX_SET_REFERENCES + ); + + self.named.extend(names.iter().map(|name| name.to_string())); + Ok(()) + } +} + +/// What a set reference resolves against: the request's sets, and the fields +/// it selects from the referring item's table. +struct SetScope<'q> { + sets: &'q Sets, + selected: &'q [String], +} + /// Compile a single query item's filters into one RowPredicate. A discriminator -/// that dispatches to several column lengths becomes its alternatives. -fn compile_item_predicates(item: &QueryItem, table: &TableDescription) -> Result { +/// that dispatches to several column lengths becomes its alternatives. The sets +/// its filters refer to are added to `referred`, before they are checked. +fn compile_item_predicates( + item: &QueryItem, + table: &TableDescription, + scope: &SetScope, + referred: &mut References, +) -> Result { check_item_limits(item, table)?; let mut col_predicates: Vec = Vec::new(); @@ -377,6 +444,13 @@ fn compile_item_predicates(item: &QueryItem, table: &TableDescription) -> Result let mut unsatisfiable = false; for (key, value) in &item.filters { + let names = set_references(key, value)?; + if !names.is_empty() { + referred.add(key, &names)?; + col_predicates.push(compile_set_filter(key, value, &names, table, scope)?); + continue; + } + // `"c": []` is a filter no row passes, not an absent filter, and it // sinks the whole item whatever its other filters say (INV-P3). Reading // it as "unconstrained" would turn "none of these addresses" into @@ -440,11 +514,7 @@ fn compile_item_predicates(item: &QueryItem, table: &TableDescription) -> Result if let Some(bool_val) = boolean_filter { col_predicates.push(col_eq(column, ScalarValue::Boolean(bool_val))); } else if is_filter_scalar(value) || value.is_array() { - let values = match value.as_array() { - Some(arr) => arr, - None => std::slice::from_ref(value), - }; - match compile_in_list(column, values, col_desc)? { + match compile_in_list(column, list_of(value), col_desc)? { Some(predicate) => col_predicates.push(predicate), None => unsatisfiable = true, } @@ -515,11 +585,7 @@ fn compile_item_predicates(item: &QueryItem, table: &TableDescription) -> Result // forms cannot drift: the scalar branch used to compare a `Utf8` // against whatever the column was, which worked on a string column // and silently matched nothing on a binary one. - let values = match value.as_array() { - Some(arr) => arr, - None => std::slice::from_ref(value), - }; - match compile_in_list(key, values, col_desc)? { + match compile_in_list(key, list_of(value), col_desc)? { Some(predicate) => col_predicates.push(predicate), None => unsatisfiable = true, } @@ -551,6 +617,204 @@ fn compile_item_predicates(item: &QueryItem, table: &TableDescription) -> Result Ok(CompiledItem::Predicate(predicate)) } +/// A filter's list. A value written alone is a list of one. +fn list_of(value: &serde_json::Value) -> &[serde_json::Value] { + match value.as_array() { + Some(values) => values, + None => std::slice::from_ref(value), + } +} + +/// The sets a filter's list refers to, each once, in the order first named. A +/// reference is an element of the list, an object holding one key, +/// `{"set": ""}` (§2.7). No other element is an object. +fn set_references<'v>(key: &str, value: &'v serde_json::Value) -> Result> { + let mut names = IndexSet::new(); + + for element in list_of(value) { + let Some(object) = element.as_object() else { + continue; + }; + + let name = object.get("set").and_then(|name| name.as_str()); + match (object.len(), name) { + (1, Some(name)) => names.insert(name), + _ => engine_bail!( + ErrorKind::InvalidFilterValue, + "invalid filter value for '{}': an object is a set reference, \ + {{\"set\": \"\"}}, and this one is {}", + key, + element + ), + }; + } + + Ok(names) +} + +/// A filter whose list refers to sets, checked in order against the filter's +/// kind (INV-Q16), the declared sets (INV-Q15), the selected fields (INV-Q18) +/// and each set's canaries (INV-Q17). The list's values mean what they mean in +/// a list without references, and a row passes when it equals one of them or +/// one of the sets passes it. +fn compile_set_filter( + key: &str, + value: &serde_json::Value, + names: &IndexSet<&str>, + table: &TableDescription, + scope: &SetScope, +) -> Result { + let column = match table.request().special_filters.get(key) { + None => key, + Some(SpecialFilter::ColumnAlias { column }) => column.as_str(), + Some(_) => engine_bail!( + ErrorKind::InvalidFilterValue, + "filter '{}' takes no list of values, so it takes no set", + key + ), + }; + let col_desc = table.column(column).ok_or_else(|| { + engine_err!( + ErrorKind::UnknownFilter, + "column '{}' not found in table", + column + ) + })?; + + // The lookup hashes the text the response renders. A number, a quantity + // padded to its width or a list has no text a client could have hashed. + let renders_text = matches!( + (&col_desc.encoding, &col_desc.data_type), + (None, ColumnType::String) + | ( + Some(JsonEncoding::HexBytes | JsonEncoding::Base58), + ColumnType::String | ColumnType::FixedBinary(_) + ) + ); + engine_ensure!( + renders_text, + ErrorKind::InvalidFilterValue, + "filter '{}' refers to a set, and a set stands in a list of hexBytes, base58 or \ + utf8 text, which column '{}' does not render", + key, + column + ); + + let values: Vec = list_of(value) + .iter() + .filter(|element| !element.is_object()) + .cloned() + .collect(); + let listed = match values.is_empty() { + true => None, + false => compile_in_list(column, &values, col_desc)?, + }; + + let mut sets = Vec::with_capacity(names.len()); + for &name in names { + let set = scope.sets.get(name).ok_or_else(|| { + engine_err!( + ErrorKind::UnknownSet, + "filter '{}' refers to set '{}', which 'sets' does not declare", + key, + name + ) + })?; + sets.push(set); + } + + // A set passes rows that are not in it, and a client can remove only the + // ones it can see. + let renders = |field: &String| table.field_columns(field).any(|c| c == column); + if !scope.selected.iter().any(renders) { + let fields: Vec = table + .output + .fields + .iter() + .filter(|field| renders(field)) + .map(|field| snake_to_camel(field)) + .collect(); + let remedy = if fields.is_empty() { + "no field of the table renders it, so no set can filter on it".to_string() + } else { + format!( + "select {} to see, and remove, the rows the set passes that are not in it", + fields.join(" or ") + ) + }; + engine_bail!( + ErrorKind::SetFieldNotSelected, + "filter '{}' refers to a set, and no selected field renders column '{}': {}", + key, + column, + remedy + ); + } + + for (&name, set) in names.iter().zip(&sets) { + for canary in &set.canaries { + check_canary(canary, set, name, key, col_desc)?; + } + } + + let sets = sets.into_iter().map(Arc::clone).collect(); + let in_sets: Arc = + Arc::new(InSetPredicate::new(sets, col_desc.encoding.clone())); + let predicate = match listed { + Some(listed) => Arc::new(AnyOfPredicate::new(vec![listed.predicate, in_sets])), + None => in_sets, + }; + + Ok(ColumnPredicate { + column: column.to_string(), + predicate, + }) +} + +/// A canary, folded as the column folds a list value, must be a value the +/// column can render, and the set must hold it (INV-Q17). Every way to build a +/// set wrong is accepted otherwise and matches almost nothing, an answer that +/// looks like a quiet stretch of chain. +fn check_canary( + canary: &str, + set: &Set, + name: &str, + key: &str, + col_desc: &ColumnDescription, +) -> Result<()> { + let value = fold_for_column(canary, col_desc); + + let renderable = match col_desc.encoding { + Some(JsonEncoding::HexBytes) => is_hex_literal(&value), + Some(JsonEncoding::Base58) => { + !value.is_empty() && value.bytes().all(|b| BASE58_ALPHABET.contains(&b)) + } + _ => true, + }; + engine_ensure!( + renderable, + ErrorKind::SetCanaryMismatch, + "set '{}' on filter '{}': canary {:?} is no value the column can render", + name, + key, + canary + ); + + engine_ensure!( + set.fuse8.contains(value.as_bytes()), + ErrorKind::SetCanaryMismatch, + "set '{}' on filter '{}': canary {:?} is not in the set, so the set was built from \ + other text than the response renders, {:?} here: check case, the 0x prefix, the \ + hash and its seed", + name, + key, + canary, + value + ); + + Ok(()) +} + /// Whether a bare value is one a filter can carry: a string, or an integer of /// either sign. `is_u64` alone leaves out every negative, which is how a /// `{"version": -1}` on a signed column used to come back as "expected array, diff --git a/src/scan/predicate.rs b/src/scan/predicate.rs index 78bad92..244f841 100644 --- a/src/scan/predicate.rs +++ b/src/scan/predicate.rs @@ -1,6 +1,9 @@ use crate::integers::{ declared_value, slot_of, stored_key, width_of, IntColumn, IntValues, IntVisitor, INT_TYPES, }; +use crate::metadata::JsonEncoding; +use crate::output::{encode_base58_bytes, encode_hex_bytes}; +use crate::sets::Set; use crate::text::StringColumn; use arrow::array::*; use arrow::buffer::BooleanBuffer; @@ -352,6 +355,15 @@ pub struct BloomFilterPredicate { num_hashes: usize, } +/// Set predicate: the text the response renders for a value passes the lookup +/// of one of a list's sets (INV-P17). +pub struct InSetPredicate { + sets: Vec>, + /// How a column stored as bytes renders. A text column holds its rendering + /// already. + encoding: Option, +} + /// Range >= predicate: value >= threshold. pub struct RangeGtePredicate { value: ScalarValue, @@ -1001,6 +1013,166 @@ impl ArrayPredicate for BloomFilterPredicate { } } +impl InSetPredicate { + pub fn new(sets: Vec>, encoding: Option) -> Self { + Self { sets, encoding } + } + + fn contains(&self, text: &[u8]) -> bool { + self.sets.iter().any(|set| set.fuse8.contains(text)) + } + + /// A verdict per row, `false` for a null. A value equal to the previous + /// one takes its verdict: values repeat in runs beside the columns a table + /// is sorted by, not only in them, and a comparison costs less than a + /// lookup where they do not. + fn verdicts<'a>( + &self, + len: usize, + value: impl Fn(usize) -> Option<&'a [u8]>, + mut passes: impl FnMut(&[u8]) -> bool, + ) -> BooleanBuffer { + let mut builder = BooleanBufferBuilder::new(len); + let mut previous: Option<(&[u8], bool)> = None; + + for row in 0..len { + let verdict = match (value(row), previous) { + (None, _) => false, + (Some(value), Some((last, verdict))) if value == last => verdict, + (Some(value), _) => { + let verdict = passes(value); + previous = Some((value, verdict)); + verdict + } + }; + builder.append(verdict); + } + + builder.finish() + } +} + +impl ArrayPredicate for InSetPredicate { + fn evaluate(&self, array: &dyn Array) -> Mask { + if let Some(mask) = through_dictionary(self, array) { + return mask; + } + + if let Some(column) = StringColumn::resolve(array) { + let values = self.verdicts( + column.len(), + |row| column.value(row).map(str::as_bytes), + |text| self.contains(text), + ); + return Ok(BooleanArray::new(values, column.nulls())); + } + + // A column stored as bytes is looked up by the text it renders to, + // never by its bytes (§4.2.11). The renderers are the response's own, + // and only these two array types reach them there. + let render: fn(&[u8], &mut Vec) = match self.encoding { + Some(JsonEncoding::HexBytes) => encode_hex_bytes, + Some(JsonEncoding::Base58) => encode_base58_bytes, + _ => return Err(unsupported("a set", array)), + }; + let mut text = Vec::new(); + let mut passes = |bytes: &[u8]| { + text.clear(); + render(bytes, &mut text); + // Without the quotes the renderer writes around it. + self.contains(&text[1..text.len() - 1]) + }; + + macro_rules! rendered { + ($($array:ty),+ $(,)?) => { + $(if let Some(arr) = array.as_any().downcast_ref::<$array>() { + let values = self.verdicts( + arr.len(), + |row| (!arr.is_null(row)).then(|| arr.value(row)), + &mut passes, + ); + return Ok(BooleanArray::new(values, arr.nulls().cloned())); + })+ + }; + } + + rendered!(FixedSizeBinaryArray, BinaryArray); + Err(unsupported("a set", array)) + } + + /// A minimum and a maximum say nothing about which values a set holds + /// (INV-P16). + fn can_skip(&self, _stats: &StatRange) -> bool { + false + } + + fn describe(&self) -> String { + let mut sets: Vec = self + .sets + .iter() + .map(|set| { + let fuse8 = &set.fuse8; + format!( + "fuse8, {} fingerprints, xxh3 {:016x}", + fuse8.fingerprints(), + fuse8.digest() + ) + }) + .collect(); + sets.sort(); + format!("in_set({}; rendered {:?})", sets.join("; "), self.encoding) + } +} + +// --------------------------------------------------------------------------- +// AnyOfPredicate +// --------------------------------------------------------------------------- + +/// A row passes when any of `parts` passes it: a list that holds values and +/// set references beside each other (§4.2.11). +pub struct AnyOfPredicate { + parts: Vec>, +} + +impl AnyOfPredicate { + pub fn new(parts: Vec>) -> Self { + Self { parts } + } +} + +impl ArrayPredicate for AnyOfPredicate { + fn evaluate(&self, array: &dyn Array) -> Mask { + let mut any: Option = None; + for part in &self.parts { + let mask = part.evaluate(array)?; + any = Some(match any { + None => mask, + Some(prev) => or_kleene(&prev, &mask).unwrap(), + }); + } + + Ok(any.unwrap_or_else(|| all(array.len(), false))) + } + + /// Only a group that no part can match: a set never rules one out, so a + /// list with one prunes no more than the set alone. + fn can_skip(&self, stats: &StatRange) -> bool { + self.parts.iter().all(|part| part.can_skip(stats)) + } + + fn matches_every_value(&self, stats: &StatRange) -> bool { + self.parts + .iter() + .any(|part| part.matches_every_value(stats)) + } + + fn describe(&self) -> String { + let mut parts: Vec = self.parts.iter().map(|part| part.describe()).collect(); + parts.sort(); + format!("any_of({})", parts.join(" | ")) + } +} + // --------------------------------------------------------------------------- // Range predicates // --------------------------------------------------------------------------- @@ -3118,6 +3290,14 @@ mod tests { r#"contains_any(Utf8: "a", "b")"#, ), (NeverPredicate.describe(), "never"), + ( + AnyOfPredicate::new(vec![ + Arc::new(NeverPredicate), + Arc::new(InListPredicate::from_u8s(&[7])), + ]) + .describe(), + r#"any_of(in(UInt8: "7") | never)"#, + ), ]; for (text, expected) in &described { assert_eq!(text, expected); @@ -3130,6 +3310,14 @@ mod tests { ListContainsAnyPredicate::new_u32(vec![2, 3, 1]).describe(), ListContainsAnyPredicate::new_u32(vec![3, 1, 2]).describe() ); + assert_eq!( + AnyOfPredicate::new(vec![ + Arc::new(InListPredicate::from_u8s(&[7])), + Arc::new(NeverPredicate), + ]) + .describe(), + described.last().unwrap().1 + ); } /// A predicate written before `describe` existed still implements the @@ -3150,4 +3338,200 @@ mod tests { assert_eq!(Opaque.describe(), std::any::type_name::()); } + + /// The verdict for a row is the lookup of the text the response renders for + /// it: stored as text at any of its physical types, through a dictionary, or + /// as bytes rendered as hex or base58. Reusing a repeated value's verdict + /// changes no verdict, and a null passes nothing. + /// + /// Covers CT-3 · INV-P17 + #[test] + fn a_set_looks_up_the_text_the_response_renders() { + use crate::sets::{wire, Fuse8, Set}; + use xorf::{BinaryFuse8, Filter}; + use xxhash_rust::xxh3::xxh3_64; + + // Runs of three, so that a verdict is reused, and a null every seventh row. + let bytes: Vec> = (0..600u32) + .map(|row| (row % 7 != 0).then_some([(row / 3 % 200) as u8; 20])) + .collect(); + + type Render = fn(&[u8], &mut Vec); + let renderers: [(JsonEncoding, Render); 2] = [ + (JsonEncoding::HexBytes, encode_hex_bytes), + (JsonEncoding::Base58, encode_base58_bytes), + ]; + for (encoding, render) in renderers { + let text = |value: &[u8]| { + let mut quoted = Vec::new(); + render(value, &mut quoted); + String::from_utf8(quoted[1..quoted.len() - 1].to_vec()).unwrap() + }; + let texts: Vec> = bytes.iter().map(|b| b.map(|b| text(&b))).collect(); + + // Every other value is a member; the rest pass only by a false + // positive, which the builder's own lookup says where. + let members: Vec = (0..200u8).step_by(2).map(|v| text(&[v; 20])).collect(); + let keys: Vec = members.iter().map(|m| xxh3_64(m.as_bytes())).collect(); + let reference = BinaryFuse8::try_from(&keys).unwrap(); + let expected: Vec = texts + .iter() + .map(|t| { + t.as_ref() + .is_some_and(|t| reference.contains(&xxh3_64(t.as_bytes()))) + }) + .collect(); + assert!(expected.contains(&true) && expected.contains(&false)); + + let set = Arc::new(Set { + fuse8: Fuse8::from_bytes(&wire(&members)).unwrap(), + canaries: vec![members[0].clone()], + }); + + let strings: Vec> = texts.iter().map(|t| t.as_deref()).collect(); + let slices: Vec> = + bytes.iter().map(|b| b.as_ref().map(|b| &b[..])).collect(); + let arrays: Vec<(&str, ArrayRef)> = vec![ + ("utf8", Arc::new(StringArray::from(strings.clone()))), + ( + "large utf8", + Arc::new(LargeStringArray::from(strings.clone())), + ), + ( + "utf8 view", + Arc::new(StringViewArray::from(strings.clone())), + ), + ( + "a dictionary", + Arc::new( + strings + .iter() + .copied() + .collect::>(), + ), + ), + ( + "fixed binary", + Arc::new( + FixedSizeBinaryArray::try_from_sparse_iter_with_size( + slices.iter().copied(), + 20, + ) + .unwrap(), + ), + ), + ("binary", Arc::new(BinaryArray::from(slices.clone()))), + ]; + + let predicate = InSetPredicate::new(vec![Arc::clone(&set)], Some(encoding.clone())); + for (stored, array) in &arrays { + let mask = predicate.evaluate(array.as_ref()).unwrap(); + let verdicts: Vec = (0..mask.len()) + .map(|row| mask.is_valid(row) && mask.value(row)) + .collect(); + assert_eq!(verdicts, expected, "{encoding:?} stored as {stored}"); + } + } + + // Bytes a `utf8` column holds have no text a client hashed, and the + // response renders large binary as neither hex nor base58 by encoding. + let set = Arc::new(Set { + fuse8: Fuse8::from_bytes(&wire(&["0xab"])).unwrap(), + canaries: vec!["0xab".to_string()], + }); + let hex = Some(JsonEncoding::HexBytes); + let refused: [(Option, ArrayRef); 3] = [ + (None, Arc::new(BinaryArray::from(vec![Some(&[0xabu8][..])]))), + ( + hex.clone(), + Arc::new(LargeBinaryArray::from(vec![Some(&[0xabu8][..])])), + ), + (hex, Arc::new(Int64Array::from(vec![0xab]))), + ]; + for (encoding, array) in refused { + let predicate = InSetPredicate::new(vec![Arc::clone(&set)], encoding.clone()); + assert!( + predicate.evaluate(array.as_ref()).is_err(), + "{:?} rendered {encoding:?} was looked up", + array.data_type() + ); + } + } + + /// A list passes what any of its values and sets passes: each of two sets + /// contributes its own rows, and so does a value listed beside them. A null + /// passes nothing. A row group is ruled out only where every part rules it + /// out, which a set never does. + /// + /// Covers CT-3 · INV-P17 + #[test] + fn a_list_passes_what_any_of_its_values_and_sets_passes() { + use crate::sets::{wire, Fuse8, Set}; + + let set = |members: &[&str]| { + Arc::new(Set { + fuse8: Fuse8::from_bytes(&wire(members)).unwrap(), + canaries: vec![members[0].to_string()], + }) + }; + let a = set(&["alice", "bob"]); + let b = set(&["carol"]); + + let texts: Vec> = (0..300) + .map(|i| match i % 6 { + 0 => None, + 1 => Some("alice".to_string()), + 2 => Some("carol".to_string()), + 3 => Some("erin".to_string()), + _ => Some(format!("other {i}")), + }) + .collect(); + let array = StringArray::from(texts.clone()); + + let in_sets = || -> Arc { + Arc::new(InSetPredicate::new(vec![a.clone(), b.clone()], None)) + }; + let erin = + || -> Arc { Arc::new(InListPredicate::from_strings(&["erin"])) }; + let list = AnyOfPredicate::new(vec![erin(), in_sets()]); + + let verdicts = |predicate: &dyn ArrayPredicate| -> Vec { + let mask = predicate.evaluate(&array).unwrap(); + (0..mask.len()) + .map(|row| mask.is_valid(row) && mask.value(row)) + .collect() + }; + let by_lookup = + |text: &str| a.fuse8.contains(text.as_bytes()) || b.fuse8.contains(text.as_bytes()); + + let expected: Vec = texts + .iter() + .map(|t| t.as_deref().is_some_and(by_lookup)) + .collect(); + assert_eq!(verdicts(in_sets().as_ref()), expected); + assert!(expected[1] && expected[2], "a member of either set passes"); + + let expected: Vec = texts + .iter() + .map(|t| t.as_deref().is_some_and(|t| t == "erin" || by_lookup(t))) + .collect(); + assert_eq!(verdicts(&list), expected); + assert!(expected[3], "a listed value passes"); + assert!(!expected[0], "a null passes nothing"); + + let elsewhere = StatRange::Text { + min: "x".to_string(), + max: "z".to_string(), + }; + assert!(erin().can_skip(&elsewhere)); + assert!(!list.can_skip(&elsewhere)); + assert!(AnyOfPredicate::new(vec![erin(), erin()]).can_skip(&elsewhere)); + + let only_erin = StatRange::Text { + min: "erin".to_string(), + max: "erin".to_string(), + }; + assert!(list.matches_every_value(&only_erin)); + assert!(!AnyOfPredicate::new(vec![in_sets()]).matches_every_value(&only_erin)); + } } diff --git a/src/sets.rs b/src/sets.rs new file mode 100644 index 0000000..86651a5 --- /dev/null +++ b/src/sets.rs @@ -0,0 +1,396 @@ +//! Sets: filters on value sets too long to send as a list (spec §2.7, §4.2.11, +//! ADR-17). +//! +//! A client builds a `fuse8` structure over its values and declares it once, +//! under the request's `sets`; filters refer to it by name. The engine never +//! builds one. It only looks values up, so [`Fuse8::contains`] is the whole +//! contract between a client's builder and the engine: any bytes it passes +//! every value for are a correct set, however they were made. + +use crate::engine_err; +use crate::error::ErrorKind; +use anyhow::Result; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine; +use indexmap::IndexMap; +use std::sync::Arc; +use xxhash_rust::xxh3::xxh3_64; + +/// `P-MAX-SET-CANARIES` (spec/09-parameters.md §9.1). Each canary is a lookup +/// for every column that refers to its set, paid before any data is read. +pub const MAX_SET_CANARIES: usize = 64; + +/// `P-MAX-SET-REFERENCES` (spec/09-parameters.md §9.1). Each reference is a +/// lookup for every row its table scans, and a set costs a client little to +/// declare again under another name, so the references are counted, not the +/// sets. +pub const MAX_SET_REFERENCES: usize = 64; + +/// The request's declared sets, by name, in the order the request gives them. +pub type Sets = IndexMap>; + +/// One declared set. +#[derive(Debug)] +pub struct Set { + pub fuse8: Fuse8, + /// A few of the set's values as the client holds them, before the column + /// folds them (INV-Q17). + pub canaries: Vec, +} + +/// A binary fuse filter with one-byte fingerprints, checked sound: no value's +/// lookup reads past its fingerprints. +pub struct Fuse8 { + seed: u64, + /// `L`, `M` and `N` of §4.2.11, widened once so the lookup needs no casts. + segment_length: u64, + segment_mask: u64, + segment_count_length: u64, + fingerprints: Box<[u8]>, + /// XXH3 of the wire bytes, so diagnostics can tell two sets apart. + digest: u64, +} + +/// The header: a 64-bit seed, then `L`, `M` and `N` as 32-bit words, all +/// little-endian. +const HEADER_BYTES: usize = 20; + +/// The bound on `L`. Past it, `i1` would read the hash bits 36–39 the +/// fingerprint takes, and a crafted structure could pass every value. +const MAX_SEGMENT_LENGTH: u64 = 1 << 18; + +impl Fuse8 { + /// The structure the bytes hold, or why its lookup could read past them. + pub fn from_bytes(bytes: &[u8]) -> Result { + let Some((header, fingerprints)) = bytes.split_first_chunk::() else { + return Err(format!( + "{} bytes, shorter than its {HEADER_BYTES}-byte header", + bytes.len() + )); + }; + + let word = |at: usize| { + let word: [u8; 4] = header[at..at + 4].try_into().unwrap(); + u64::from(u32::from_le_bytes(word)) + }; + let seed = u64::from_le_bytes(header[..8].try_into().unwrap()); + let (length, mask, count_length) = (word(8), word(12), word(16)); + + let sound = length.is_power_of_two() + && length <= MAX_SEGMENT_LENGTH + && mask == length - 1 + && count_length > 0 + && count_length % length == 0 + && fingerprints.len() as u64 >= count_length + 2 * length; + if !sound { + return Err(format!( + "the header says L = {length}, M = {mask}, N = {count_length} over {} \ + fingerprints; a sound one has L a power of two up to 2^18, M = L - 1, N a \ + positive multiple of L, and at least N + 2L fingerprints", + fingerprints.len() + )); + } + + Ok(Fuse8 { + seed, + segment_length: length, + segment_mask: mask, + segment_count_length: count_length, + fingerprints: fingerprints.into(), + digest: xxh3_64(bytes), + }) + } + + /// Whether `value`, the text the response renders, passes the lookup of + /// §4.2.11. Every value the set was built from does; others pass at the + /// format's false-positive rate. + #[inline] + pub fn contains(&self, value: &[u8]) -> bool { + let hash = mix(xxh3_64(value).wrapping_add(self.seed)); + let fingerprint = (hash ^ (hash >> 32)) as u8; + + // Soundness bounds every index below N + 2L, and so below the length. + let i0 = ((u128::from(hash) * u128::from(self.segment_count_length)) >> 64) as u64; + let i1 = (i0 + self.segment_length) ^ ((hash >> 18) & self.segment_mask); + let i2 = (i0 + 2 * self.segment_length) ^ (hash & self.segment_mask); + + let f = &self.fingerprints; + fingerprint ^ f[i0 as usize] ^ f[i1 as usize] ^ f[i2 as usize] == 0 + } + + pub fn fingerprints(&self) -> usize { + self.fingerprints.len() + } + + pub fn digest(&self) -> u64 { + self.digest + } +} + +/// The fingerprints are hundreds of kilobytes, and no reader wants them. +impl std::fmt::Debug for Fuse8 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Fuse8") + .field("fingerprints", &self.fingerprints.len()) + .field("digest", &format_args!("{:016x}", self.digest)) + .finish() + } +} + +/// MurmurHash3's 64-bit finaliser. +#[inline] +fn mix(mut x: u64) -> u64 { + x ^= x >> 33; + x = x.wrapping_mul(0xff51afd7ed558ccd); + x ^= x >> 33; + x = x.wrapping_mul(0xc4ceb9fe1a85ec53); + x ^ (x >> 33) +} + +/// The request's `sets`, each checked well-formed (INV-Q15, INV-Q17). Whether +/// filters refer to them is the plan's to check, since only it resolves the +/// references. +pub fn parse_sets(value: Option<&serde_json::Value>) -> Result { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(Sets::new()); + }; + let declared = value.as_object().ok_or_else(|| { + engine_err!( + ErrorKind::MalformedRequest, + "'sets' must be an object, got {}", + value + ) + })?; + + declared + .iter() + .map(|(name, set)| { + let set = parse_set(set) + .map_err(|why| engine_err!(ErrorKind::InvalidSet, "set '{}': {}", name, why))?; + Ok((name.clone(), Arc::new(set))) + }) + .collect() +} + +fn parse_set(value: &serde_json::Value) -> Result { + let object = value + .as_object() + .ok_or_else(|| format!("expected an object holding 'fuse8' and 'canaries', got {value}"))?; + + // A key the set does not know may be a structure this engine predates, and + // guessing what it holds would answer without the filter. + if let Some(key) = object + .keys() + .find(|key| !matches!(key.as_str(), "fuse8" | "canaries")) + { + return Err(format!( + "unknown key '{key}': 'fuse8' is the only structure, beside 'canaries'" + )); + } + + let encoded = object + .get("fuse8") + .ok_or("no structure: expected 'fuse8'")? + .as_str() + .ok_or("'fuse8' must be a base 64 string")?; + let bytes = BASE64 + .decode(encoded) + .map_err(|e| format!("'fuse8' is not canonical base 64: {e}"))?; + let fuse8 = Fuse8::from_bytes(&bytes).map_err(|why| format!("'fuse8': {why}"))?; + + let canaries: Vec = object + .get("canaries") + .and_then(|canaries| canaries.as_array()) + .and_then(|canaries| { + canaries + .iter() + .map(|canary| canary.as_str().map(str::to_owned)) + .collect() + }) + .ok_or("'canaries' must be an array of strings: a few of the set's values")?; + if !(1..=MAX_SET_CANARIES).contains(&canaries.len()) { + return Err(format!( + "{} canaries; a set carries at least one and at most {MAX_SET_CANARIES}", + canaries.len() + )); + } + + Ok(Set { fuse8, canaries }) +} + +/// The wire form of a set over `values`, built by `xorf`, the builder the +/// lookup was written against. +#[cfg(test)] +pub(crate) fn wire>(values: &[S]) -> Vec { + let keys: Vec = values + .iter() + .map(|v| xxh3_64(v.as_ref().as_bytes())) + .collect(); + let filter = xorf::BinaryFuse8::try_from(&keys).unwrap(); + + let mut bytes = filter.descriptor.seed.to_le_bytes().to_vec(); + bytes.extend(filter.descriptor.segment_length.to_le_bytes()); + bytes.extend(filter.descriptor.segment_length_mask.to_le_bytes()); + bytes.extend(filter.descriptor.segment_count_length.to_le_bytes()); + bytes.extend(filter.fingerprints.iter()); + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + use xorf::{BinaryFuse8, Filter}; + + fn header(length: u32, mask: u32, count_length: u32, fingerprints: usize) -> Vec { + let mut bytes = 7u64.to_le_bytes().to_vec(); + bytes.extend(length.to_le_bytes()); + bytes.extend(mask.to_le_bytes()); + bytes.extend(count_length.to_le_bytes()); + bytes.resize(HEADER_BYTES + fingerprints, 0); + bytes + } + + fn address(i: u64) -> String { + format!("0x{:040x}", i.wrapping_mul(0x9e3779b97f4a7c15)) + } + + /// The lookup agrees with `xorf`'s on members and non-members alike, over + /// sizes that give `xorf` different segment lengths. + /// + /// Covers CT-3 · INV-P17 + #[test] + fn the_lookup_agrees_with_the_builders_own() { + for n in [1u64, 2, 3, 10, 100, 1_000, 10_000, 100_000] { + let values: Vec = (0..n).map(address).collect(); + let refs: Vec<&str> = values.iter().map(String::as_str).collect(); + let keys: Vec = values.iter().map(|v| xxh3_64(v.as_bytes())).collect(); + let reference = BinaryFuse8::try_from(&keys).unwrap(); + let set = Fuse8::from_bytes(&wire(&refs)).unwrap(); + + for value in &values { + assert!( + set.contains(value.as_bytes()), + "{n} values: {value} missing" + ); + } + + let mut passed = 0; + for i in n..n + 20_000 { + let other = address(i); + let ours = set.contains(other.as_bytes()); + assert_eq!(ours, reference.contains(&xxh3_64(other.as_bytes()))); + passed += usize::from(ours); + } + // About one in 256; far more means the lookup reads the wrong bytes. + assert!(passed < 300, "{n} values: {passed} of 20 000 others passed"); + } + } + + /// Each condition of §4.2.11's soundness, broken alone. + /// + /// Covers CT-2 · INV-Q15 + #[test] + fn an_unsound_header_is_refused() { + let sound = header(4, 3, 8, 16); + assert!(Fuse8::from_bytes(&sound).is_ok()); + + let cases = [ + ("short", sound[..HEADER_BYTES - 1].to_vec()), + ("L not a power of two", header(3, 2, 9, 15)), + ("L zero", header(0, u32::MAX, 8, 16)), + ( + "L over 2^18", + header(1 << 19, (1 << 19) - 1, 1 << 19, 3 << 19), + ), + ("M not L - 1", header(4, 7, 8, 16)), + ("N zero", header(4, 3, 0, 8)), + ("N not a multiple of L", header(4, 3, 6, 14)), + ("too few fingerprints", header(4, 3, 8, 15)), + ]; + for (what, bytes) in cases { + assert!(Fuse8::from_bytes(&bytes).is_err(), "{what} was accepted"); + } + + let largest = header(1 << 18, (1 << 18) - 1, 1 << 18, 3 << 18); + assert!(Fuse8::from_bytes(&largest).is_ok()); + } + + /// The largest index a sound header allows is the last fingerprint, so a + /// lookup never reads past it whatever the hash. + /// + /// Covers CT-2 · INV-Q15 + #[test] + fn a_sound_header_keeps_every_lookup_inside() { + for (length, count_length) in [(1, 1), (1, 5), (4, 4), (4, 64), (1 << 18, 1 << 18)] { + let set = Fuse8::from_bytes(&header( + length, + length - 1, + count_length, + (count_length + 2 * length) as usize, + )) + .unwrap(); + for i in 0..20_000u64 { + set.contains(&i.to_le_bytes()); + } + } + } + + fn parse(sets: serde_json::Value) -> Result { + parse_sets(Some(&sets)) + } + + fn kind(result: Result) -> Option { + crate::error::error_kind(&result.expect_err("the sets must be refused")) + } + + #[test] + fn a_set_is_well_formed() { + let fuse8 = BASE64.encode(wire(&["0xab", "0xcd"])); + let set = |set: serde_json::Value| serde_json::json!({ "s": set }); + + let parsed = parse(set( + serde_json::json!({"fuse8": fuse8, "canaries": ["0xab"]}), + )) + .unwrap(); + assert_eq!(parsed["s"].canaries, ["0xab"]); + assert!(parse_sets(None).unwrap().is_empty()); + assert!(parse_sets(Some(&serde_json::Value::Null)) + .unwrap() + .is_empty()); + + let unpadded = fuse8.trim_end_matches('='); + let invalid = [ + serde_json::json!("not an object"), + serde_json::json!({"canaries": ["0xab"]}), + serde_json::json!({"fuse16": fuse8, "canaries": ["0xab"]}), + serde_json::json!({"fuse8": fuse8, "fuse16": fuse8, "canaries": ["0xab"]}), + serde_json::json!({"fuse8": 1, "canaries": ["0xab"]}), + serde_json::json!({"fuse8": format!("{fuse8}\n"), "canaries": ["0xab"]}), + serde_json::json!({"fuse8": BASE64.encode(header(4, 3, 8, 15)), "canaries": ["0xab"]}), + serde_json::json!({"fuse8": fuse8}), + serde_json::json!({"fuse8": fuse8, "canaries": []}), + serde_json::json!({"fuse8": fuse8, "canaries": "0xab"}), + serde_json::json!({"fuse8": fuse8, "canaries": [1]}), + serde_json::json!({"fuse8": fuse8, "canaries": vec!["0xab"; MAX_SET_CANARIES + 1]}), + ]; + for value in invalid { + assert_eq!( + kind(parse(set(value.clone()))), + Some(ErrorKind::InvalidSet), + "{value}" + ); + } + + // Unpadded only differs from canonical where the length is not a + // multiple of three. + if unpadded != fuse8 { + let value = serde_json::json!({"fuse8": unpadded, "canaries": ["0xab"]}); + assert_eq!(kind(parse(set(value))), Some(ErrorKind::InvalidSet)); + } + + assert_eq!( + kind(parse(serde_json::json!([]))), + Some(ErrorKind::MalformedRequest) + ); + } +} diff --git a/tests/conformance/ct2_request/bounds.rs b/tests/conformance/ct2_request/bounds.rs index 8a71bf7..5307e09 100644 --- a/tests/conformance/ct2_request/bounds.rs +++ b/tests/conformance/ct2_request/bounds.rs @@ -1,9 +1,13 @@ //! The bounds cost, not correctness, motivates: a request the engine will answer //! must be one it can afford to answer. +use serde_json::{json, Map, Value}; +use sqd_query_engine::error::{error_kind, ErrorKind}; use sqd_query_engine::query::{compile, parse_query}; +use sqd_query_engine::sets::MAX_SET_REFERENCES; use crate::harness::fixtures::meta; +use crate::harness::sets::BuiltSet; /// Each bloom value is a separate hash-and-probe over every row, so the length /// of the list is a cost multiplier the client picks. `P-MAX-BLOOM-VALUES` caps @@ -130,3 +134,75 @@ fn a_request_is_bounded_in_bytes() { .to_string(); assert!(err.contains("bytes"), "got: {err}"); } + +/// Each set reference is a lookup for every row its table scans, and a set +/// declared under many names is many lookups. `P-MAX-SET-REFERENCES` caps the +/// references a request makes, summed across every list of every item request, +/// a name counted once per list. +/// +/// Covers CT-2 · INV-Q13 +#[test] +fn set_references_are_bounded_in_number() { + let evm = meta("evm"); + + // Every set is the same structure under its own name, which is what makes + // this cheap to send. + let value = "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc"; + let set = BuiltSet::of(&[value]).declared(&[value]); + let reference = |i: usize| json!({ "set": format!("s{i}") }); + let outcome = |lists: Vec>| { + let declared: Map = lists + .iter() + .flatten() + .filter_map(|r| r["set"].as_str()) + .map(|name| (name.to_owned(), set.clone())) + .collect(); + let items: Vec = lists + .into_iter() + .map(|list| json!({ "address": list })) + .collect(); + let query = json!({ + "type": "evm", + "sets": declared, + "fields": { "log": { "address": true } }, + "logs": items, + }) + .to_string(); + + let err = parse_query(query.as_bytes(), &evm) + .and_then(|parsed| compile(&parsed, &evm)) + .err()?; + Some(error_kind(&err).unwrap_or_else(|| panic!("refused without a kind: {err:#}"))) + }; + + let one_list = |count: usize| vec![(0..count).map(reference).collect()]; + assert_eq!(outcome(one_list(MAX_SET_REFERENCES)), None); + assert_eq!( + outcome(one_list(MAX_SET_REFERENCES + 1)), + Some(ErrorKind::RequestTooLarge) + ); + + // Two references per item, so the count runs across items. + let spread = |count: usize| { + (0..count) + .step_by(2) + .map(|i| (i..count.min(i + 2)).map(reference).collect()) + .collect() + }; + assert_eq!(outcome(spread(MAX_SET_REFERENCES)), None); + assert_eq!( + outcome(spread(MAX_SET_REFERENCES + 1)), + Some(ErrorKind::RequestTooLarge) + ); + + // One set, referred to from more items than the cap: each is a lookup. + let one_set_everywhere = vec![vec![reference(0)]; MAX_SET_REFERENCES + 1]; + assert_eq!( + outcome(one_set_everywhere), + Some(ErrorKind::RequestTooLarge) + ); + + // A name written again in one list is one lookup. + let repeated = vec![vec![reference(0); 10 * MAX_SET_REFERENCES]]; + assert_eq!(outcome(repeated), None); +} diff --git a/tests/conformance/ct2_request/mod.rs b/tests/conformance/ct2_request/mod.rs index 4e7c8f3..37bba07 100644 --- a/tests/conformance/ct2_request/mod.rs +++ b/tests/conformance/ct2_request/mod.rs @@ -9,5 +9,6 @@ mod bounds; mod fields; mod fork; mod kinds; +mod sets; mod surface; mod values; diff --git a/tests/conformance/ct2_request/sets.rs b/tests/conformance/ct2_request/sets.rs new file mode 100644 index 0000000..5216e9d --- /dev/null +++ b/tests/conformance/ct2_request/sets.rs @@ -0,0 +1,476 @@ +//! Sets (§2.7): declared, referred to, and vouched for by their canaries, all +//! before any data is read. + +use serde_json::{json, Value}; +use sqd_query_engine::error::{error_kind, ErrorKind}; +use sqd_query_engine::metadata::DatasetDescription; +use sqd_query_engine::query::{compile, parse_query}; +use sqd_query_engine::sets::MAX_SET_CANARIES; +use xxhash_rust::xxh3::{xxh3_64, xxh3_64_with_seed}; + +use crate::harness::fixtures::{bundled_catalogs, meta}; +use crate::harness::sets::{every_filter, BuiltSet, Rendering}; + +/// What validating `query` comes to: `None` when it compiles, the error's kind +/// otherwise. +fn outcome(catalog: &DatasetDescription, query: &Value) -> Option { + let query = query.to_string(); + let err = parse_query(query.as_bytes(), catalog) + .and_then(|parsed| compile(&parsed, catalog)) + .err()?; + + Some(error_kind(&err).unwrap_or_else(|| panic!("refused without a kind: {err:#}"))) +} + +/// A log request filtered on a set of addresses, the address selected. +fn logs_on(set: Value) -> Value { + json!({ + "type": "evm", + "sets": { "s": set }, + "fields": { "log": { "address": true } }, + "logs": [{ "address": { "set": "s" } }], + }) +} + +const ADDRESSES: [&str; 4] = [ + "0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc", + "0x0d4a11d5EEaaC28EC3F61d100daF4d40471f1852", + "0xA478c2975Ab1Ea89e8196811F51A7B7Ade33eB11", + "0xBb2b8038a1640196FbE3e38816F3e67Cba72D940", +]; + +fn lowercase() -> Vec { + ADDRESSES.iter().map(|a| a.to_lowercase()).collect() +} + +/// A reference names a declared set, a declared set has a reference, and each +/// set holds exactly one structure, `fuse8`, in canonical base 64 and sound. +/// +/// Covers CT-2 · INV-Q15 +#[test] +fn a_set_is_declared_used_and_well_formed() { + let evm = meta("evm"); + let set = BuiltSet::of(&lowercase()); + let fuse8 = set.wire(); + let canaries = ["0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc"]; + + assert_eq!(outcome(&evm, &logs_on(set.declared(&canaries))), None); + + let undeclared = json!({ + "type": "evm", + "fields": { "log": { "address": true } }, + "logs": [{ "address": { "set": "s" } }], + }); + assert_eq!(outcome(&evm, &undeclared), Some(ErrorKind::UnknownSet)); + + let unused = json!({ "type": "evm", "sets": { "unused": set.declared(&canaries) } }); + assert_eq!(outcome(&evm, &unused), Some(ErrorKind::InvalidSet)); + + let one_of_two_unused = json!({ + "type": "evm", + "sets": { "s": set.declared(&canaries), "t": set.declared(&canaries) }, + "fields": { "log": { "address": true } }, + "logs": [{ "address": { "set": "s" } }], + }); + assert_eq!( + outcome(&evm, &one_of_two_unused), + Some(ErrorKind::InvalidSet) + ); + + // A reference in a list, beside values and other references, is a + // reference, and so is one named twice. + let in_a_list = |second: &str| { + json!({ + "type": "evm", + "sets": { "s": set.declared(&canaries), "t": set.declared(&canaries) }, + "fields": { "log": { "address": true } }, + "logs": [{ + "address": [ADDRESSES[1], { "set": "s" }, { "set": second }, { "set": "s" }], + }], + }) + }; + assert_eq!(outcome(&evm, &in_a_list("t")), None); + assert_eq!(outcome(&evm, &in_a_list("u")), Some(ErrorKind::UnknownSet)); + + // A reference in an item that matches nothing is still a reference, and + // still checked. + let beside_an_empty_list = json!({ + "type": "evm", + "sets": { "s": set.declared(&canaries) }, + "fields": { "log": { "address": true } }, + "logs": [{ "address": { "set": "s" }, "topic0": [] }], + }); + assert_eq!(outcome(&evm, &beside_an_empty_list), None); + + let mut short = base64_bytes(&fuse8); + short.pop(); + let unpadded = fuse8.trim_end_matches('=').to_string(); + let malformed = [ + ("no structure", json!({ "canaries": canaries })), + ( + "two structures", + json!({ "fuse8": fuse8, "fuse16": fuse8, "canaries": canaries }), + ), + ( + "an unknown structure", + json!({ "fuse16": fuse8, "canaries": canaries }), + ), + ( + "an unknown key", + json!({ "fuse8": fuse8, "canaries": canaries, "x": 1 }), + ), + ( + "a number for bytes", + json!({ "fuse8": 1, "canaries": canaries }), + ), + ( + "base 64 with a line break", + json!({ "fuse8": format!("{fuse8}\n"), "canaries": canaries }), + ), + ( + "fingerprints short of N + 2L", + json!({ "fuse8": base64_encode(&short), "canaries": canaries }), + ), + ("not an object", json!(fuse8)), + ]; + for (what, declared) in malformed { + assert_eq!( + outcome(&evm, &logs_on(declared)), + Some(ErrorKind::InvalidSet), + "a set with {what}" + ); + } + + // Padding is only missing where the length is not a multiple of three. + if unpadded != fuse8 { + let declared = json!({ "fuse8": unpadded, "canaries": canaries }); + assert_eq!( + outcome(&evm, &logs_on(declared)), + Some(ErrorKind::InvalidSet), + "unpadded base 64" + ); + } + + let not_an_object = json!({ "type": "evm", "sets": [] }); + assert_eq!( + outcome(&evm, ¬_an_object), + Some(ErrorKind::MalformedRequest) + ); +} + +fn base64_bytes(text: &str) -> Vec { + use base64::Engine; + base64::engine::general_purpose::STANDARD + .decode(text) + .unwrap() +} + +fn base64_encode(bytes: &[u8]) -> String { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +/// A set reference is accepted on exactly the filters that take a list of +/// `hexBytes`, `base58` or `utf8` text, on every filter of every bundled +/// catalog, tables and aliases alike, written alone or in a list beside a +/// value. Elsewhere it is `InvalidFilterValue`, and so is an object that is not +/// a reference. +/// +/// Covers CT-2 · INV-Q16, INV-Q18 +#[test] +fn a_set_stands_only_in_a_list_of_text() { + let mut accepted = 0; + let mut refused = 0; + + for catalog in bundled_catalogs() { + for filter in every_filter(&catalog) { + let rendering = filter.rendering.unwrap_or(Rendering::Utf8); + let sample = rendering.sample(); + let selected: serde_json::Map = filter + .fields + .iter() + .map(|field| (field.clone(), json!(true))) + .collect(); + + let expected = match (filter.rendering, filter.fields.is_empty()) { + (None, _) => Some(ErrorKind::InvalidFilterValue), + (Some(_), true) => Some(ErrorKind::SetFieldNotSelected), + (Some(_), false) => None, + }; + + for value in [json!({ "set": "s" }), json!([sample, { "set": "s" }])] { + let query = json!({ + "type": catalog.name, + "sets": { "s": BuiltSet::of(&[sample]).declared(&[sample]) }, + "fields": { (filter.output.clone()): selected }, + (filter.request.clone()): [{ (filter.key.clone()): value }], + }); + assert_eq!( + outcome(&catalog, &query), + expected, + "{}: {} on {}.{}", + catalog.name, + value, + filter.request, + filter.key + ); + } + + if expected.is_none() { + accepted += 1; + } else { + refused += 1; + } + } + } + + // EVM alone has seventeen hex filters, and every catalog some that take no + // set; far fewer means the walk lost a catalog. + assert!(accepted > 50, "only {accepted} filters took a set"); + assert!(refused > 20, "only {refused} filters refused one"); + + // The cases the invariant names, so the walk's own reading of the catalog + // is not the only witness. + let evm = meta("evm"); + let solana = meta("solana"); + let hex = Rendering::Hex.sample(); + let set = BuiltSet::of(&[hex]).declared(&[hex]); + + let from = json!({ + "type": "evm", + "sets": { "s": set }, + "fields": { "transaction": { "from": true } }, + "transactions": [{ "from": { "set": "s" } }], + }); + assert_eq!(outcome(&evm, &from), None); + + let nonce = json!({ + "type": "evm", + "sets": { "s": set }, + "fields": { "transaction": { "nonce": true } }, + "transactions": [{ "firstNonce": { "set": "s" } }], + }); + assert_eq!(outcome(&evm, &nonce), Some(ErrorKind::InvalidFilterValue)); + + let discriminator = json!({ + "type": "solana", + "sets": { "s": set }, + "fields": { "instruction": { "data": true } }, + "instructions": [{ "discriminator": { "set": "s" } }], + }); + assert_eq!( + outcome(&solana, &discriminator), + Some(ErrorKind::InvalidFilterValue) + ); + + for reference in [ + json!({}), + json!({ "set": 1 }), + json!({ "set": "s", "canaries": [hex] }), + json!({ "name": "s" }), + ] { + for value in [reference.clone(), json!([hex, { "set": "s" }, reference])] { + let query = json!({ + "type": "evm", + "sets": { "s": set }, + "fields": { "log": { "address": true } }, + "logs": [{ "address": { "set": "s" } }, { "address": value }], + }); + assert_eq!( + outcome(&evm, &query), + Some(ErrorKind::InvalidFilterValue), + "{value} as a filter value" + ); + } + } +} + +/// Every way to build a set wrong is refused by its canaries, and the right set +/// passes them however the client cases them. +/// +/// Covers CT-2 · INV-Q17 +#[test] +fn a_sets_canaries_are_values_it_holds() { + let evm = meta("evm"); + let solana = meta("solana"); + let lowercase = lowercase(); + + // The canaries are the values as the client holds them, checksummed. + let wrong: [(&str, BuiltSet); 5] = [ + ("checksummed text", BuiltSet::of(&ADDRESSES)), + ( + "text without 0x", + BuiltSet::of(&lowercase.iter().map(|a| &a[2..]).collect::>()), + ), + ( + "the raw bytes", + BuiltSet::of_keys(lowercase.iter().map(|a| { + let bytes: Vec = (2..a.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&a[i..i + 2], 16).unwrap()) + .collect(); + xxh3_64(&bytes) + })), + ), + ( + "another seed", + BuiltSet::of_keys(lowercase.iter().map(|a| xxh3_64_with_seed(a.as_bytes(), 1))), + ), + ( + "another hash", + BuiltSet::of_keys(lowercase.iter().map(|a| { + use std::hash::{Hash, Hasher}; + let mut hasher = std::hash::DefaultHasher::new(); + a.hash(&mut hasher); + hasher.finish() + })), + ), + ]; + for (how, set) in wrong { + assert_eq!( + outcome(&evm, &logs_on(set.declared(&ADDRESSES))), + Some(ErrorKind::SetCanaryMismatch), + "a set built from {how}" + ); + } + + let right = BuiltSet::of(&lowercase); + let upper_cased: Vec = ADDRESSES.iter().map(|a| a.to_uppercase()).collect(); + assert_eq!(outcome(&evm, &logs_on(right.declared(&ADDRESSES))), None); + assert_eq!(outcome(&evm, &logs_on(right.declared(&upper_cased))), None); + + // One bad canary among good ones is enough, and a canary must be a value the + // column can hold even where the structure happens to hold it. + // The non-member is one the builder's own lookup rejects: a false positive + // passes any canary check, about once in 256. + let not_a_member = (1..=9) + .map(|digit| format!("0x{}", digit.to_string().repeat(40))) + .find(|value| !right.passes(value)) + .unwrap(); + let not_a_member = [ADDRESSES[0], not_a_member.as_str()]; + assert_eq!( + outcome(&evm, &logs_on(right.declared(¬_a_member))), + Some(ErrorKind::SetCanaryMismatch) + ); + let odd = "0xb4e16d0168e52d35cacd2c6185b44281ec28c9d"; + assert_eq!( + outcome(&evm, &logs_on(BuiltSet::of(&[odd]).declared(&[odd]))), + Some(ErrorKind::SetCanaryMismatch), + "a hex canary with an odd number of digits" + ); + + let hex = Rendering::Hex.sample(); + let hex_on_base58 = json!({ + "type": "solana", + "sets": { "s": BuiltSet::of(&[hex]).declared(&[hex]) }, + "fields": { "instruction": { "programId": true } }, + "instructions": [{ "programId": { "set": "s" } }], + }); + assert_eq!( + outcome(&solana, &hex_on_base58), + Some(ErrorKind::SetCanaryMismatch) + ); + + // The canaries are checked for every column that refers to the set, each + // folding them its own way: an upper-cased address passes on a hex column + // and is not the text a `utf8` column holds. Four of them, so that a false + // positive does not decide it. + let two_columns = json!({ + "type": "evm", + "sets": { "s": right.declared(&upper_cased) }, + "fields": { "log": { "address": true }, "trace": { "type": true } }, + "logs": [{ "address": { "set": "s" } }], + "traces": [{ "type": { "set": "s" } }], + }); + assert_eq!( + outcome(&evm, &two_columns), + Some(ErrorKind::SetCanaryMismatch) + ); + + // Every set a list refers to is checked. + let one_wrong_of_two = json!({ + "type": "evm", + "sets": { + "right": right.declared(&ADDRESSES), + "wrong": BuiltSet::of(&ADDRESSES).declared(&ADDRESSES), + }, + "fields": { "log": { "address": true } }, + "logs": [{ "address": [{ "set": "right" }, { "set": "wrong" }] }], + }); + assert_eq!( + outcome(&evm, &one_wrong_of_two), + Some(ErrorKind::SetCanaryMismatch) + ); + + let too_many = vec![ADDRESSES[0]; MAX_SET_CANARIES + 1]; + for canaries in [Vec::new(), too_many] { + assert_eq!( + outcome(&evm, &logs_on(right.declared(&canaries))), + Some(ErrorKind::InvalidSet), + "{} canaries", + canaries.len() + ); + } + let most = vec![ADDRESSES[0]; MAX_SET_CANARIES]; + assert_eq!(outcome(&evm, &logs_on(right.declared(&most))), None); +} + +/// A set-filtered column is selected, directly or through the field that rolls +/// it, since the client has to see the rows the set passes that are not in it. +/// +/// Covers CT-2 · INV-Q18 +#[test] +fn a_set_filtered_column_reaches_the_response() { + let evm = meta("evm"); + let hex = Rendering::Hex.sample(); + let set = BuiltSet::of(&[hex]).declared(&[hex]); + + let query = |key: &str, fields: Value| { + json!({ + "type": "evm", + "sets": { "s": set }, + "fields": fields, + "logs": [{ (key): { "set": "s" } }], + }) + }; + + let cases = [ + ("address", json!({ "log": { "address": true } }), None), + ("address", json!({}), Some(ErrorKind::SetFieldNotSelected)), + ( + "address", + json!({ "log": { "address": false, "data": true } }), + Some(ErrorKind::SetFieldNotSelected), + ), + ( + "address", + json!({ "transaction": { "to": true } }), + Some(ErrorKind::SetFieldNotSelected), + ), + ("topic0", json!({ "log": { "topics": true } }), None), + ( + "topic0", + json!({ "log": { "address": true } }), + Some(ErrorKind::SetFieldNotSelected), + ), + ]; + for (key, fields, expected) in cases { + assert_eq!( + outcome(&evm, &query(key, fields.clone())), + expected, + "a set on logs.{key} with fields {fields}" + ); + } + + // A value listed beside the reference does not make the rows visible. + let beside_a_value = json!({ + "type": "evm", + "sets": { "s": set }, + "fields": {}, + "logs": [{ "address": [hex, { "set": "s" }] }], + }); + assert_eq!( + outcome(&evm, &beside_a_value), + Some(ErrorKind::SetFieldNotSelected) + ); +} diff --git a/tests/conformance/ct3_filter_algebra/mod.rs b/tests/conformance/ct3_filter_algebra/mod.rs index 28adf79..6563d6a 100644 --- a/tests/conformance/ct3_filter_algebra/mod.rs +++ b/tests/conformance/ct3_filter_algebra/mod.rs @@ -14,5 +14,6 @@ mod gte_const; mod items; mod laws; mod pruning; +mod sets; mod surface; mod values; diff --git a/tests/conformance/ct3_filter_algebra/sets.rs b/tests/conformance/ct3_filter_algebra/sets.rs new file mode 100644 index 0000000..b2826d1 --- /dev/null +++ b/tests/conformance/ct3_filter_algebra/sets.rs @@ -0,0 +1,328 @@ +//! A set passes exactly the rows its lookup passes (§4.2.11), on every filter +//! that takes one, against the rows a fixture chunk holds. + +use serde_json::{json, Map, Value}; +use sqd_query_engine::metadata::DatasetDescription; +use std::collections::{BTreeMap, BTreeSet}; + +use crate::harness::fixtures::{fixture_tree_has, meta, run_page, FIXTURE_DATASETS}; +use crate::harness::json::{items_in, items_of, parse_response}; +use crate::harness::sets::{every_filter, BuiltSet, CatalogFilter}; + +/// The rows of one request key on one fixture chunk, with every field that +/// renders a column a set may filter on, cut to about twenty thousand. +struct Rows<'a> { + dataset: &'a str, + catalog: &'a DatasetDescription, + request: &'a str, + response_key: String, + fields: Value, + /// The block every query is cut at. + last: u64, + every_row: Vec<(u64, String)>, + /// Every string the rows carry. A filtered column's text is among them, + /// wherever the field that renders it nests it. + seen: BTreeSet, +} + +impl<'a> Rows<'a> { + fn read( + dataset: &'a str, + catalog: &'a DatasetDescription, + request: &'a str, + filters: &[&CatalogFilter], + ) -> Rows<'a> { + let first = filters[0]; + let table = catalog.table(&first.table).unwrap(); + let selected: Map = filters + .iter() + .flat_map(|filter| &filter.fields) + .map(|field| (field.clone(), json!(true))) + .collect(); + + let mut rows = Rows { + dataset, + catalog, + request, + response_key: table.request_name(&first.table).to_string(), + fields: json!({ "block": { "number": true } }), + last: 0, + every_row: Vec::new(), + seen: BTreeSet::new(), + }; + + // Counted first with no field selected, which costs little. A query + // cut at the same block holds fewer rows, so one page of it reaches + // that block too. + let (counts, page_end) = rows.page(json!({}), json!({}), None); + let mut counted = 0; + rows.last = parse_response(&counts) + .iter() + .find_map(|block| { + counted += items_in(block, &rows.response_key).len(); + (counted >= 20_000).then(|| block["header"]["number"].as_u64().unwrap()) + }) + .unwrap_or(page_end); + + rows.fields = json!({ "block": { "number": true }, (first.output.clone()): selected }); + let every_row = rows.answer(json!({}), json!({})); + for (_, item) in items_of(&every_row, &rows.response_key) { + strings(&item, &mut rows.seen); + } + rows.every_row = sorted(&every_row, &rows.response_key); + + rows + } + + /// One page of the request with a single item, and the block it ends at. + fn page(&self, item: Value, sets: Value, to: Option) -> (Vec, u64) { + let query = json!({ + "type": self.catalog.name, + "fromBlock": 0, + "toBlock": to, + "sets": sets, + "fields": self.fields, + (self.request): [item], + }); + + run_page(self.dataset, self.catalog, query.to_string().as_bytes()) + .unwrap_or_else(|e| panic!("{}: {}: {e:#}", self.dataset, self.request)) + .unwrap_or_else(|| panic!("{}: the range misses the chunk", self.dataset)) + } + + /// The rows up to the cut, in one page. + fn answer(&self, item: Value, sets: Value) -> Vec { + let (body, ends_at) = self.page(item, sets, Some(self.last)); + assert_eq!( + ends_at, self.last, + "{}: {}: a narrower query stopped short", + self.dataset, self.request + ); + body + } +} + +/// Every string anywhere in `value`. +fn strings(value: &Value, found: &mut BTreeSet) { + match value { + Value::String(text) => { + found.insert(text.clone()); + } + Value::Array(values) => values.iter().for_each(|v| strings(v, found)), + Value::Object(members) => members.values().for_each(|v| strings(v, found)), + _ => {} + } +} + +/// The items under `key`, sorted, with their blocks. A multiset rather than a +/// set: the query selects little more than the filtered fields, so two rows of +/// one block often render alike, and a set would count them once. +fn sorted(body: &[u8], key: &str) -> Vec<(u64, String)> { + let mut rows: Vec<(u64, String)> = items_of(body, key) + .into_iter() + .map(|(block, item)| (block, item.to_string())) + .collect(); + rows.sort(); + rows +} + +/// Whether every row of `part` is in `whole` as many times, both sorted. +fn contains_all(whole: &[(u64, String)], part: &[(u64, String)]) -> bool { + let mut whole = whole.iter().peekable(); + + part.iter().all(|row| { + while whole.next_if(|candidate| *candidate < row).is_some() {} + whole.next_if(|candidate| *candidate == row).is_some() + }) +} + +/// Fails at the first row where the answers by a set and by its lookup differ. +fn same_rows(what: &str, by_set: &[(u64, String)], by_lookup: &[(u64, String)]) { + let differs = + (0..by_set.len().max(by_lookup.len())).find(|&i| by_set.get(i) != by_lookup.get(i)); + + if let Some(at) = differs { + panic!( + "{what}: {} rows by the set, {} by the lookup; first difference at row {at}: \ + {:?} by the set, {:?} by the lookup", + by_set.len(), + by_lookup.len(), + by_set.get(at), + by_lookup.get(at) + ); + } +} + +/// What one comparison saw: whether the set left any row out, and how many rows +/// it passed beyond its members'. +struct Compared { + narrowed: bool, + extra_rows: usize, +} + +/// A set on `filter` against a list of every value the builder's own lookup +/// passes; `None` where the rows hold no value to build one from. +fn compare(rows: &Rows, filter: &CatalogFilter) -> Option { + let table = rows.catalog.table(&filter.table).unwrap(); + let column = table.column(filter.column.as_ref().unwrap()).unwrap(); + let rendering = filter.rendering.unwrap(); + let what = format!( + "{}: a set on {}.{}", + rows.dataset, filter.request, filter.key + ); + + let seen: BTreeSet = rows + .seen + .iter() + .map(|text| match column.folds_case() { + true => text.to_ascii_lowercase(), + false => text.clone(), + }) + .collect(); + + // Half the values, or fewer where so many would not fit in a request as the + // list the set is compared against. + let candidates: Vec<&String> = seen.iter().filter(|text| rendering.holds(text)).collect(); + let step = candidates.len().div_ceil(10_000).max(2); + let members: Vec<&String> = candidates.iter().step_by(step).copied().collect(); + if members.is_empty() { + return None; + } + + let set = BuiltSet::of(&members); + let canaries: Vec<&String> = members + .iter() + .step_by(members.len().div_ceil(4)) + .copied() + .collect(); + let passed: Vec<&String> = seen.iter().filter(|text| set.passes(text)).collect(); + + let key = filter.key.clone(); + let by_set = sorted( + &rows.answer( + json!({ (key.clone()): { "set": "s" } }), + json!({ "s": set.declared(&canaries) }), + ), + &rows.response_key, + ); + let by_lookup = sorted( + &rows.answer(json!({ (key.clone()): passed }), json!({})), + &rows.response_key, + ); + let by_members = sorted( + &rows.answer(json!({ (key): members }), json!({})), + &rows.response_key, + ); + + same_rows(&what, &by_set, &by_lookup); + assert!( + contains_all(&by_set, &by_members), + "{what} lost rows of its members" + ); + + // The same members as a list of values and two sets beside them, which + // passes the values and what either set's lookup passes. + if members.len() >= 3 { + let third = members.len() / 3; + let (listed, rest) = members.split_at(third); + let (in_a, in_b) = rest.split_at(third); + let (a, b) = (BuiltSet::of(in_a), BuiltSet::of(in_b)); + + let mut list: Vec = listed.iter().map(|value| json!(value)).collect(); + list.extend([json!({ "set": "a" }), json!({ "set": "b" })]); + let sets = json!({ "a": a.declared(&in_a[..1]), "b": b.declared(&in_b[..1]) }); + let by_list = sorted( + &rows.answer(json!({ (filter.key.clone()): list }), sets), + &rows.response_key, + ); + + let listed: BTreeSet<&String> = listed.iter().copied().collect(); + let passed: Vec<&String> = seen + .iter() + .filter(|text| listed.contains(text) || a.passes(text) || b.passes(text)) + .collect(); + let by_lookup = sorted( + &rows.answer(json!({ (filter.key.clone()): passed }), json!({})), + &rows.response_key, + ); + + same_rows( + &format!("{what}, listed beside values"), + &by_list, + &by_lookup, + ); + assert!( + contains_all(&by_list, &by_members), + "{what}, listed beside values, lost rows of its members" + ); + } + + Some(Compared { + narrowed: by_set.len() < rows.every_row.len(), + extra_rows: by_set.len() - by_members.len(), + }) +} + +/// For every filter of every fixture catalog that takes a set, a set is built +/// from half the values its column renders, or ten thousand of them. The answer +/// must equal the answer to a list of every value the builder's own lookup +/// passes, which is the rows the lookup passes: a list holds each row it names, +/// and no other. Since that list holds the members, the answer also holds the +/// answer to the members as a list. +/// +/// The members are sent again as a list of a third of them and two sets of the +/// rest, and the answer must equal the answer to the listed values and every +/// value either set's lookup passes. +/// +/// The list is drawn from every string the table's rows carry, a superset of the +/// column's values: a string the column does not hold matches nothing. +/// +/// Covers CT-3 · INV-P17 +#[test] +#[ignore = "requires external fixture data"] +fn a_set_passes_exactly_what_its_lookup_passes() { + let mut compared = 0; + let mut narrowed = 0; + let mut extra_rows = 0; + + for (dataset, name) in FIXTURE_DATASETS { + if !fixture_tree_has(dataset) { + continue; + } + let catalog = meta(name); + + // By the request key they are sent under, so that its rows are read once. + let filters = every_filter(&catalog); + let mut by_request: BTreeMap<&str, Vec<&CatalogFilter>> = BTreeMap::new(); + for filter in &filters { + if filter.rendering.is_some() && !filter.fields.is_empty() { + by_request.entry(&filter.request).or_default().push(filter); + } + } + + for (request, filters) in &by_request { + let rows = Rows::read(dataset, &catalog, request, filters); + for filter in filters { + let Some(seen) = compare(&rows, filter) else { + continue; + }; + compared += 1; + narrowed += usize::from(seen.narrowed); + extra_rows += seen.extra_rows; + } + } + } + + if fixture_tree_has("ethereum") { + assert!(compared > 20, "only {compared} filters were compared"); + // A set that passes every row cannot tell an engine that drops rows from + // one that does not; a column of one value makes such a set. + assert!( + narrowed * 2 > compared, + "only {narrowed} of {compared} sets left any row out" + ); + // Nor can the equality tell an engine that drops false positives from + // one that keeps them, unless some set passed one. + assert!(extra_rows > 0, "no set passed a row outside its members"); + } +} diff --git a/tests/conformance/harness/fixtures.rs b/tests/conformance/harness/fixtures.rs index 440f0bf..00943c7 100644 --- a/tests/conformance/harness/fixtures.rs +++ b/tests/conformance/harness/fixtures.rs @@ -57,6 +57,26 @@ pub fn run( run_against_bytes(metadata, &fixture_chunk(dataset), query_json) } +/// One page of a query against a bundled dataset's fixture chunk, and the block +/// the page ends at; `None` when the range misses the chunk. A page ends where +/// its weight runs out, so a query that matches much covers less of the chunk +/// than one that matches little. +pub fn run_page( + dataset: &str, + metadata: &DatasetDescription, + query: &[u8], +) -> anyhow::Result, u64)>> { + let parsed = parse_query(query, metadata)?; + let plan = compile(&parsed, metadata)?; + + Ok( + execute_plan(&plan, metadata, &fixture_chunk(dataset))?.map(|output| { + let last = output.last_block(); + (output.into_json_lines(), last) + }), + ) +} + /// Run a query against a chunk named directly — a synthetic one, or a fixture /// chunk rewritten for the test. pub fn run_against( diff --git a/tests/conformance/harness/mod.rs b/tests/conformance/harness/mod.rs index 11a89b2..45759fa 100644 --- a/tests/conformance/harness/mod.rs +++ b/tests/conformance/harness/mod.rs @@ -16,6 +16,7 @@ pub mod generator; pub mod guard; pub mod hex; pub mod json; +pub mod sets; pub mod sol_like; pub mod store; pub mod synthetic; diff --git a/tests/conformance/harness/sets.rs b/tests/conformance/harness/sets.rs new file mode 100644 index 0000000..8b2a297 --- /dev/null +++ b/tests/conformance/harness/sets.rs @@ -0,0 +1,184 @@ +//! The filters a set may stand in for, and sets built the way a client builds +//! them. +//! +//! [INV-Q16] lets a set stand only in a list of text, so a test of sets +//! reaches only as far as the filters it tries. Reading them off the catalog +//! keeps a filter added later inside every such test. +//! +//! [INV-Q16]: ../../../spec/07-invariants.md#inv-q16 + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine; +use sqd_query_engine::metadata::{ColumnType, DatasetDescription, JsonEncoding, SpecialFilter}; +use sqd_query_engine::output::snake_to_camel; +use xorf::{BinaryFuse8, Filter}; +use xxhash_rust::xxh3::xxh3_64; + +/// Every filter of a catalog, under the request key a client sends it in. +pub struct CatalogFilter { + /// A table's request name, or an alias over it. + pub request: String, + /// The filter key, camelCased the way a client writes it. + pub key: String, + /// The catalog's name for the table it reads. + pub table: String, + /// The text a set on it is looked up by, or `None` when it takes no set. + pub rendering: Option, + /// The column it reads, for a filter that takes a list of values. + pub column: Option, + /// The output name of its table, the key of a selection under `fields`. + pub output: String, + /// The fields that render its column, camelCased. + pub fields: Vec, +} + +/// The text a column renders, which is what a set is looked up by (§4.2.11). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Rendering { + Hex, + Base58, + Utf8, +} + +impl Rendering { + /// A value of this rendering, as a client would hold it. + pub fn sample(self) -> &'static str { + match self { + Rendering::Hex => "0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc", + Rendering::Base58 => "So11111111111111111111111111111111111111112", + Rendering::Utf8 => "transfer", + } + } + + /// Whether `value` is one a column of this rendering can hold. + pub fn holds(self, value: &str) -> bool { + const BASE58: &[u8] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + + match self { + Rendering::Hex => value.strip_prefix("0x").is_some_and(|digits| { + digits.len() % 2 == 0 && digits.bytes().all(|b| b.is_ascii_hexdigit()) + }), + Rendering::Base58 => !value.is_empty() && value.bytes().all(|b| BASE58.contains(&b)), + Rendering::Utf8 => true, + } + } +} + +/// Every filter of `catalog`, on its tables and on the aliases over them. +pub fn every_filter(catalog: &DatasetDescription) -> Vec { + let mut found = Vec::new(); + + for (name, table) in &catalog.tables { + let surface = table.request(); + for key in &surface.filters { + let column = match surface.special_filters.get(key) { + None => Some(key), + Some(SpecialFilter::ColumnAlias { column }) => Some(column), + Some(_) => None, + }; + found.push(filter(catalog, table.request_name(name), key, name, column)); + } + } + + for (request, alias) in &catalog.aliases { + for key in &alias.filters { + let column = match alias.special_filters.get(key) { + None => Some(key), + Some(SpecialFilter::ColumnAlias { column }) => Some(column), + Some(_) => None, + }; + found.push(filter(catalog, request, key, &alias.table, column)); + } + } + + found +} + +fn filter( + catalog: &DatasetDescription, + request: &str, + key: &str, + table: &str, + column: Option<&String>, +) -> CatalogFilter { + let description = catalog.table(table).unwrap(); + let declared = column.and_then(|column| description.column(column)); + + let rendering = declared.and_then(|c| match (&c.encoding, &c.data_type) { + (None, ColumnType::String) => Some(Rendering::Utf8), + (Some(JsonEncoding::HexBytes), ColumnType::String | ColumnType::FixedBinary(_)) => { + Some(Rendering::Hex) + } + (Some(JsonEncoding::Base58), ColumnType::String | ColumnType::FixedBinary(_)) => { + Some(Rendering::Base58) + } + _ => None, + }); + + let fields = match column { + Some(column) => description + .output + .fields + .iter() + .filter(|field| description.field_columns(field).any(|c| c == column)) + .map(|field| snake_to_camel(field)) + .collect(), + None => Vec::new(), + }; + + CatalogFilter { + request: request.to_string(), + key: snake_to_camel(key), + table: table.to_string(), + rendering, + column: column.cloned(), + output: description.output.name.clone().unwrap_or_default(), + fields, + } +} + +/// A set as a client builds it: a binary fuse filter over the XXH3 of each +/// value's text, by the builder the lookup was written against. +pub struct BuiltSet { + filter: BinaryFuse8, +} + +impl BuiltSet { + pub fn of>(values: &[S]) -> BuiltSet { + BuiltSet::of_keys(values.iter().map(|v| xxh3_64(v.as_ref().as_bytes()))) + } + + /// Over keys made some other way, as a client that builds a set wrong does. + pub fn of_keys(keys: impl IntoIterator) -> BuiltSet { + let mut keys: Vec = keys.into_iter().collect(); + keys.sort_unstable(); + keys.dedup(); + + BuiltSet { + filter: BinaryFuse8::try_from(&keys).unwrap(), + } + } + + /// The `fuse8` value a request carries. + pub fn wire(&self) -> String { + let descriptor = &self.filter.descriptor; + let mut bytes = descriptor.seed.to_le_bytes().to_vec(); + bytes.extend(descriptor.segment_length.to_le_bytes()); + bytes.extend(descriptor.segment_length_mask.to_le_bytes()); + bytes.extend(descriptor.segment_count_length.to_le_bytes()); + bytes.extend(self.filter.fingerprints.iter()); + + BASE64.encode(bytes) + } + + /// The set as it is declared under `sets`. + pub fn declared>(&self, canaries: &[S]) -> serde_json::Value { + let canaries: Vec<&str> = canaries.iter().map(AsRef::as_ref).collect(); + serde_json::json!({ "fuse8": self.wire(), "canaries": canaries }) + } + + /// The builder's own lookup of `value`'s text, independent of the engine's. + pub fn passes(&self, value: &str) -> bool { + self.filter.contains(&xxh3_64(value.as_bytes())) + } +} From 30621139f4cc6ebfc969ae300cbcb8ea69b312b1 Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Fri, 23 Oct 2026 18:04:08 +0300 Subject: [PATCH 2/2] feat(query): make the request bound a 16 MiB backstop The 2 MiB came from the reference's HTTP server, not from the engine. The portal's admission and the network's 4 MiB message are the bounds a deployment works under, and a set is bounded by nothing but its bytes, so the engine's bound now sits above them, where neither has to wait for an engine release. Parsing the worst request, short list values, takes up to 452 MiB at 16 MiB. Co-Authored-By: Claude Opus 5.5 (1M context) --- spec/09-parameters.md | 9 +++--- spec/GAPS.md | 2 +- .../ADR-13-request-resource-bounds.md | 29 +++++++++++++++++++ ...alue-sets-travel-as-fuse8-with-canaries.md | 5 +++- src/query/parse.rs | 6 ++-- tests/conformance/ct2_request/bounds.rs | 10 +++---- 6 files changed, 48 insertions(+), 13 deletions(-) diff --git a/spec/09-parameters.md b/spec/09-parameters.md index cd963d5..7a6c5ba 100644 --- a/spec/09-parameters.md +++ b/spec/09-parameters.md @@ -27,7 +27,7 @@ Checked before any chunk data is read ([INV-E2](07-invariants.md#inv-e2)). | `P-MAX-BLOOM-VALUES` | Values in one bloom filter ([INV-Q10](07-invariants.md#inv-q10)) | 10 | 10 | | `P-MAX-DISCRIMINATOR-FILTERS` | Discriminator-family filters in one item request ([INV-Q11](07-invariants.md#inv-q11)) | 1 | 1 | | `P-MAX-DISCRIMINATOR-BYTES` | Bytes in one discriminator value ([INV-Q12](07-invariants.md#inv-q12)) | 16 | 16 | -| `P-MAX-REQUEST-BYTES` | Whole-request size bound ([INV-Q13](07-invariants.md#inv-q13)) | 2 MiB | 2 MiB | +| `P-MAX-REQUEST-BYTES` | Whole-request size bound ([INV-Q13](07-invariants.md#inv-q13)) | 16 MiB | 16 MiB | | `P-MAX-IN-LIST` | Values in one `inList` ([INV-Q13](07-invariants.md#inv-q13)) | 100 000 | 100 000 | | `P-MAX-SET-CANARIES` | Canaries in one set ([INV-Q17](07-invariants.md#inv-q17)) | 64 | ⚠ 64 | | `P-MAX-SET-REFERENCES` | Set references summed across every list of every item request, a name counted once per list ([INV-Q13](07-invariants.md#inv-q13)) | 64 | ⚠ 64 | @@ -38,9 +38,10 @@ a differently-named discriminator is bounded by the same rule with no code chang ([INV-X1](07-invariants.md#inv-x1)). `P-MAX-REQUEST-BYTES` bounds parsing work and total request representation, and -it is the only bound on a set's size (§2.7). `P-MAX-IN-LIST` independently bounds the -set built for one filter, including the case where short values allow a large -list to fit below the byte cap ([ADR-13](decisions/ADR-13-request-resource-bounds.md)). +it is the only bound on a set's size (§2.7). It is a backstop above the bounds a +deployment works under, the portal's admission and the network's 4 MiB message. +`P-MAX-IN-LIST` independently bounds the set built for one filter, including the case where short values allow a large list to fit below the +byte cap ([ADR-13](decisions/ADR-13-request-resource-bounds.md)). `P-MAX-SET-CANARIES` bounds the work a set's canaries add to validation: each is one lookup, for every column that refers to the set. It is a proposal of diff --git a/spec/GAPS.md b/spec/GAPS.md index d85e97c..7dc78e4 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -329,7 +329,7 @@ worker answers. | Behaviour | Reference | Spec | Reason | |---|---|---|---| -| Request bounds | No byte or list-length bound in the engine; the transport admits a little over 4 MiB | `P-MAX-REQUEST-BYTES` and `P-MAX-IN-LIST`, refused as `RequestTooLarge` ([ADR-13](decisions/ADR-13-request-resource-bounds.md)) | The engine states and enforces its own resource contract. *Terminal* for a body between the two bounds; the largest request on record is 1.13 MiB with 21 821 addresses, so the margin is under 2× on bytes. | +| Request bounds | No byte or list-length bound in the engine; the transport admits a little over 4 MiB | `P-MAX-REQUEST-BYTES` and `P-MAX-IN-LIST`, refused as `RequestTooLarge` ([ADR-13](decisions/ADR-13-request-resource-bounds.md)) | The engine states and enforces its own resource contract. The byte bound sits above the transport's, so no request the transport carries meets it; the list bound is *terminal* above 100 000 values, and the largest request on record holds 21 821. | | `null` as a filter value | Absent: `"address": null` is no constraint | `InvalidFilterValue` | Every filter value has a form the kind accepts and `null` is not one. *Terminal*; also present in earlier revisions of this engine. | | `hyperliquidReplicaCmds` aliases | `orderActions`, `cancelActions`, `cancelByCloidActions`, `batchModifyActions` add no `actionType` predicate — `orderActions: [{user: [U]}]` returns every action type for `U` | Each alias carries an implicit `actionType` predicate ([§3.8](03-catalog.md#38-hyperliquidreplicacmds)) | The alias name promises orders; the reference does not keep the promise. **Silent:** 7 092 rows there, 3 856 here for the same request. The reference also reads `asset`/`cloid` columns the parquet does not have for `cancelByCloidActions.containsAsset`/`containsCloid`, so those filters error there and answer here. | | Item-request cap counting | `hyperliquidReplicaCmds` counts only the `actions` array; its four aliases are uncounted | Count every item request, uniformly ([INV-Q5](07-invariants.md#inv-q5)) | An uncounted alias is an unbounded scan. *Terminal* above a hundred items through aliases. | diff --git a/spec/decisions/ADR-13-request-resource-bounds.md b/spec/decisions/ADR-13-request-resource-bounds.md index 21bd1e0..3125331 100644 --- a/spec/decisions/ADR-13-request-resource-bounds.md +++ b/spec/decisions/ADR-13-request-resource-bounds.md @@ -34,3 +34,32 @@ match the target values. These limits are compatibility parameters rather than protocol constants. A future change must update the parameter table and record the rationale in this ADR. + +## Revision: 16 MiB + +`P-MAX-REQUEST-BYTES` is now **16 MiB**, a backstop rather than the bound a +deployment works under. It was raised with sets +([ADR-17](ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md)). + +The 2 MiB came from the reference's transport, not from anything the engine +needs: it was the default body limit of the reference's HTTP server. Two bounds +now sit in front of the engine. The portal admits what it decides to, and the +network carries a query of up to 4 MiB and a kibibyte of envelope to a worker, +refusing a larger one before it is sent. An engine bound below them refuses +requests the rest of the path accepts, and one equal to them makes every change +to them wait for an engine release. So the engine's bound sits well above them, +and stops only a body no deployment sends. + +Sets are why that matters. A set is bounded by nothing but the bytes that carry +it, about 1.5 bytes of base 64 per value: the network's 4 MiB carries about 2.8 +million values. + +What a bound this size admits was measured on one core, parsing and compiling +the worst request, one of short list values. At 4 MiB it took up to 113 MiB of +memory and 70 ms, at 16 MiB up to 452 MiB and 0.3 s. A set the same size took 9 +and 36 MiB. The cost of lists is the parsed JSON tree, about 72 bytes a value, +so an engine embedded where no transport bounds the request should have one +that does. + +`P-MAX-IN-LIST` is unchanged. It bounds the hash set a list builds, and a set +builds none. diff --git a/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md b/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md index 30a677a..c2e3c76 100644 --- a/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md +++ b/spec/decisions/ADR-17-large-value-sets-travel-as-fuse8-with-canaries.md @@ -108,4 +108,7 @@ port takes about 87 ms, and is a second implementation to keep in step with the lookup. `P-MAX-REQUEST-BYTES` still bounds a request, and fuse8 costs about 1.5 bytes of -base 64 per value, so one request carries a set of about 1.4 million values. +base 64 per value. With this record the bound becomes a 16 MiB backstop +([ADR-13](ADR-13-request-resource-bounds.md)). What bounds a set in practice is +the network's 4 MiB message, about 2.8 million values, and the portal decides how +much of it a client may use. diff --git a/src/query/parse.rs b/src/query/parse.rs index 0601c23..ffab81c 100644 --- a/src/query/parse.rs +++ b/src/query/parse.rs @@ -89,8 +89,10 @@ const MAX_ITEM_REQUESTS: usize = 100; /// `P-MAX-REQUEST-BYTES` (spec/09-parameters.md §9.1). Checked against the raw /// body, before it is parsed: the parse itself is the first thing a large -/// request makes the engine pay for. -const MAX_REQUEST_BYTES: usize = 2 * 1024 * 1024; +/// request makes the engine pay for. A backstop: the portal's admission and the +/// network's 4 MiB message are the bounds a deployment works under, and this +/// one sits above them so that either can rise without an engine release. +const MAX_REQUEST_BYTES: usize = 16 * 1024 * 1024; /// `P-MAX-IN-LIST` (spec/09-parameters.md §9.1). A filter list becomes a hash set /// built before a single row is read, and the request cap alone does not bound diff --git a/tests/conformance/ct2_request/bounds.rs b/tests/conformance/ct2_request/bounds.rs index 5307e09..f59758e 100644 --- a/tests/conformance/ct2_request/bounds.rs +++ b/tests/conformance/ct2_request/bounds.rs @@ -98,7 +98,7 @@ fn an_in_list_is_bounded_in_length() { }; let at_the_cap = query(100_000); - assert!(at_the_cap.len() < 2 * 1024 * 1024); + assert!(at_the_cap.len() < 16 * 1024 * 1024); parse_query(&at_the_cap, &evm).expect("a list at the cap must be accepted"); let err = parse_query(&query(100_001), &evm) @@ -123,12 +123,12 @@ fn a_request_is_bounded_in_bytes() { .into_bytes() }; - let under = query(700_000); - assert!(under.len() < 2 * 1024 * 1024); + let under = query(8_000_000); + assert!(under.len() < 16 * 1024 * 1024); parse_query(&under, &evm).expect("a request under the cap must be parsed"); - let over = query(1_200_000); - assert!(over.len() > 2 * 1024 * 1024); + let over = query(8_500_000); + assert!(over.len() > 16 * 1024 * 1024); let err = parse_query(&over, &evm) .expect_err("a request over the cap must be refused") .to_string();