From f640b3877facf61868a00221194d67438215e85b Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Fri, 16 Oct 2026 01:35:59 +0300 Subject: [PATCH] Move sqd-network to main with the dial-failure lookup fixes 11f4230 is a commit of the closed sqd-network#216 branch. Its retries stop after a lookup that cannot find a worker that is still offline, and it lacks the open_stream hang fixes from sqd-network#217. main vendors libp2p-stream, so the patch for the libp2p fork goes away. The shared DataChunk parser in sqd-messages 3.1.0 reads everything after the second dash as the hash, so the router now strips /blocks.parquet from an object key before it parses the chunk id. Bump pings-collector to 2.9.1, logs-collector to 2.5.1 and portal-logs-collector to 1.1.7. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 236 +++++++++++++++------ Cargo.toml | 12 +- crates/collector-utils/src/storage.rs | 1 + crates/logs-collector/Cargo.toml | 2 +- crates/pings-collector/Cargo.toml | 2 +- crates/portal-logs-collector/Cargo.toml | 2 +- crates/router-controller/src/controller.rs | 13 +- crates/router/src/dataset.rs | 51 ++++- 8 files changed, 229 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ce8d2d..262b600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,18 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if 1.0.5", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -2153,6 +2165,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -2231,6 +2255,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2558,6 +2588,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2566,7 +2605,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -2584,6 +2623,15 @@ dependencies = [ "fxhash", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "hashlink" version = "0.10.0" @@ -3357,10 +3405,17 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libp2p" version = "0.56.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce71348bf5838e46449ae240631117b487073d5f347c06d434caddcb91dceb5a" dependencies = [ "bytes 1.12.1", "either", @@ -3393,7 +3448,8 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16ccf824ee859ca83df301e1c0205270206223fd4b1f2e512a693e1912a8f4a" dependencies = [ "libp2p-core", "libp2p-identity", @@ -3403,7 +3459,8 @@ dependencies = [ [[package]] name = "libp2p-autonat" version = "0.15.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab5e25c49a7d48dac83d95d8f3bac0a290d8a5df717012f6e34ce9886396c0b" dependencies = [ "async-trait", "asynchronous-codec", @@ -3427,7 +3484,8 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18b8b607cf3bfa2f8c57db9c7d8569a315d5cc0a282e6bfd5ebfc0a9840b2a0" dependencies = [ "libp2p-core", "libp2p-identity", @@ -3436,8 +3494,9 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.43.1" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.43.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249128cd37a2199aff30a7675dffa51caf073b51aa612d2f544b19932b9aebca" dependencies = [ "either", "fnv", @@ -3454,14 +3513,15 @@ dependencies = [ "rw-stream-sink", "thiserror 2.0.21", "tracing", - "unsigned-varint", + "unsigned-varint 0.8.0", "web-time", ] [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b770c1c8476736ca98c578cba4b505104ff8e842c2876b528925f9766379f9a" dependencies = [ "async-trait", "futures 0.3.34", @@ -3475,8 +3535,9 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.50.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.49.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3573f3d8e30bd62cda336df5c7c1041a1caa40a648376b8e1e274d585c0ed25c" dependencies = [ "async-channel", "asynchronous-codec", @@ -3488,7 +3549,7 @@ dependencies = [ "futures 0.3.34", "futures-timer", "getrandom 0.2.17", - "hashlink", + "hashlink 0.9.1", "hex_fmt", "libp2p-core", "libp2p-identity", @@ -3506,7 +3567,8 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab792a8b68fdef443a62155b01970c81c3aadab5e659621b063ef252a8e65e8" dependencies = [ "asynchronous-codec", "either", @@ -3525,14 +3587,15 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.12" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9525f3831544f7ae497bde79adf114ef127b0fbbb97edbbf692a80408636421c" dependencies = [ "bs58", "ed25519-dalek", "hkdf", "multihash", - "quick-protobuf", + "prost 0.14.4", "rand 0.8.8", "serde", "sha2", @@ -3543,8 +3606,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.49.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d3fd632a5872ec804d37e7413ceea20588f69d027a0fa3c46f82574f4dee60" dependencies = [ "asynchronous-codec", "bytes 1.12.1", @@ -3571,7 +3635,8 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66872d0f1ffcded2788683f76931be1c52e27f343edb93bc6d0bcd8887be443" dependencies = [ "futures 0.3.34", "hickory-proto", @@ -3581,7 +3646,7 @@ dependencies = [ "libp2p-swarm", "rand 0.8.8", "smallvec 1.16.1", - "socket2 0.6.5", + "socket2 0.5.10", "tokio 1.53.1", "tracing", ] @@ -3589,7 +3654,8 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805a555148522cb3414493a5153451910cb1a146c53ffbf4385708349baf62b7" dependencies = [ "futures 0.3.34", "libp2p-core", @@ -3606,7 +3672,8 @@ dependencies = [ [[package]] name = "libp2p-ping" version = "0.47.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74bb7fcdfd9fead4144a3859da0b49576f171a8c8c7c0bfc7c541921d25e60d3" dependencies = [ "futures 0.3.34", "futures-timer", @@ -3620,8 +3687,9 @@ dependencies = [ [[package]] name = "libp2p-quic" -version = "0.13.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dcc597d70bf7f6f30cbe07081802c836184e48416e89e9ce73a0ba2c56a319e" dependencies = [ "futures 0.3.34", "futures-timer", @@ -3630,10 +3698,11 @@ dependencies = [ "libp2p-identity", "libp2p-tls", "quinn", + "quinn-proto", "rand 0.8.8", "ring 0.17.14", "rustls 0.23.45", - "socket2 0.6.5", + "socket2 0.5.10", "thiserror 2.0.21", "tokio 1.53.1", "tracing", @@ -3642,7 +3711,8 @@ dependencies = [ [[package]] name = "libp2p-request-response" version = "0.29.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f1cca83488b90102abac7b67d5c36fc65bc02ed47620228af7ed002e6a1478" dependencies = [ "async-trait", "futures 0.3.34", @@ -3655,29 +3725,17 @@ dependencies = [ "tracing", ] -[[package]] -name = "libp2p-stream" -version = "0.4.0-alpha" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" -dependencies = [ - "futures 0.3.34", - "libp2p-core", - "libp2p-identity", - "libp2p-swarm", - "rand 0.8.8", - "tracing", -] - [[package]] name = "libp2p-swarm" -version = "0.47.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce88c6c4bf746c8482480345ea3edfd08301f49e026889d1cbccfa1808a9ed9e" dependencies = [ "either", "fnv", "futures 0.3.34", "futures-timer", - "hashlink", + "hashlink 0.10.0", "libp2p-core", "libp2p-identity", "multistream-select", @@ -3691,7 +3749,8 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c" dependencies = [ "heck 0.5.0", "quote", @@ -3700,8 +3759,9 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.44.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb6585b9309699f58704ec9ab0bb102eca7a3777170fa91a8678d73ca9cafa93" dependencies = [ "futures 0.3.34", "futures-timer", @@ -3716,7 +3776,8 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ff65a82e35375cbc31ebb99cacbbf28cb6c4fefe26bf13756ddcf708d40080" dependencies = [ "futures 0.3.34", "futures-rustls", @@ -3733,8 +3794,9 @@ dependencies = [ [[package]] name = "libp2p-upnp" -version = "0.5.1" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4757e65fe69399c1a243bbb90ec1ae5a2114b907467bf09f3575e899815bb8d3" dependencies = [ "futures 0.3.34", "futures-timer", @@ -3798,7 +3860,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "logs-collector" -version = "2.5.0" +version = "2.5.1" dependencies = [ "anyhow", "async-trait", @@ -4002,7 +4064,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.8.0", "url", ] @@ -4026,7 +4088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" dependencies = [ "serde", - "unsigned-varint", + "unsigned-varint 0.8.0", ] [[package]] @@ -4038,14 +4100,15 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" dependencies = [ "bytes 1.12.1", "futures 0.3.34", + "log", "pin-project", "smallvec 1.16.1", - "tracing", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -4640,7 +4703,7 @@ dependencies = [ [[package]] name = "pings-collector" -version = "2.9.0" +version = "2.9.1" dependencies = [ "anyhow", "async-trait", @@ -4708,7 +4771,7 @@ dependencies = [ [[package]] name = "portal-logs-collector" -version = "1.1.6" +version = "1.1.7" dependencies = [ "anyhow", "clap", @@ -4919,6 +4982,16 @@ dependencies = [ "prost-derive 0.13.5", ] +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes 1.12.1", + "prost-derive 0.14.4", +] + [[package]] name = "prost-build" version = "0.12.6" @@ -4966,6 +5039,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "prost-types" version = "0.12.6" @@ -4993,13 +5079,14 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" dependencies = [ "asynchronous-codec", "bytes 1.12.1", "quick-protobuf", - "thiserror 2.0.21", - "unsigned-varint", + "thiserror 1.0.69", + "unsigned-varint 0.8.0", ] [[package]] @@ -5030,6 +5117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "bytes 1.12.1", + "fastbloom", "getrandom 0.4.3", "lru-slab", "rand 0.10.3", @@ -5646,7 +5734,8 @@ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/kalabukdima/rust-libp2p.git?rev=fac3b0c9#fac3b0c97757d6d8d6042f5423d9467a94ea80b0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" dependencies = [ "futures 0.3.34", "pin-project", @@ -6131,7 +6220,7 @@ dependencies = [ [[package]] name = "sqd-contract-client" version = "1.3.0" -source = "git+https://github.com/subsquid/sqd-network.git?rev=11f4230#11f42309cb6e68365c8b7277555108d9cc3d968c" +source = "git+https://github.com/subsquid/sqd-network.git?rev=e1460fd#e1460fd00cfd16d8c8baefdc357cfc359551dd90" dependencies = [ "async-trait", "clap", @@ -6149,12 +6238,18 @@ dependencies = [ "url", ] +[[package]] +name = "sqd-data-chunk" +version = "0.1.0" +source = "git+https://github.com/subsquid/sqd-network.git?rev=e1460fd#e1460fd00cfd16d8c8baefdc357cfc359551dd90" + [[package]] name = "sqd-messages" -version = "2.1.0" -source = "git+https://github.com/subsquid/sqd-network.git?rev=11f4230#11f42309cb6e68365c8b7277555108d9cc3d968c" +version = "3.1.0" +source = "git+https://github.com/subsquid/sqd-network.git?rev=e1460fd#e1460fd00cfd16d8c8baefdc357cfc359551dd90" dependencies = [ "bytemuck", + "bytes 1.12.1", "flate2", "hex", "libp2p-identity", @@ -6163,12 +6258,13 @@ dependencies = [ "semver 1.0.28", "serde", "sha3", + "sqd-data-chunk", ] [[package]] name = "sqd-network-transport" version = "3.1.0" -source = "git+https://github.com/subsquid/sqd-network.git?rev=11f4230#11f42309cb6e68365c8b7277555108d9cc3d968c" +source = "git+https://github.com/subsquid/sqd-network.git?rev=e1460fd#e1460fd00cfd16d8c8baefdc357cfc359551dd90" dependencies = [ "anyhow", "async-trait", @@ -6180,13 +6276,16 @@ dependencies = [ "lazy_static", "libp2p", "libp2p-connection-limits", - "libp2p-stream", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", "libp2p-swarm-derive", "log", "lru", "parking_lot 0.12.5", "pin-project", "prost 0.13.5", + "rand 0.9.5", "serde", "sqd-contract-client", "sqd-messages", @@ -6194,6 +6293,7 @@ dependencies = [ "tokio 1.53.1", "tokio-stream", "tokio-util", + "tracing", ] [[package]] @@ -7116,6 +7216,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + [[package]] name = "unsigned-varint" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index a45f7a8..4338dec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,12 +3,6 @@ members = ["crates/*"] resolver = "2" [workspace.dependencies] -sqd-contract-client = { git = "https://github.com/subsquid/sqd-network.git", rev = "11f4230", version = "1.3.0" } -sqd-messages = { git = "https://github.com/subsquid/sqd-network.git", rev = "11f4230", version = "2.1.0", features = ["bitstring"] } -sqd-network-transport = { git = "https://github.com/subsquid/sqd-network.git", rev = "11f4230", version = "3.1.0" } - -# The sqd-network fork builds its crates against its in-tree libp2p-identity (path = "identity"), so -# consumers must redirect the crates.io libp2p-identity to the same source, otherwise two incompatible -# copies of libp2p_identity are linked. See the fork's "Use libp2p-identity from the fork" commit. -[patch.crates-io] -libp2p-identity = { git = "https://github.com/kalabukdima/rust-libp2p.git", rev = "fac3b0c9" } +sqd-contract-client = { git = "https://github.com/subsquid/sqd-network.git", rev = "e1460fd", version = "1.3.0" } +sqd-messages = { git = "https://github.com/subsquid/sqd-network.git", rev = "e1460fd", version = "3.1.0", features = ["bitstring"] } +sqd-network-transport = { git = "https://github.com/subsquid/sqd-network.git", rev = "e1460fd", version = "3.1.0" } diff --git a/crates/collector-utils/src/storage.rs b/crates/collector-utils/src/storage.rs index 6462f22..53e6a29 100644 --- a/crates/collector-utils/src/storage.rs +++ b/crates/collector-utils/src/storage.rs @@ -588,6 +588,7 @@ mod tests { compression: Default::default(), query_engine: Default::default(), output_format: Default::default(), + chunk_version: Default::default(), }; query.sign(&client_keypair, worker_id).unwrap(); diff --git a/crates/logs-collector/Cargo.toml b/crates/logs-collector/Cargo.toml index 5830417..1d2a935 100644 --- a/crates/logs-collector/Cargo.toml +++ b/crates/logs-collector/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "logs-collector" -version = "2.5.0" +version = "2.5.1" edition = "2021" [dependencies] diff --git a/crates/pings-collector/Cargo.toml b/crates/pings-collector/Cargo.toml index 774c6f2..996b528 100644 --- a/crates/pings-collector/Cargo.toml +++ b/crates/pings-collector/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pings-collector" -version = "2.9.0" +version = "2.9.1" edition = "2021" [dependencies] diff --git a/crates/portal-logs-collector/Cargo.toml b/crates/portal-logs-collector/Cargo.toml index e31836c..9545133 100644 --- a/crates/portal-logs-collector/Cargo.toml +++ b/crates/portal-logs-collector/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "portal-logs-collector" -version = "1.1.6" +version = "1.1.7" edition = "2021" [dependencies] diff --git a/crates/router-controller/src/controller.rs b/crates/router-controller/src/controller.rs index 6ab7844..06f5913 100644 --- a/crates/router-controller/src/controller.rs +++ b/crates/router-controller/src/controller.rs @@ -521,15 +521,12 @@ mod tests { .set_datasets((0..2).map(|i| (i.to_string(), i.to_string()))) .build(); + let chunk = |top, first_block, last_block| { + DataChunk::new(top, first_block, last_block, "abcde").expect("valid chunk") + }; let chunks = [ - vec![ - DataChunk::new(0, 0, 10, "".to_string()), - DataChunk::new(0, 11, 200, "".to_string()), - ], - vec![ - DataChunk::new(0, 0, 5, "".to_string()), - DataChunk::new(6, 6, 20, "".to_string()), - ], + vec![chunk(0, 0, 10), chunk(0, 11, 200)], + vec![chunk(0, 0, 5), chunk(6, 6, 20)], ]; for w in 0..8 { diff --git a/crates/router/src/dataset.rs b/crates/router/src/dataset.rs index 90f6f8f..92a0aca 100644 --- a/crates/router/src/dataset.rs +++ b/crates/router/src/dataset.rs @@ -14,6 +14,17 @@ fn invalid_object_key(key: &str) -> String { format!("invalid object key - {key}") } +/// The chunk of a `/blocks.parquet` key; `None` for the chunk's other files. +fn chunk_of_blocks_key(key: &str) -> Result, String> { + let Some(chunk_id) = key.strip_suffix("/blocks.parquet") else { + return Ok(None); + }; + + DataChunk::from_str(chunk_id) + .map(Some) + .map_err(|()| invalid_object_key(key)) +} + fn trim_trailing_slash(value: &str) -> String { let mut value = value.to_string(); value.pop(); @@ -84,11 +95,8 @@ impl Storage for S3Storage { let mut chunks = vec![]; for object in &objects { if let Some(key) = object.key() { - if key.ends_with("blocks.parquet") { - match DataChunk::from_str(key) { - Ok(chunk) => chunks.push(chunk), - Err(..) => return Err(invalid_object_key(key)), - } + if let Some(chunk) = chunk_of_blocks_key(key)? { + chunks.push(chunk); } } } @@ -127,3 +135,36 @@ impl S3Storage { Ok(items) } } + +#[cfg(test)] +mod tests { + use super::chunk_of_blocks_key; + + #[test] + fn reads_the_chunk_from_a_full_object_key() { + let key = "0000000000/0000000000-0000000099-abcde/blocks.parquet"; + + let chunk = chunk_of_blocks_key(key) + .unwrap() + .expect("a blocks.parquet key"); + + assert_eq!(chunk.to_string(), "0000000000/0000000000-0000000099-abcde"); + } + + #[test] + fn skips_the_other_files_of_a_chunk() { + let key = "0000000000/0000000000-0000000099-abcde/transactions.parquet"; + + assert_eq!(chunk_of_blocks_key(key), Ok(None)); + } + + #[test] + fn rejects_a_malformed_chunk_id() { + let key = "0000000000/0000000099-abcde/blocks.parquet"; + + assert_eq!( + chunk_of_blocks_key(key), + Err(format!("invalid object key - {key}")) + ); + } +}