diff --git a/Cargo.lock b/Cargo.lock index f30a75be..a8316e1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -425,7 +425,7 @@ dependencies = [ "foldhash", "getrandom 0.3.3", "hashbrown 0.15.5", - "indexmap 2.11.0", + "indexmap 2.11.1", "itoa", "k256", "keccak-asm", @@ -856,7 +856,7 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck 0.5.0", - "indexmap 2.11.0", + "indexmap 2.11.1", "proc-macro-error2", "proc-macro2", "quote", @@ -1014,7 +1014,7 @@ dependencies = [ "arrayvec", "derive_arbitrary", "derive_more", - "nybbles 0.4.3", + "nybbles 0.4.4", "proptest", "proptest-derive", "serde", @@ -1035,12 +1035,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -2192,7 +2186,7 @@ dependencies = [ "boa_interner", "boa_macros", "boa_string", - "indexmap 2.11.0", + "indexmap 2.11.1", "num-bigint", "rustc-hash 2.1.1", ] @@ -2218,7 +2212,7 @@ dependencies = [ "fast-float2", "hashbrown 0.15.5", "icu_normalizer 1.5.0", - "indexmap 2.11.0", + "indexmap 2.11.1", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -2264,7 +2258,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.15.5", - "indexmap 2.11.0", + "indexmap 2.11.1", "once_cell", "phf", "rustc-hash 2.1.1", @@ -2591,17 +2585,16 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-link 0.1.3", + "windows-link 0.2.0", ] [[package]] @@ -3102,7 +3095,7 @@ dependencies = [ "crossterm_winapi", "document-features", "parking_lot 0.12.4", - "rustix 1.0.8", + "rustix 1.1.2", "winapi", ] @@ -3749,12 +3742,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -4176,7 +4169,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "wasi 0.14.4+wasi-0.2.4", + "wasi 0.14.5+wasi-0.2.4", "wasm-bindgen", ] @@ -4294,7 +4287,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.11.0", + "indexmap 2.11.1", "slab", "tokio", "tokio-util", @@ -4313,7 +4306,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.11.0", + "indexmap 2.11.1", "slab", "tokio", "tokio-util", @@ -5054,9 +5047,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" dependencies = [ "arbitrary", "equivalent", @@ -5726,9 +5719,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -5919,7 +5912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" dependencies = [ "base64 0.22.1", - "indexmap 2.11.0", + "indexmap 2.11.1", "metrics", "metrics-util", "quanta", @@ -6329,9 +6322,9 @@ dependencies = [ [[package]] name = "nybbles" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63cb50036b1ad148038105af40aaa70ff24d8a14fbc44ae5c914e1348533d12e" +checksum = "f0418987d1aaed324d95b4beffc93635e19be965ed5d63ec07a35980fe3b71a4" dependencies = [ "alloy-rlp", "arbitrary", @@ -8535,7 +8528,7 @@ dependencies = [ "alloy-evm", "alloy-primitives", "alloy-rlp", - "nybbles 0.4.3", + "nybbles 0.4.4", "reth-storage-errors", "thiserror 2.0.16", ] @@ -8677,7 +8670,7 @@ dependencies = [ "byteorder", "dashmap 6.1.0", "derive_more", - "indexmap 2.11.0", + "indexmap 2.11.1", "parking_lot 0.12.4", "reth-mdbx-sys", "smallvec", @@ -10308,7 +10301,7 @@ dependencies = [ "derive_more", "hash-db", "itertools 0.14.0", - "nybbles 0.4.3", + "nybbles 0.4.4", "plain_hasher", "reth-codecs", "reth-primitives-traits", @@ -10962,6 +10955,7 @@ dependencies = [ "serde", "thiserror 2.0.16", "tokio", + "tracing", ] [[package]] @@ -11197,15 +11191,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.8" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags 2.9.4", "errno", "libc", - "linux-raw-sys 0.9.4", - "windows-sys 0.60.2", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.0", ] [[package]] @@ -11369,11 +11363,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -12052,7 +12046,7 @@ version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "itoa", "memchr", "ryu", @@ -12090,7 +12084,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.0", + "indexmap 2.11.1", "schemars 0.9.0", "schemars 1.0.4", "serde", @@ -12402,7 +12396,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink 0.10.0", - "indexmap 2.11.0", + "indexmap 2.11.1", "log", "memchr", "native-tls", @@ -12768,15 +12762,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.21.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" +checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix 1.0.8", - "windows-sys 0.60.2", + "rustix 1.1.2", + "windows-sys 0.61.0", ] [[package]] @@ -13080,7 +13074,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "serde", "serde_spanned", "toml_datetime", @@ -13103,7 +13097,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.11.0", + "indexmap 2.11.1", "pin-project-lite", "slab", "sync_wrapper", @@ -13418,9 +13412,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "unicode-normalization" @@ -13660,9 +13654,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.4+wasi-0.2.4" +version = "0.14.5+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.0+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a" +checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" dependencies = [ "wit-bindgen", ] @@ -14522,7 +14525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" dependencies = [ "libc", - "rustix 1.0.8", + "rustix 1.1.2", ] [[package]] diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 0dc17d3c..3c6e370b 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -30,7 +30,7 @@ use rollup_node_manager::{ }; use rollup_node_primitives::{BlockInfo, NodeConfig}; use rollup_node_providers::{ - BlobSource, DatabaseL1MessageProvider, FullL1Provider, L1MessageProvider, L1Provider, + BlobProvidersBuilder, DatabaseL1MessageProvider, FullL1Provider, L1MessageProvider, L1Provider, SystemContractProvider, }; use rollup_node_sequencer::{L1MessageInclusionMode, Sequencer}; @@ -63,9 +63,9 @@ pub struct ScrollRollupNodeConfig { /// Engine driver args. #[command(flatten)] pub engine_driver_args: EngineDriverArgs, - /// The beacon provider arguments. + /// The blob provider arguments. #[command(flatten)] - pub beacon_provider_args: BeaconProviderArgs, + pub blob_provider_args: BlobProviderArgs, /// The L1 provider arguments #[command(flatten)] pub l1_provider_args: L1ProviderArgs, @@ -317,12 +317,15 @@ impl ScrollRollupNodeConfig { // Construct the l1 provider. let l1_messages_provider = DatabaseL1MessageProvider::new(db.clone(), 0); - let blob_provider = self - .beacon_provider_args - .blob_source - .provider(self.beacon_provider_args.url) - .await - .expect("failed to construct L1 blob provider"); + let blob_providers_builder = BlobProvidersBuilder { + beacon: self.blob_provider_args.beacon_node_urls, + s3: self.blob_provider_args.s3_url, + anvil: self.blob_provider_args.anvil_url, + mock: self.blob_provider_args.mock, + }; + let blob_provider = + blob_providers_builder.build().await.expect("failed to construct L1 blob provider"); + let l1_provider = FullL1Provider::new(blob_provider, l1_messages_provider.clone()).await; // Construct the Sequencer. @@ -565,26 +568,31 @@ pub struct L1ProviderArgs { /// The arguments for the Beacon provider. #[derive(Debug, Default, Clone, clap::Args)] -pub struct BeaconProviderArgs { - /// The URL for the Beacon chain. - #[arg(long = "beacon.url", id = "beacon_url", value_name = "BEACON_URL")] - pub url: Option, - /// The blob source for the provider. +pub struct BlobProviderArgs { + /// The URLs for the beacon node blob provider. #[arg( - long = "beacon.blob-source", - id = "beacon_blob_source", - value_name = "BEACON_BLOB_SOURCE", - default_value = "mock" + long = "blob.beacon_node_urls", + id = "blob_beacon_node_urls", + value_name = "BLOB_BEACON_NODE_URLS" )] - pub blob_source: BlobSource, + pub beacon_node_urls: Option>, + /// The URL for the s3 blob provider. + #[arg(long = "blob.s3_url", id = "blob_s3_url", value_name = "BLOB_S3_URL")] + pub s3_url: Option, + /// The URL for the anvil blob provider. + #[arg(long = "blob.anvil_url", id = "blob_anvil_url", value_name = "BLOB_ANVIL_URL")] + pub anvil_url: Option, + /// Enable the mock blob source. + #[arg(long = "blob.mock")] + pub mock: bool, /// The compute units per second for the provider. - #[arg(long = "beacon.cups", id = "beacon_compute_units_per_second", value_name = "BEACON_COMPUTE_UNITS_PER_SECOND", default_value_t = constants::PROVIDER_COMPUTE_UNITS_PER_SECOND)] + #[arg(long = "blob.cups", id = "blob_compute_units_per_second", value_name = "BLOB_COMPUTE_UNITS_PER_SECOND", default_value_t = constants::PROVIDER_COMPUTE_UNITS_PER_SECOND)] pub compute_units_per_second: u64, /// The max amount of retries for the provider. - #[arg(long = "beacon.max-retries", id = "beacon_max_retries", value_name = "BEACON_MAX_RETRIES", default_value_t = constants::PROVIDER_MAX_RETRIES)] + #[arg(long = "blob.max-retries", id = "blob_max_retries", value_name = "BLOB_MAX_RETRIES", default_value_t = constants::PROVIDER_MAX_RETRIES)] pub max_retries: u32, /// The initial backoff for the provider. - #[arg(long = "beacon.initial-backoff", id = "beacon_initial_backoff", value_name = "BEACON_INITIAL_BACKOFF", default_value_t = constants::PROVIDER_INITIAL_BACKOFF)] + #[arg(long = "blob.initial-backoff", id = "blob_initial_backoff", value_name = "BLOB_INITIAL_BACKOFF", default_value_t = constants::PROVIDER_INITIAL_BACKOFF)] pub initial_backoff: u64, } @@ -798,7 +806,7 @@ mod tests { engine_driver_args: EngineDriverArgs::default(), chain_orchestrator_args: ChainOrchestratorArgs::default(), l1_provider_args: L1ProviderArgs::default(), - beacon_provider_args: BeaconProviderArgs::default(), + blob_provider_args: BlobProviderArgs::default(), network_args: NetworkArgs::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs { @@ -829,7 +837,7 @@ mod tests { engine_driver_args: EngineDriverArgs::default(), chain_orchestrator_args: ChainOrchestratorArgs::default(), l1_provider_args: L1ProviderArgs::default(), - beacon_provider_args: BeaconProviderArgs::default(), + blob_provider_args: BlobProviderArgs::default(), network_args: NetworkArgs::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs { @@ -858,7 +866,7 @@ mod tests { chain_orchestrator_args: ChainOrchestratorArgs::default(), engine_driver_args: EngineDriverArgs::default(), l1_provider_args: L1ProviderArgs::default(), - beacon_provider_args: BeaconProviderArgs::default(), + blob_provider_args: BlobProviderArgs::default(), network_args: NetworkArgs::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -882,7 +890,7 @@ mod tests { engine_driver_args: EngineDriverArgs::default(), chain_orchestrator_args: ChainOrchestratorArgs::default(), l1_provider_args: L1ProviderArgs::default(), - beacon_provider_args: BeaconProviderArgs::default(), + blob_provider_args: BlobProviderArgs::default(), network_args: NetworkArgs::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -902,7 +910,7 @@ mod tests { engine_driver_args: EngineDriverArgs::default(), chain_orchestrator_args: ChainOrchestratorArgs::default(), l1_provider_args: L1ProviderArgs::default(), - beacon_provider_args: BeaconProviderArgs::default(), + blob_provider_args: BlobProviderArgs::default(), network_args: NetworkArgs::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), diff --git a/crates/node/src/test_utils.rs b/crates/node/src/test_utils.rs index 37c14906..dcd21008 100644 --- a/crates/node/src/test_utils.rs +++ b/crates/node/src/test_utils.rs @@ -3,7 +3,7 @@ use crate::{ConsensusArgs, GasPriceOracleArgs}; use super::{ - BeaconProviderArgs, ChainOrchestratorArgs, DatabaseArgs, EngineDriverArgs, L1ProviderArgs, + BlobProviderArgs, ChainOrchestratorArgs, DatabaseArgs, EngineDriverArgs, L1ProviderArgs, ScrollRollupNode, ScrollRollupNodeConfig, SequencerArgs, }; use alloy_primitives::Bytes; @@ -21,7 +21,6 @@ use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs, TxPoolArgs use reth_provider::providers::BlockchainProvider; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::TaskManager; -use rollup_node_providers::BlobSource; use rollup_node_sequencer::L1MessageInclusionMode; use std::{path::PathBuf, sync::Arc}; use tokio::sync::Mutex; @@ -155,10 +154,7 @@ pub fn default_test_scroll_rollup_node_config() -> ScrollRollupNodeConfig { allow_empty_blocks: true, ..Default::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -194,10 +190,7 @@ pub fn default_sequencer_test_scroll_rollup_node_config() -> ScrollRollupNodeCon l1_message_inclusion_mode: L1MessageInclusionMode::BlockDepth(0), allow_empty_blocks: true, }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), diff --git a/crates/node/tests/e2e.rs b/crates/node/tests/e2e.rs index 8112c24c..3445be8a 100644 --- a/crates/node/tests/e2e.rs +++ b/crates/node/tests/e2e.rs @@ -24,7 +24,7 @@ use rollup_node::{ default_sequencer_test_scroll_rollup_node_config, default_test_scroll_rollup_node_config, generate_tx, setup_engine, }, - BeaconProviderArgs, ChainOrchestratorArgs, ConsensusAlgorithm, ConsensusArgs, DatabaseArgs, + BlobProviderArgs, ChainOrchestratorArgs, ConsensusAlgorithm, ConsensusArgs, DatabaseArgs, EngineDriverArgs, GasPriceOracleArgs, L1ProviderArgs, NetworkArgs as ScrollNetworkArgs, RollupNodeContext, RollupNodeExtApiClient, ScrollRollupNode, ScrollRollupNodeConfig, SequencerArgs, @@ -32,7 +32,6 @@ use rollup_node::{ use rollup_node_chain_orchestrator::ChainOrchestratorEvent; use rollup_node_manager::{RollupManagerCommand, RollupManagerEvent}; use rollup_node_primitives::{sig_encode_hash, BatchCommitData, ConsensusUpdate}; -use rollup_node_providers::BlobSource; use rollup_node_sequencer::L1MessageInclusionMode; use rollup_node_watcher::L1Notification; use scroll_alloy_consensus::TxL1Message; @@ -68,10 +67,7 @@ async fn can_bridge_l1_messages() -> eyre::Result<()> { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -167,10 +163,7 @@ async fn can_sequence_and_gossip_blocks() { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -268,10 +261,7 @@ async fn can_penalize_peer_for_invalid_block() { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -832,11 +822,9 @@ async fn graceful_shutdown_consolidates_most_recent_batch_on_startup() -> eyre:: let mut config = default_test_scroll_rollup_node_config(); let path = node.inner.config.datadir().db().join("scroll.db?mode=rwc"); let path = PathBuf::from("sqlite://".to_string() + &*path.to_string_lossy()); - config.beacon_provider_args.url = Some( - "http://dummy:8545" - .parse() - .expect("valid url that will not be used as test batches use calldata"), - ); + config.blob_provider_args.beacon_node_urls = Some(vec!["http://dummy:8545" + .parse() + .expect("valid url that will not be used as test batches use calldata")]); config.hydrate(node.inner.config.clone()).await?; let (_, events) = ScrollWireProtocolHandler::new(ScrollWireConfig::new(true)); diff --git a/crates/node/tests/sync.rs b/crates/node/tests/sync.rs index 907c304b..5e389879 100644 --- a/crates/node/tests/sync.rs +++ b/crates/node/tests/sync.rs @@ -14,13 +14,12 @@ use rollup_node::{ default_sequencer_test_scroll_rollup_node_config, default_test_scroll_rollup_node_config, setup_engine, }, - BeaconProviderArgs, ChainOrchestratorArgs, ConsensusArgs, DatabaseArgs, EngineDriverArgs, + BlobProviderArgs, ChainOrchestratorArgs, ConsensusArgs, DatabaseArgs, EngineDriverArgs, GasPriceOracleArgs, L1ProviderArgs, NetworkArgs, ScrollRollupNodeConfig, SequencerArgs, }; use rollup_node_chain_orchestrator::ChainOrchestratorEvent; use rollup_node_manager::RollupManagerEvent; use rollup_node_primitives::BlockInfo; -use rollup_node_providers::BlobSource; use rollup_node_sequencer::L1MessageInclusionMode; use rollup_node_watcher::L1Notification; use scroll_alloy_consensus::TxL1Message; @@ -63,12 +62,12 @@ async fn test_should_consolidate_to_block_15k() -> eyre::Result<()> { allow_empty_blocks: true, ..Default::default() }, - beacon_provider_args: BeaconProviderArgs { - url: Some(Url::parse("https://eth-beacon-chain.drpc.org/rest/")?), + blob_provider_args: BlobProviderArgs { + beacon_node_urls: Some(vec![Url::parse("https://eth-beacon-chain.drpc.org/rest/")?]), compute_units_per_second: 100, max_retries: 10, initial_backoff: 100, - blob_source: BlobSource::Beacon, + ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), @@ -206,10 +205,7 @@ async fn test_should_consolidate_after_optimistic_sync() -> eyre::Result<()> { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -457,10 +453,7 @@ async fn test_consolidation() -> eyre::Result<()> { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), @@ -632,10 +625,7 @@ async fn test_chain_orchestrator_shallow_reorg_with_gap() -> eyre::Result<()> { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: Default::default(), gas_price_oracle_args: GasPriceOracleArgs::default(), consensus_args: ConsensusArgs::noop(), diff --git a/crates/providers/Cargo.toml b/crates/providers/Cargo.toml index 281e826a..ecbeab68 100644 --- a/crates/providers/Cargo.toml +++ b/crates/providers/Cargo.toml @@ -38,6 +38,7 @@ reqwest = { workspace = true, features = ["json"] } serde = { workspace = true, features = ["derive"] } thiserror.workspace = true tokio = { workspace = true, default-features = false } +tracing.workspace = true [dev-dependencies] scroll-db = { workspace = true, features = ["test-utils"] } diff --git a/crates/providers/src/l1/blob/mod.rs b/crates/providers/src/l1/blob/mod.rs index 2cb19f65..60575673 100644 --- a/crates/providers/src/l1/blob/mod.rs +++ b/crates/providers/src/l1/blob/mod.rs @@ -13,11 +13,15 @@ mod s3; pub use s3::S3BlobProvider; use crate::L1ProviderError; -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; use alloy_eips::eip4844::Blob; use alloy_primitives::B256; -use eyre::OptionExt; +use tracing::{debug, info, warn}; + /// An instance of the trait can be used to fetch L1 blob data. #[async_trait::async_trait] #[auto_impl::auto_impl(Arc, &)] @@ -30,38 +34,181 @@ pub trait BlobProvider: Sync + Send { ) -> Result>, L1ProviderError>; } -/// The blob source for the beacon provider. -#[derive(Debug, clap::ValueEnum, Default, Clone)] -pub enum BlobSource { +/// The builder for the blob provider. +#[derive(Debug, Clone)] +pub struct BlobProvidersBuilder { /// Beacon client blob source. - #[default] - Beacon, - /// Mocked source. - Mock, + pub beacon: Option>, /// AWS S3 blob source. - S3, + pub s3: Option, /// Anvil sequencer blob source. - Anvil, + pub anvil: Option, + /// Mocked source. + pub mock: bool, +} + +impl BlobProvidersBuilder { + /// Returns an [`BlobProviders`]. + pub async fn build(&self) -> eyre::Result { + if self.mock { + info!(target: "scroll::providers", "Running with mock blob provider - all other blob provider configurations are ignored"); + return Ok(BlobProviders::new( + vec![], + vec![Arc::new(MockBeaconProvider::default()) as Arc], + )); + } + + let beacon_providers = if let Some(beacon_urls) = &self.beacon { + let mut providers = Vec::new(); + for url in beacon_urls { + providers.push(Arc::new(BeaconClientProvider::new_http(url.clone()).await) + as Arc); + } + providers + } else { + Vec::new() + }; + + let mut backup_providers: Vec> = vec![]; + if let Some(s3) = &self.s3 { + backup_providers + .push(Arc::new(S3BlobProvider::new_http(s3.clone())) as Arc); + } + if let Some(anvil) = &self.anvil { + backup_providers.push(Arc::new(AnvilBlobProvider::new_http(anvil.clone())) as Arc); + } + + if beacon_providers.is_empty() && backup_providers.is_empty() { + return Err(eyre::eyre!("No blob providers available")); + } + + Ok(BlobProviders::new(beacon_providers, backup_providers)) + } +} + +/// A blob provider that implements round-robin load balancing across multiple providers. +#[derive(Clone)] +pub struct BlobProviders { + /// beacon providers + beacon_providers: Vec>, + /// The list of underlying backup blob providers. + backup_providers: Vec>, + /// Atomic counter for round-robin selection for beacon providers. + beacon_counter: Arc, + /// Atomic counter for round-robin selection for backup providers. + backup_counter: Arc, +} + +impl std::fmt::Debug for BlobProviders { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlobProviders") + .field("beacon_providers_count", &self.beacon_providers.len()) + .field("backup_providers_count", &self.backup_providers.len()) + .field("beacon_counter", &self.beacon_counter.load(Ordering::Relaxed)) + .field("backup_counter", &self.backup_counter.load(Ordering::Relaxed)) + .finish() + } +} + +impl BlobProviders { + /// Creates a new round-robin blob provider. + pub fn new( + beacon_providers: Vec>, + backup_providers: Vec>, + ) -> Self { + Self { + beacon_providers, + backup_providers, + beacon_counter: Arc::new(AtomicUsize::new(0)), + backup_counter: Arc::new(AtomicUsize::new(0)), + } + } + + /// Try providers in round-robin order and return the first successful blob found. + async fn try_providers( + &self, + providers: &[Arc], + counter: &AtomicUsize, + provider_type: &str, + block_timestamp: u64, + hash: B256, + ) -> Result>, L1ProviderError> { + if providers.is_empty() { + return Err(L1ProviderError::Other("No providers available")); + } + + let start_index = counter.load(Ordering::Relaxed) % providers.len(); + + for i in 0..providers.len() { + let provider_index = (start_index + i) % providers.len(); + let provider = &providers[provider_index]; + + // update the counter to the next provider round-robin index. + counter.store(provider_index + 1, Ordering::Relaxed); + + match provider.blob(block_timestamp, hash).await { + Ok(blob) => { + return Ok(blob); + } + Err(err) => { + debug!(target: "scroll::providers", ?hash, ?block_timestamp, ?provider_index, ?err, provider_type, "provider failed to fetch blob"); + } + } + } + + // All providers tried, none had the blob (but some may have responded successfully) + Ok(None) + } } -impl BlobSource { - /// Returns an [`Arc`] for the provided URL. - pub async fn provider(&self, url: Option) -> eyre::Result> { - Ok(match self { - Self::Beacon => Arc::new( - BeaconClientProvider::new_http( - url.ok_or_eyre("missing url for consensus client provider")?, +#[async_trait::async_trait] +impl BlobProvider for BlobProviders { + async fn blob( + &self, + block_timestamp: u64, + hash: B256, + ) -> Result>, L1ProviderError> { + // First try beacon providers + if !self.beacon_providers.is_empty() { + match self + .try_providers( + &self.beacon_providers, + &self.beacon_counter, + "beacon", + block_timestamp, + hash, + ) + .await + { + Ok(blob) => return Ok(blob), + Err(err) => { + debug!(target: "scroll::providers", ?hash, ?block_timestamp, ?err, "All beacon providers failed, trying backup providers"); + } + } + } + + // Try backup providers + if !self.backup_providers.is_empty() { + match self + .try_providers( + &self.backup_providers, + &self.backup_counter, + "backup", + block_timestamp, + hash, ) - .await, - ), - Self::Mock => Arc::new(MockBeaconProvider::default()), - Self::S3 => Arc::new(S3BlobProvider::new_http( - url.ok_or_eyre("missing url for s3 blob provider")?, - )), - Self::Anvil => Arc::new(AnvilBlobProvider::new_http( - url.ok_or_eyre("missing url for anvil blob provider")?, - )), - }) + .await + { + Ok(blob) => return Ok(blob), + Err(err) => { + debug!(target: "scroll::providers", ?hash, ?block_timestamp, ?err, "All backup providers failed"); + } + } + } + + // All providers failed + warn!(target: "scroll::providers", ?hash, ?block_timestamp, "All providers failed to fetch blob"); + Err(L1ProviderError::Other("All blob providers failed")) } } @@ -89,4 +236,126 @@ mod tests { assert!(blob.is_some()); } + + #[tokio::test] + async fn test_blob_providers() { + let source = BlobProvidersBuilder { beacon: None, s3: None, anvil: None, mock: true }; + + let provider = source.build().await.unwrap(); + let result = provider.blob(0, B256::ZERO).await.unwrap(); + assert!(result.is_none()); // MockBeaconProvider returns None + } + + #[tokio::test] + async fn test_blob_providers_with_backup() { + let providers = BlobProviders::new( + vec![], + vec![ + Arc::new(MockBeaconProvider::default()) as Arc, + Arc::new(MockBeaconProvider::default()) as Arc, + ], + ); + + // Test multiple calls to ensure round-robin behavior + for _ in 0..5 { + let result = providers.blob(0, B256::ZERO).await.unwrap(); + assert!(result.is_none()); + } + } + + #[tokio::test] + async fn test_blob_providers_beacon_priority() { + // Test that beacon providers are tried first + let providers = BlobProviders::new( + vec![Arc::new(MockBeaconProvider::default()) as Arc], + vec![ + Arc::new(MockBeaconProvider::default()) as Arc, + Arc::new(MockBeaconProvider::default()) as Arc, + ], + ); + + let result = providers.blob(0, B256::ZERO).await.unwrap(); + assert!(result.is_none()); // MockBeaconProvider returns None + } + + #[tokio::test] + async fn test_blob_providers_round_robin() { + // Create a provider with only backup providers to test round-robin + let providers = BlobProviders::new( + vec![], + vec![ + Arc::new(MockBeaconProvider::default()) as Arc, + Arc::new(MockBeaconProvider::default()) as Arc, + Arc::new(MockBeaconProvider::default()) as Arc, + ], + ); + + // Test multiple calls to verify round-robin behavior + for i in 0..10 { + let result = providers.blob(0, B256::from([i as u8; 32])).await.unwrap(); + assert!(result.is_none()); // MockBeaconProvider always returns None + } + } + + #[tokio::test] + async fn test_blob_providers_empty_fails() { + let providers = BlobProviders::new(vec![], vec![]); + + // Should fail when no providers are available + let result = providers.blob(0, B256::ZERO).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_blob_providers_builder_mock() { + let builder = BlobProvidersBuilder { beacon: None, s3: None, anvil: None, mock: true }; + + let providers = builder.build().await.unwrap(); + let result = providers.blob(0, B256::ZERO).await.unwrap(); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_blob_providers_builder_no_providers() { + let builder = BlobProvidersBuilder { beacon: None, s3: None, anvil: None, mock: false }; + + // Should fail when no providers are configured + let result = builder.build().await; + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("No blob providers available")); + } + + #[tokio::test] + async fn test_blob_providers_clone() { + let providers = BlobProviders::new( + vec![Arc::new(MockBeaconProvider::default()) as Arc], + vec![Arc::new(MockBeaconProvider::default()) as Arc], + ); + + // Test that BlobProviders can be cloned + let cloned_providers = providers.clone(); + + // Both should work independently + let result1 = providers.blob(0, B256::ZERO).await.unwrap(); + let result2 = cloned_providers.blob(0, B256::ZERO).await.unwrap(); + + assert!(result1.is_none()); + assert!(result2.is_none()); + } + + #[tokio::test] + async fn test_blob_providers_debug_format() { + let providers = BlobProviders::new( + vec![Arc::new(MockBeaconProvider::default()) as Arc], + vec![ + Arc::new(MockBeaconProvider::default()) as Arc, + Arc::new(MockBeaconProvider::default()) as Arc, + ], + ); + + let debug_str = format!("{:?}", providers); + assert!(debug_str.contains("BlobProviders")); + assert!(debug_str.contains("beacon_providers_count")); + assert!(debug_str.contains("backup_providers_count")); + } } diff --git a/crates/providers/src/lib.rs b/crates/providers/src/lib.rs index f81ad306..c842210a 100644 --- a/crates/providers/src/lib.rs +++ b/crates/providers/src/lib.rs @@ -10,8 +10,8 @@ pub use block::BlockDataProvider; mod l1; pub use l1::{ blob::{ - AnvilBlobProvider, BeaconClientProvider, BlobProvider, BlobSource, MockBeaconProvider, - S3BlobProvider, + AnvilBlobProvider, BeaconClientProvider, BlobProvider, BlobProviders, BlobProvidersBuilder, + MockBeaconProvider, S3BlobProvider, }, message::{DatabaseL1MessageProvider, L1MessageProvider}, system_contract::{SystemContractProvider, AUTHORIZED_SIGNER_STORAGE_SLOT}, diff --git a/crates/sequencer/tests/e2e.rs b/crates/sequencer/tests/e2e.rs index cd6851c5..73e7fc14 100644 --- a/crates/sequencer/tests/e2e.rs +++ b/crates/sequencer/tests/e2e.rs @@ -11,13 +11,13 @@ use reth_scroll_node::test_utils::setup; use rollup_node::{ constants::SCROLL_GAS_LIMIT, test_utils::{default_test_scroll_rollup_node_config, setup_engine}, - BeaconProviderArgs, ChainOrchestratorArgs, ConsensusArgs, DatabaseArgs, EngineDriverArgs, + BlobProviderArgs, ChainOrchestratorArgs, ConsensusArgs, DatabaseArgs, EngineDriverArgs, GasPriceOracleArgs, L1ProviderArgs, NetworkArgs, ScrollRollupNodeConfig, SequencerArgs, SignerArgs, }; use rollup_node_manager::RollupManagerEvent; use rollup_node_primitives::{sig_encode_hash, BlockInfo, L1MessageEnvelope}; -use rollup_node_providers::{BlobSource, DatabaseL1MessageProvider, ScrollRootProvider}; +use rollup_node_providers::{DatabaseL1MessageProvider, ScrollRootProvider}; use rollup_node_sequencer::{L1MessageInclusionMode, Sequencer}; use rollup_node_signer::SignerEvent; use scroll_alloy_consensus::TxL1Message; @@ -509,10 +509,7 @@ async fn can_sequence_blocks_with_private_key_file() -> eyre::Result<()> { allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: SignerArgs { key_file: Some(temp_file.path().to_path_buf()), aws_kms_key_id: None, @@ -603,10 +600,7 @@ async fn can_sequence_blocks_with_hex_key_file_without_prefix() -> eyre::Result< allow_empty_blocks: true, ..SequencerArgs::default() }, - beacon_provider_args: BeaconProviderArgs { - blob_source: BlobSource::Mock, - ..Default::default() - }, + blob_provider_args: BlobProviderArgs { mock: true, ..Default::default() }, signer_args: SignerArgs { key_file: Some(temp_file.path().to_path_buf()), aws_kms_key_id: None, diff --git a/tests/launch_rollup_node_follower.bash b/tests/launch_rollup_node_follower.bash index eb3bca81..b70f716f 100644 --- a/tests/launch_rollup_node_follower.bash +++ b/tests/launch_rollup_node_follower.bash @@ -12,4 +12,5 @@ exec rollup-node node --chain /l2reth/l2reth-genesis-e2e.json --datadir=/l2reth --rpc.max-connections=5000 \ --trusted-peers enode://8fc4f6dfd0a2ebf56560d0b0ef5e60ad7bcb01e13f929eae53a4c77086d9c1e74eb8b8c8945035d25c6287afdd871f0d41b3fd7e189697decd0f13538d1ac620@l2geth-sequencer:30303,enode://e7f7e271f62bd2b697add14e6987419758c97e83b0478bd948f5f2d271495728e7edef5bd78ad65258ac910f28e86928ead0c42ee51f2a0168d8ca23ba939766@rollup-node-sequencer:30303 \ --engine.sync-at-startup false \ - --l1.url http://l1-node:8545 + --l1.url http://l1-node:8545 \ + --blob.mock diff --git a/tests/launch_rollup_node_sequencer.bash b/tests/launch_rollup_node_sequencer.bash index 5e9d021b..0f78c01c 100644 --- a/tests/launch_rollup_node_sequencer.bash +++ b/tests/launch_rollup_node_sequencer.bash @@ -25,5 +25,5 @@ exec rollup-node node --chain /l2reth/l2reth-genesis-e2e.json --datadir=/l2reth --p2p-secret-key /l2reth/nodekey \ --trusted-peers enode://8fc4f6dfd0a2ebf56560d0b0ef5e60ad7bcb01e13f929eae53a4c77086d9c1e74eb8b8c8945035d25c6287afdd871f0d41b3fd7e189697decd0f13538d1ac620@l2geth-sequencer:30303 \ --engine.sync-at-startup false \ - --l1.url http://l1-node:8545 - + --l1.url http://l1-node:8545 \ + --blob.mock