From 0f51698237118ad65df78fc4ea1825e0f8af5354 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 22 Jul 2025 15:57:25 +1000 Subject: [PATCH 01/18] Attempt to fix malloc_utils features --- common/malloc_utils/Cargo.toml | 14 ++++++++++---- common/malloc_utils/src/lib.rs | 35 +++++++++++++++++++++------------- lighthouse/Cargo.toml | 14 +++----------- lighthouse/src/main.rs | 10 +--------- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml index 89973493b4e..d01118caf04 100644 --- a/common/malloc_utils/Cargo.toml +++ b/common/malloc_utils/Cargo.toml @@ -4,23 +4,29 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = { workspace = true } +# Features are not rich enough to express the complexity of our defaults, so we choose to always +# compile the jemalloc dependency, but avoid using it if the `sysmalloc` feature is set or it +# is unsupported by the target platform (Windows). [features] +default = [] mallinfo2 = [] -jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] jemalloc-profiling = ["tikv-jemallocator/profiling"] +# Force the use of system malloc (or glibc) rather than jemalloc. +# This is a no-op on Windows where jemalloc is always disabled. +sysmalloc = [] [dependencies] libc = "0.2.79" metrics = { workspace = true } parking_lot = { workspace = true } -tikv-jemalloc-ctl = { version = "0.6.0", optional = true, features = ["stats"] } +tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } [target.'cfg(not(target_os = "linux"))'.dependencies] -tikv-jemallocator = { version = "0.6.0", optional = true, features = ["stats"] } +tikv-jemallocator = { version = "0.6.0", features = ["stats"] } # Jemalloc's background_threads feature requires Linux (pthreads). [target.'cfg(target_os = "linux")'.dependencies] -tikv-jemallocator = { version = "0.6.0", optional = true, features = [ +tikv-jemallocator = { version = "0.6.0", features = [ "stats", "background_threads", ] } diff --git a/common/malloc_utils/src/lib.rs b/common/malloc_utils/src/lib.rs index 50d2785a744..686f3159830 100644 --- a/common/malloc_utils/src/lib.rs +++ b/common/malloc_utils/src/lib.rs @@ -24,29 +24,27 @@ //! detecting `glibc` are best-effort. If this crate throws errors about undefined external //! functions, then try to compile with the `not_glibc_interface` module. -#[cfg(all( - target_os = "linux", - not(target_env = "musl"), - not(feature = "jemalloc") -))] +#[cfg(all(target_os = "linux", not(target_env = "musl"), feature = "sysmalloc"))] pub mod glibc; -#[cfg(feature = "jemalloc")] +#[cfg(all(unix, not(feature = "sysmalloc")))] pub mod jemalloc; pub use interface::*; -#[cfg(all( - target_os = "linux", - not(target_env = "musl"), - not(feature = "jemalloc") -))] +// Glibc malloc is the default on non-musl Linux if the sysmalloc feature is enabled. +#[cfg(all(target_os = "linux", not(target_env = "musl"), feature = "sysmalloc"))] mod interface { pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator; pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics; + + pub fn allocator_name() -> String { + "glibc".to_string() + } } -#[cfg(feature = "jemalloc")] +// Jemalloc is the default on UNIX (including musl) unless the sysmalloc feature is enabled. +#[cfg(all(unix, not(feature = "sysmalloc")))] mod interface { #[allow(dead_code)] pub fn configure_memory_allocator() -> Result<(), String> { @@ -54,11 +52,18 @@ mod interface { } pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics; + + pub fn allocator_name() -> String { + match crate::jemalloc::page_size() { + Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024), + Err(e) => format!("jemalloc (error: {e:?})"), + } + } } #[cfg(all( any(not(target_os = "linux"), target_env = "musl"), - not(feature = "jemalloc") + feature = "sysmalloc" ))] mod interface { #[allow(dead_code, clippy::unnecessary_wraps)] @@ -68,4 +73,8 @@ mod interface { #[allow(dead_code)] pub fn scrape_allocator_metrics() {} + + pub fn allocator_name() -> String { + "system".to_string() + } } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index a6549f55744..f3ab4e244bf 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Sigma Prime "] edition = { workspace = true } autotests = false rust-version = "1.83.0" +resolver = "2" # Prevent cargo-udeps from flagging the dummy package `target_check`, which exists only # to assert properties of the compilation target. @@ -35,11 +36,8 @@ beacon-node-leveldb = ["store/leveldb"] beacon-node-redb = ["store/redb"] # Supports console subscriber for debugging console-subscriber = ["console-subscriber/default"] -# Turns off jemalloc so that heaptrack may be used to analyse memory usage. -heaptrack = [] - -# Deprecated. This is now enabled by default on non windows targets (unless heaptrack is enabled). -jemalloc = [] +# Force the use of the system memory allocator rather than jemalloc. +sysmalloc = ["malloc_utils/sysmalloc"] [dependencies] account_manager = { "path" = "../account_manager" } @@ -73,12 +71,6 @@ unused_port = { workspace = true } validator_client = { workspace = true } validator_manager = { path = "../validator_manager" } -[target.'cfg(any(target_os = "windows", features = "heaptrack"))'.dependencies] -malloc_utils = { workspace = true } - -[target.'cfg(not(any(target_os = "windows", features = "heaptrack")))'.dependencies] -malloc_utils = { workspace = true, features = ["jemalloc"] } - [dev-dependencies] beacon_node_fallback = { workspace = true } beacon_processor = { workspace = true } diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 28b8c7b8fc4..a3a1da7b60d 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -74,15 +74,7 @@ fn bls_hardware_acceleration() -> bool { } fn allocator_name() -> String { - #[cfg(any(feature = "heaptrack", target_os = "windows"))] - { - "system".to_string() - } - #[cfg(not(any(feature = "heaptrack", target_os = "windows")))] - match malloc_utils::jemalloc::page_size() { - Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024), - Err(e) => format!("jemalloc (error: {e:?})"), - } + malloc_utils::allocator_name() } fn build_profile_name() -> String { From dc7ca5a62d03c90ce447c41c21f877d28ccad2a7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 22 Jul 2025 16:33:51 +1000 Subject: [PATCH 02/18] Fix lcli --- Makefile | 2 +- lcli/Cargo.toml | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a2d07a87ee2..15d61b1515d 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ BUILD_PATH_RISCV64 = "target/$(RISCV64_TAG)/release" PINNED_NIGHTLY ?= nightly # List of features to use when cross-compiling. Can be overridden via the environment. -CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc,beacon-node-leveldb,beacon-node-redb +CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,beacon-node-leveldb,beacon-node-redb # Cargo profile for Cross builds. Default is for local builds, CI uses an override. CROSS_PROFILE ?= release diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index a54c10dc680..a5fea986a21 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -5,13 +5,9 @@ version = "7.1.0" authors = ["Paul Hauner "] edition = { workspace = true } -[package.metadata.cargo-udeps.ignore] -normal = ["malloc_utils"] - [features] portable = ["bls/supranational-portable"] fake_crypto = ['bls/fake_crypto'] -jemalloc = ["malloc_utils/jemalloc"] [dependencies] account_utils = { workspace = true } @@ -32,7 +28,6 @@ hex = { workspace = true } lighthouse_network = { workspace = true } lighthouse_version = { workspace = true } log = { workspace = true } -malloc_utils = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } From 8b61b0809ac27c3218df6ea8fa0f03ab3949fdab Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 23 Jul 2025 09:56:24 +1000 Subject: [PATCH 03/18] Update cargo lock --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index af5d63e97f4..36376c8a6e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5070,7 +5070,6 @@ dependencies = [ "lighthouse_network", "lighthouse_version", "log", - "malloc_utils", "rayon", "serde", "serde_json", From ebdf9a5145e25191127cb722b21b7c0f2220094a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 23 Jul 2025 15:47:02 +1000 Subject: [PATCH 04/18] Make jemalloc optional again --- common/malloc_utils/Cargo.toml | 13 +++++++++---- lighthouse/Cargo.toml | 7 ++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml index d01118caf04..5cada3b9652 100644 --- a/common/malloc_utils/Cargo.toml +++ b/common/malloc_utils/Cargo.toml @@ -8,8 +8,13 @@ edition = { workspace = true } # compile the jemalloc dependency, but avoid using it if the `sysmalloc` feature is set or it # is unsupported by the target platform (Windows). [features] -default = [] +default = ["jemalloc"] mallinfo2 = [] +# The jemalloc feature enables the compilation of jemalloc dependencies. Jemalloc is also the +# default allocator, unless `sysmalloc` is set. +# +# It should be turned off on Windows. +jemalloc = ["tikv-jemalloc-ctl", "tikv-jemallocator"] jemalloc-profiling = ["tikv-jemallocator/profiling"] # Force the use of system malloc (or glibc) rather than jemalloc. # This is a no-op on Windows where jemalloc is always disabled. @@ -19,14 +24,14 @@ sysmalloc = [] libc = "0.2.79" metrics = { workspace = true } parking_lot = { workspace = true } -tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } +tikv-jemalloc-ctl = { version = "0.6.0", optional = true, features = ["stats"] } [target.'cfg(not(target_os = "linux"))'.dependencies] -tikv-jemallocator = { version = "0.6.0", features = ["stats"] } +tikv-jemallocator = { version = "0.6.0", optional = true, features = ["stats"] } # Jemalloc's background_threads feature requires Linux (pthreads). [target.'cfg(target_os = "linux")'.dependencies] -tikv-jemallocator = { version = "0.6.0", features = [ +tikv-jemallocator = { version = "0.6.0", optional = true, features = [ "stats", "background_threads", ] } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index f3ab4e244bf..7f6692e9a22 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -56,7 +56,6 @@ ethereum_hashing = { workspace = true } futures = { workspace = true } lighthouse_version = { workspace = true } logging = { workspace = true } -malloc_utils = { workspace = true } metrics = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -71,6 +70,12 @@ unused_port = { workspace = true } validator_client = { workspace = true } validator_manager = { path = "../validator_manager" } +[target.'cfg(not(target_os = "windows"))'.dependencies] +malloc_utils = { workspace = true, default-features = true } + +[target.'cfg(target_os = "windows")'.dependencies] +malloc_utils = { workspace = true, default-features = false } + [dev-dependencies] beacon_node_fallback = { workspace = true } beacon_processor = { workspace = true } From 68df20f546fb770b1cd62d576a00c3029b149ea0 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 23 Jul 2025 07:05:36 +1000 Subject: [PATCH 05/18] Fix cfg conditions and default-features --- Cargo.toml | 2 +- common/malloc_utils/src/lib.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6737ff22c5e..211f0e1d0c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -180,7 +180,7 @@ logging = { path = "common/logging" } logroller = "0.1.8" lru = "0.12" lru_cache = { path = "common/lru_cache" } -malloc_utils = { path = "common/malloc_utils" } +malloc_utils = { path = "common/malloc_utils", default-features = false } maplit = "1" merkle_proof = { path = "consensus/merkle_proof" } metrics = { path = "common/metrics" } diff --git a/common/malloc_utils/src/lib.rs b/common/malloc_utils/src/lib.rs index 686f3159830..fded6e7ac53 100644 --- a/common/malloc_utils/src/lib.rs +++ b/common/malloc_utils/src/lib.rs @@ -24,7 +24,7 @@ //! detecting `glibc` are best-effort. If this crate throws errors about undefined external //! functions, then try to compile with the `not_glibc_interface` module. -#[cfg(all(target_os = "linux", not(target_env = "musl"), feature = "sysmalloc"))] +#[cfg(all(feature = "sysmalloc", target_os = "linux", not(target_env = "musl")))] pub mod glibc; #[cfg(all(unix, not(feature = "sysmalloc")))] @@ -33,7 +33,7 @@ pub mod jemalloc; pub use interface::*; // Glibc malloc is the default on non-musl Linux if the sysmalloc feature is enabled. -#[cfg(all(target_os = "linux", not(target_env = "musl"), feature = "sysmalloc"))] +#[cfg(all(feature = "sysmalloc", target_os = "linux", not(target_env = "musl")))] mod interface { pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator; pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics; @@ -61,9 +61,9 @@ mod interface { } } -#[cfg(all( - any(not(target_os = "linux"), target_env = "musl"), - feature = "sysmalloc" +#[cfg(any( + not(unix), + all(feature = "sysmalloc", any(not(target_os = "linux"), target_env = "musl")) ))] mod interface { #[allow(dead_code, clippy::unnecessary_wraps)] From 2bd64dd3916a16ac0f74853c182dcb85805265aa Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 10:38:56 +1000 Subject: [PATCH 06/18] Turn off jemalloc feature by default --- Cargo.toml | 2 +- common/malloc_utils/Cargo.toml | 10 ++++++---- lighthouse/Cargo.toml | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 211f0e1d0c5..6737ff22c5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -180,7 +180,7 @@ logging = { path = "common/logging" } logroller = "0.1.8" lru = "0.12" lru_cache = { path = "common/lru_cache" } -malloc_utils = { path = "common/malloc_utils", default-features = false } +malloc_utils = { path = "common/malloc_utils" } maplit = "1" merkle_proof = { path = "consensus/merkle_proof" } metrics = { path = "common/metrics" } diff --git a/common/malloc_utils/Cargo.toml b/common/malloc_utils/Cargo.toml index 5cada3b9652..39c7137d4cb 100644 --- a/common/malloc_utils/Cargo.toml +++ b/common/malloc_utils/Cargo.toml @@ -4,11 +4,13 @@ version = "0.1.0" authors = ["Paul Hauner "] edition = { workspace = true } -# Features are not rich enough to express the complexity of our defaults, so we choose to always -# compile the jemalloc dependency, but avoid using it if the `sysmalloc` feature is set or it -# is unsupported by the target platform (Windows). +# Features are not rich enough to express the complexity of our defaults, so we choose to just +# use the jemalloc feature to control whether the dependency is compiled, but avoid using it if +# the `sysmalloc` feature is set. +# +# On Windows, setting the jemalloc feature will result in a compile-time error. [features] -default = ["jemalloc"] +default = [] mallinfo2 = [] # The jemalloc feature enables the compilation of jemalloc dependencies. Jemalloc is also the # default allocator, unless `sysmalloc` is set. diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 7f6692e9a22..9de8f7f3245 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -71,10 +71,10 @@ validator_client = { workspace = true } validator_manager = { path = "../validator_manager" } [target.'cfg(not(target_os = "windows"))'.dependencies] -malloc_utils = { workspace = true, default-features = true } +malloc_utils = { workspace = true, features = ["jemalloc"] } [target.'cfg(target_os = "windows")'.dependencies] -malloc_utils = { workspace = true, default-features = false } +malloc_utils = { workspace = true, features = [] } [dev-dependencies] beacon_node_fallback = { workspace = true } From 399b328dcc3a6c54bae820d928fa6692161bbc69 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 11:33:05 +1000 Subject: [PATCH 07/18] Fix compilation when jemalloc feature is off --- common/malloc_utils/src/lib.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/common/malloc_utils/src/lib.rs b/common/malloc_utils/src/lib.rs index fded6e7ac53..9a5ea3c2bae 100644 --- a/common/malloc_utils/src/lib.rs +++ b/common/malloc_utils/src/lib.rs @@ -24,16 +24,25 @@ //! detecting `glibc` are best-effort. If this crate throws errors about undefined external //! functions, then try to compile with the `not_glibc_interface` module. -#[cfg(all(feature = "sysmalloc", target_os = "linux", not(target_env = "musl")))] +#[cfg(all( + any(feature = "sysmalloc", not(feature = "jemalloc")), + target_os = "linux", + not(target_env = "musl") +))] pub mod glibc; -#[cfg(all(unix, not(feature = "sysmalloc")))] +#[cfg(all(unix, not(feature = "sysmalloc"), feature = "jemalloc"))] pub mod jemalloc; pub use interface::*; -// Glibc malloc is the default on non-musl Linux if the sysmalloc feature is enabled. -#[cfg(all(feature = "sysmalloc", target_os = "linux", not(target_env = "musl")))] +// Glibc malloc is the default on non-musl Linux if the sysmalloc feature is enabled, or jemalloc +// is disabled. +#[cfg(all( + any(feature = "sysmalloc", not(feature = "jemalloc")), + target_os = "linux", + not(target_env = "musl") +))] mod interface { pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator; pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics; @@ -44,7 +53,7 @@ mod interface { } // Jemalloc is the default on UNIX (including musl) unless the sysmalloc feature is enabled. -#[cfg(all(unix, not(feature = "sysmalloc")))] +#[cfg(all(unix, not(feature = "sysmalloc"), feature = "jemalloc"))] mod interface { #[allow(dead_code)] pub fn configure_memory_allocator() -> Result<(), String> { @@ -63,7 +72,10 @@ mod interface { #[cfg(any( not(unix), - all(feature = "sysmalloc", any(not(target_os = "linux"), target_env = "musl")) + all( + any(feature = "sysmalloc", not(feature = "jemalloc")), + any(not(target_os = "linux"), target_env = "musl") + ) ))] mod interface { #[allow(dead_code, clippy::unnecessary_wraps)] From d8ea35c3d277c5f920124099a6f8504cd8448c08 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 11:38:31 +1000 Subject: [PATCH 08/18] Revert unnecessary changes in lcli (keep jemalloc) --- lcli/Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index a5fea986a21..a54c10dc680 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -5,9 +5,13 @@ version = "7.1.0" authors = ["Paul Hauner "] edition = { workspace = true } +[package.metadata.cargo-udeps.ignore] +normal = ["malloc_utils"] + [features] portable = ["bls/supranational-portable"] fake_crypto = ['bls/fake_crypto'] +jemalloc = ["malloc_utils/jemalloc"] [dependencies] account_utils = { workspace = true } @@ -28,6 +32,7 @@ hex = { workspace = true } lighthouse_network = { workspace = true } lighthouse_version = { workspace = true } log = { workspace = true } +malloc_utils = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } From 1a7b518ded2b6de3059100676ce78463505ac145 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 11:42:36 +1000 Subject: [PATCH 09/18] Ensure lcli uses jemalloc on Linux --- lcli/Cargo.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index a54c10dc680..da9d2d3ade2 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -11,7 +11,6 @@ normal = ["malloc_utils"] [features] portable = ["bls/supranational-portable"] fake_crypto = ['bls/fake_crypto'] -jemalloc = ["malloc_utils/jemalloc"] [dependencies] account_utils = { workspace = true } @@ -32,7 +31,6 @@ hex = { workspace = true } lighthouse_network = { workspace = true } lighthouse_version = { workspace = true } log = { workspace = true } -malloc_utils = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -45,3 +43,9 @@ tracing-subscriber = { workspace = true } tree_hash = { workspace = true } types = { workspace = true } validator_dir = { workspace = true } + +[target.'cfg(not(target_os = "windows"))'.dependencies] +malloc_utils = { workspace = true, features = ["jemalloc"] } + +[target.'cfg(target_os = "windows")'.dependencies] +malloc_utils = { workspace = true, features = [] } From 3545196df7d69bd752949faba9e6e5662f7cbafc Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 11:44:05 +1000 Subject: [PATCH 10/18] Fix cargo lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index c54b7290846..3786392feb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5070,6 +5070,7 @@ dependencies = [ "lighthouse_network", "lighthouse_version", "log", + "malloc_utils", "rayon", "serde", "serde_json", From 94d9f38ad8bf64d11d7b8fa942b2b061e617337f Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 12:47:50 +1000 Subject: [PATCH 11/18] Fix book --- book/src/installation_source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/book/src/installation_source.md b/book/src/installation_source.md index 0aa8a99a5e5..023833c149b 100644 --- a/book/src/installation_source.md +++ b/book/src/installation_source.md @@ -166,8 +166,8 @@ Commonly used features include: - `slasher-lmdb`: support for the LMDB slasher backend. Enabled by default. - `slasher-mdbx`: support for the MDBX slasher backend. - `beacon-node-leveldb`: support for the leveldb backend. Enabled by default. -- `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS. - Not supported on Windows. +- `sysmalloc`: use the system memory allocator rather than jemalloc. This is always enabled on + Windows. - `spec-minimal`: support for the minimal preset (useful for testing). Default features (e.g. `slasher-lmdb`, `beacon-node-leveldb`) may be opted out of using the `--no-default-features` From f2780b0111ea5e51d08740b4b17c9a75159e9d70 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 25 Jul 2025 16:10:49 +1000 Subject: [PATCH 12/18] mdlint --- book/src/installation_source.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/book/src/installation_source.md b/book/src/installation_source.md index 023833c149b..f035d1d843b 100644 --- a/book/src/installation_source.md +++ b/book/src/installation_source.md @@ -178,8 +178,6 @@ E.g. CARGO_INSTALL_EXTRA_FLAGS="--no-default-features" make ``` -[jemalloc]: https://jemalloc.net/ - ## Compilation Profiles You can customise the compiler settings used to compile Lighthouse via From 8de9157716de9eaf5fd598c7a1dec7628c85cdd6 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 28 Jul 2025 09:55:09 -0700 Subject: [PATCH 13/18] fix ssz formatting --- beacon_node/http_api/src/light_client.rs | 23 ++++++++------ beacon_node/http_api/tests/tests.rs | 20 ++++++++++++ common/eth2/src/lib.rs | 26 +++++++++++++++ common/eth2/src/types.rs | 40 ++++++++++++++++++++---- 4 files changed, 93 insertions(+), 16 deletions(-) diff --git a/beacon_node/http_api/src/light_client.rs b/beacon_node/http_api/src/light_client.rs index f9559d738ea..1b97191bdc1 100644 --- a/beacon_node/http_api/src/light_client.rs +++ b/beacon_node/http_api/src/light_client.rs @@ -34,13 +34,15 @@ pub fn get_light_client_updates( match accept_header { Some(api_types::Accept::Ssz) => { let response_chunks = light_client_updates - .iter() - .map(|update| map_light_client_update_to_ssz_chunk::(&chain, update)) - .collect::>(); + .into_iter() + .flat_map(|update| { + map_light_client_update_to_response_chunk::(&chain, update).as_ssz_bytes() + }) + .collect(); Response::builder() .status(200) - .body(response_chunks.as_ssz_bytes()) + .body(response_chunks) .map(|res: Response>| add_ssz_content_type_header(res)) .map_err(|e| { warp_utils::reject::custom_server_error(format!( @@ -146,23 +148,24 @@ pub fn validate_light_client_updates_request( Ok(()) } -fn map_light_client_update_to_ssz_chunk( +fn map_light_client_update_to_response_chunk( chain: &BeaconChain, - light_client_update: &LightClientUpdate, -) -> LightClientUpdateResponseChunk { + light_client_update: LightClientUpdate, +) -> LightClientUpdateResponseChunk { let epoch = light_client_update .attested_header_slot() .epoch(T::EthSpec::slots_per_epoch()); let fork_digest = chain.compute_fork_digest(epoch); - let payload = light_client_update.as_ssz_bytes(); - let response_chunk_len = fork_digest.len() + payload.len(); + let response_chunk_len = fork_digest.len() + light_client_update.ssz_bytes_len(); let response_chunk = LightClientUpdateResponseChunkInner { context: fork_digest, - payload, + payload: light_client_update, }; + println!("response chunk len {:?}", response_chunk_len); + LightClientUpdateResponseChunk { response_chunk_len: response_chunk_len as u64, response_chunk, diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 5ac8cd91864..cceb1566468 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2213,6 +2213,24 @@ impl ApiTester { self } + pub async fn test_get_beacon_light_client_updates_ssz(self) -> Self { + let current_epoch = self.chain.epoch().unwrap(); + let current_sync_committee_period = current_epoch + .sync_committee_period(&self.chain.spec) + .unwrap(); + + match self + .client + .get_beacon_light_client_updates_ssz::(current_sync_committee_period, 1) + .await + { + Ok(result) => result, + Err(e) => panic!("query failed incorrectly: {e:?}"), + }; + + self + } + pub async fn test_get_beacon_light_client_updates(self) -> Self { let current_epoch = self.chain.epoch().unwrap(); let current_sync_committee_period = current_epoch @@ -7048,6 +7066,8 @@ async fn get_light_client_updates() { ApiTester::new_from_config(config) .await .test_get_beacon_light_client_updates() + .await + .test_get_beacon_light_client_updates_ssz() .await; } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index a129f9c4fa5..6b0c6824fd8 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -981,6 +981,32 @@ impl BeaconNodeHttpClient { }) } + /// `GET beacon/light_client/updates` + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_beacon_light_client_updates_ssz( + &self, + start_period: u64, + count: u64, + ) -> Result>, Error> { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("light_client") + .push("updates"); + + path.query_pairs_mut() + .append_pair("start_period", &start_period.to_string()); + + path.query_pairs_mut() + .append_pair("count", &count.to_string()); + + self.get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_beacon_blocks_ssz) + .await + } + /// `GET beacon/light_client/bootstrap` /// /// Returns `Ok(None)` on a 404 error. diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 07b5cb50166..1186c318ab6 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -11,6 +11,7 @@ use multiaddr::Multiaddr; use reqwest::header::HeaderMap; use serde::{Deserialize, Deserializer, Serialize}; use serde_utils::quoted_u64::Quoted; +use ssz::Encode; use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; use std::fmt::{self, Display}; @@ -823,16 +824,43 @@ pub struct LightClientUpdatesQuery { pub count: u64, } -#[derive(Encode, Decode)] -pub struct LightClientUpdateResponseChunk { +pub struct LightClientUpdateResponseChunk { pub response_chunk_len: u64, - pub response_chunk: LightClientUpdateResponseChunkInner, + pub response_chunk: LightClientUpdateResponseChunkInner, } -#[derive(Encode, Decode)] -pub struct LightClientUpdateResponseChunkInner { +impl Encode for LightClientUpdateResponseChunk { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_bytes_len(&self) -> usize { + 0_u64.ssz_bytes_len() + self.response_chunk.context.len() + + match &self.response_chunk.payload { + LightClientUpdate::Altair(lc) => lc.ssz_bytes_len(), + LightClientUpdate::Capella(lc) => lc.ssz_bytes_len(), + LightClientUpdate::Deneb(lc) => lc.ssz_bytes_len(), + LightClientUpdate::Electra(lc) => lc.ssz_bytes_len(), + LightClientUpdate::Fulu(lc) => lc.ssz_bytes_len(), + } + } + + fn ssz_append(&self, buf: &mut Vec) { + buf.extend_from_slice(&self.response_chunk_len.to_le_bytes()); + buf.extend_from_slice(&self.response_chunk.context); + match &self.response_chunk.payload { + LightClientUpdate::Altair(lc) => lc.ssz_append(buf), + LightClientUpdate::Capella(lc) => lc.ssz_append(buf), + LightClientUpdate::Deneb(lc) => lc.ssz_append(buf), + LightClientUpdate::Electra(lc) => lc.ssz_append(buf), + LightClientUpdate::Fulu(lc) => lc.ssz_append(buf), + }; + } +} + +pub struct LightClientUpdateResponseChunkInner { pub context: [u8; 4], - pub payload: Vec, + pub payload: LightClientUpdate, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] From 0165b6eb83a88bb449d4dd95212497116773f9fa Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 29 Jul 2025 11:24:07 -0700 Subject: [PATCH 14/18] fmt --- common/eth2/src/types.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 1186c318ab6..d86d311b0e1 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -835,7 +835,8 @@ impl Encode for LightClientUpdateResponseChunk { } fn ssz_bytes_len(&self) -> usize { - 0_u64.ssz_bytes_len() + self.response_chunk.context.len() + 0_u64.ssz_bytes_len() + + self.response_chunk.context.len() + match &self.response_chunk.payload { LightClientUpdate::Altair(lc) => lc.ssz_bytes_len(), LightClientUpdate::Capella(lc) => lc.ssz_bytes_len(), From ce33753b77398bb127690fd114d6293e0a77277a Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 5 Aug 2025 14:30:08 -0700 Subject: [PATCH 15/18] remove print --- beacon_node/http_api/src/light_client.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/beacon_node/http_api/src/light_client.rs b/beacon_node/http_api/src/light_client.rs index 1b97191bdc1..8fd47c52f29 100644 --- a/beacon_node/http_api/src/light_client.rs +++ b/beacon_node/http_api/src/light_client.rs @@ -164,8 +164,6 @@ fn map_light_client_update_to_response_chunk( payload: light_client_update, }; - println!("response chunk len {:?}", response_chunk_len); - LightClientUpdateResponseChunk { response_chunk_len: response_chunk_len as u64, response_chunk, From 79975a0999179b5d1a505100faff7f3ac6be1ed2 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 12 Aug 2025 10:51:06 +1000 Subject: [PATCH 16/18] Fix wordlist --- wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/wordlist.txt b/wordlist.txt index fdb2f43e42c..0391af78cb7 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -1,3 +1,4 @@ +allocator APIs ARMv AUR From 67b45a9ea6b15d710e028a184879b1fbf96dbbe6 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Tue, 12 Aug 2025 17:42:28 -0700 Subject: [PATCH 17/18] clean up --- common/eth2/src/lib.rs | 2 +- common/eth2/src/types.rs | 16 ++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 6b0c6824fd8..bf0694dcedd 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -1003,7 +1003,7 @@ impl BeaconNodeHttpClient { path.query_pairs_mut() .append_pair("count", &count.to_string()); - self.get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.get_beacon_blocks_ssz) + self.get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.default) .await } diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index d86d311b0e1..0a60d0bc73a 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -837,25 +837,13 @@ impl Encode for LightClientUpdateResponseChunk { fn ssz_bytes_len(&self) -> usize { 0_u64.ssz_bytes_len() + self.response_chunk.context.len() - + match &self.response_chunk.payload { - LightClientUpdate::Altair(lc) => lc.ssz_bytes_len(), - LightClientUpdate::Capella(lc) => lc.ssz_bytes_len(), - LightClientUpdate::Deneb(lc) => lc.ssz_bytes_len(), - LightClientUpdate::Electra(lc) => lc.ssz_bytes_len(), - LightClientUpdate::Fulu(lc) => lc.ssz_bytes_len(), - } + + self.response_chunk.payload.ssz_bytes_len() } fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&self.response_chunk_len.to_le_bytes()); buf.extend_from_slice(&self.response_chunk.context); - match &self.response_chunk.payload { - LightClientUpdate::Altair(lc) => lc.ssz_append(buf), - LightClientUpdate::Capella(lc) => lc.ssz_append(buf), - LightClientUpdate::Deneb(lc) => lc.ssz_append(buf), - LightClientUpdate::Electra(lc) => lc.ssz_append(buf), - LightClientUpdate::Fulu(lc) => lc.ssz_append(buf), - }; + self.response_chunk.payload.ssz_append(buf); } } From cd12c14847f0d354c4abd739fb97bcc69ac03d80 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 13 Aug 2025 14:10:56 +1000 Subject: [PATCH 18/18] Yeet env_logger into the sun --- Cargo.lock | 46 +------------------ Cargo.toml | 1 - consensus/state_processing/Cargo.toml | 1 - .../src/per_epoch_processing/tests.rs | 3 -- lcli/Cargo.toml | 1 - lcli/src/main.rs | 18 ++++++-- 6 files changed, 15 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c8e4f1bd056..79d332f9ea4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -709,17 +709,6 @@ dependencies = [ "url", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "auto_impl" version = "0.5.0" @@ -2743,19 +2732,6 @@ dependencies = [ "regex", ] -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "environment" version = "0.1.2" @@ -4054,15 +4030,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.9" @@ -5058,7 +5025,6 @@ dependencies = [ "clap", "clap_utils", "deposit_contract", - "env_logger 0.9.3", "environment", "eth2", "eth2_network_config", @@ -7491,7 +7457,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "env_logger 0.8.4", + "env_logger", "log", "rand 0.8.5", ] @@ -8961,7 +8927,6 @@ dependencies = [ "beacon_chain", "bls", "derivative", - "env_logger 0.9.3", "ethereum_hashing", "ethereum_ssz", "ethereum_ssz_derive", @@ -9262,15 +9227,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.4.2" diff --git a/Cargo.toml b/Cargo.toml index 38edb126a03..087cab6f8b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,7 +136,6 @@ dirs = "3" discv5 = { version = "0.9", features = ["libp2p"] } doppelganger_service = { path = "validator_client/doppelganger_service" } either = "1.9" -env_logger = "0.9" environment = { path = "lighthouse/environment" } eth2 = { path = "common/eth2" } eth2_config = { path = "common/eth2_config" } diff --git a/consensus/state_processing/Cargo.toml b/consensus/state_processing/Cargo.toml index 3acf491f23e..fc55bde9c67 100644 --- a/consensus/state_processing/Cargo.toml +++ b/consensus/state_processing/Cargo.toml @@ -41,5 +41,4 @@ types = { workspace = true } [dev-dependencies] beacon_chain = { workspace = true } -env_logger = { workspace = true } tokio = { workspace = true } diff --git a/consensus/state_processing/src/per_epoch_processing/tests.rs b/consensus/state_processing/src/per_epoch_processing/tests.rs index 6dd3f316c13..f042e8766c6 100644 --- a/consensus/state_processing/src/per_epoch_processing/tests.rs +++ b/consensus/state_processing/src/per_epoch_processing/tests.rs @@ -3,13 +3,10 @@ use crate::per_epoch_processing::process_epoch; use beacon_chain::test_utils::BeaconChainHarness; use beacon_chain::types::{EthSpec, MinimalEthSpec}; use bls::{FixedBytesExtended, Hash256}; -use env_logger::{Builder, Env}; use types::Slot; #[tokio::test] async fn runs_without_error() { - Builder::from_env(Env::default().default_filter_or("error")).init(); - let harness = BeaconChainHarness::builder(MinimalEthSpec) .default_spec() .deterministic_keypairs(8) diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index a54c10dc680..a95b45089ce 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -20,7 +20,6 @@ bls = { workspace = true } clap = { workspace = true } clap_utils = { workspace = true } deposit_contract = { workspace = true } -env_logger = { workspace = true } environment = { workspace = true } eth2 = { workspace = true } eth2_network_config = { workspace = true } diff --git a/lcli/src/main.rs b/lcli/src/main.rs index fb471914dab..a21dfd4386c 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -18,12 +18,10 @@ use parse_ssz::run_parse_ssz; use std::path::PathBuf; use std::process; use std::str::FromStr; -use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::{filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt}; use types::{EthSpec, EthSpecId}; fn main() { - env_logger::init(); - let matches = Command::new("Lighthouse CLI Tool") .version(lighthouse_version::VERSION) .display_order(0) @@ -653,7 +651,7 @@ fn main() { } fn run(env_builder: EnvironmentBuilder, matches: &ArgMatches) -> Result<(), String> { - let (env_builder, _file_logging_layer, _stdout_logging_layer, _sse_logging_layer_opt) = + let (env_builder, file_logging_layer, stdout_logging_layer, _sse_logging_layer_opt) = env_builder .multi_threaded_tokio_runtime() .map_err(|e| format!("should start tokio runtime: {:?}", e))? @@ -682,6 +680,18 @@ fn run(env_builder: EnvironmentBuilder, matches: &ArgMatches) -> .build() .map_err(|e| format!("should build env: {:?}", e))?; + let mut logging_layers = vec![file_logging_layer]; + if let Some(stdout) = stdout_logging_layer { + logging_layers.push(stdout); + } + let logging_result = tracing_subscriber::registry() + .with(logging_layers) + .try_init(); + + if let Err(e) = logging_result { + eprintln!("Failed to initialize logger: {e}"); + } + // Determine testnet-dir path or network name depending on CLI flags. let (testnet_dir, network_name) = if let Some(testnet_dir) = parse_optional::(matches, "testnet-dir")? {