merge queue: embarking unstable (5ebb44e), #7872, #7806 and #7770 together

Cargo.toml

@@ -136,7 +136,6 @@ dirs = "3"
 discv5 = { version = "0.9", features = ["libp2p"] }
 doppelganger_service = { path = "validator_client/doppelganger_service" }
 either = "1.9"
-env_logger = "0.9"
 environment = { path = "lighthouse/environment" }
 eth2 = { path = "common/eth2" }
 eth2_config = { path = "common/eth2_config" }

Makefile

@@ -16,7 +16,7 @@ BUILD_PATH_RISCV64 = "target/$(RISCV64_TAG)/release"
 PINNED_NIGHTLY ?= nightly
 
 # List of features to use when cross-compiling. Can be overridden via the environment.
-CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,jemalloc,beacon-node-leveldb,beacon-node-redb
+CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx,slasher-redb,beacon-node-leveldb,beacon-node-redb
 
 # Cargo profile for Cross builds. Default is for local builds, CI uses an override.
 CROSS_PROFILE ?= release

beacon_node/http_api/src/light_client.rs

@@ -34,13 +34,15 @@ pub fn get_light_client_updates<T: BeaconChainTypes>(
     match accept_header {
         Some(api_types::Accept::Ssz) => {
             let response_chunks = light_client_updates
-                .iter()
-                .map(|update| map_light_client_update_to_ssz_chunk::<T>(&chain, update))
-                .collect::<Vec<LightClientUpdateResponseChunk>>();
+                .into_iter()
+                .flat_map(|update| {
+                    map_light_client_update_to_response_chunk::<T>(&chain, update).as_ssz_bytes()
+                })
+                .collect();
 
             Response::builder()
                 .status(200)
-                .body(response_chunks.as_ssz_bytes())
+                .body(response_chunks)
                 .map(|res: Response<Vec<u8>>| add_ssz_content_type_header(res))
                 .map_err(|e| {
                     warp_utils::reject::custom_server_error(format!(
@@ -146,21 +148,20 @@ pub fn validate_light_client_updates_request<T: BeaconChainTypes>(
     Ok(())
 }
 
-fn map_light_client_update_to_ssz_chunk<T: BeaconChainTypes>(
+fn map_light_client_update_to_response_chunk<T: BeaconChainTypes>(
     chain: &BeaconChain<T>,
-    light_client_update: &LightClientUpdate<T::EthSpec>,
-) -> LightClientUpdateResponseChunk {
+    light_client_update: LightClientUpdate<T::EthSpec>,
+) -> LightClientUpdateResponseChunk<T::EthSpec> {
     let epoch = light_client_update
         .attested_header_slot()
         .epoch(T::EthSpec::slots_per_epoch());
     let fork_digest = chain.compute_fork_digest(epoch);
 
-    let payload = light_client_update.as_ssz_bytes();
-    let response_chunk_len = fork_digest.len() + payload.len();
+    let response_chunk_len = fork_digest.len() + light_client_update.ssz_bytes_len();
 
     let response_chunk = LightClientUpdateResponseChunkInner {
         context: fork_digest,
-        payload,
+        payload: light_client_update,
     };
 
     LightClientUpdateResponseChunk {

beacon_node/http_api/tests/tests.rs

@@ -2216,6 +2216,24 @@ impl ApiTester {
         self
     }
 
+    pub async fn test_get_beacon_light_client_updates_ssz(self) -> Self {
+        let current_epoch = self.chain.epoch().unwrap();
+        let current_sync_committee_period = current_epoch
+            .sync_committee_period(&self.chain.spec)
+            .unwrap();
+
+        match self
+            .client
+            .get_beacon_light_client_updates_ssz::<E>(current_sync_committee_period, 1)
+            .await
+        {
+            Ok(result) => result,
+            Err(e) => panic!("query failed incorrectly: {e:?}"),
+        };
+
+        self
+    }
+
     pub async fn test_get_beacon_light_client_updates(self) -> Self {
         let current_epoch = self.chain.epoch().unwrap();
         let current_sync_committee_period = current_epoch
@@ -7073,6 +7091,8 @@ async fn get_light_client_updates() {
     ApiTester::new_from_config(config)
         .await
         .test_get_beacon_light_client_updates()
+        .await
+        .test_get_beacon_light_client_updates_ssz()
         .await;
 }
 

book/src/installation_source.md

@@ -166,8 +166,8 @@ Commonly used features include:
 - `slasher-lmdb`: support for the LMDB slasher backend. Enabled by default.
 - `slasher-mdbx`: support for the MDBX slasher backend.
 - `beacon-node-leveldb`: support for the leveldb backend. Enabled by default.
-- `jemalloc`: use [`jemalloc`][jemalloc] to allocate memory. Enabled by default on Linux and macOS.
-  Not supported on Windows.
+- `sysmalloc`: use the system memory allocator rather than jemalloc. This is always enabled on
+  Windows.
 - `spec-minimal`: support for the minimal preset (useful for testing).
 
 Default features (e.g. `slasher-lmdb`, `beacon-node-leveldb`) may be opted out of using the `--no-default-features`
@@ -178,8 +178,6 @@ E.g.
 CARGO_INSTALL_EXTRA_FLAGS="--no-default-features" make
 ```
 
-[jemalloc]: https://jemalloc.net/
-
 ## Compilation Profiles
 
 You can customise the compiler settings used to compile Lighthouse via

common/eth2/src/lib.rs

@@ -981,6 +981,32 @@ impl BeaconNodeHttpClient {
         })
     }
 
+    /// `GET beacon/light_client/updates`
+    ///
+    /// Returns `Ok(None)` on a 404 error.
+    pub async fn get_beacon_light_client_updates_ssz<E: EthSpec>(
+        &self,
+        start_period: u64,
+        count: u64,
+    ) -> Result<Option<Vec<u8>>, Error> {
+        let mut path = self.eth_path(V1)?;
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("beacon")
+            .push("light_client")
+            .push("updates");
+
+        path.query_pairs_mut()
+            .append_pair("start_period", &start_period.to_string());
+
+        path.query_pairs_mut()
+            .append_pair("count", &count.to_string());
+
+        self.get_bytes_opt_accept_header(path, Accept::Ssz, self.timeouts.default)
+            .await
+    }
+
     /// `GET beacon/light_client/bootstrap`
     ///
     /// Returns `Ok(None)` on a 404 error.

common/eth2/src/types.rs

@@ -11,6 +11,7 @@ use multiaddr::Multiaddr;
 use reqwest::header::HeaderMap;
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_utils::quoted_u64::Quoted;
+use ssz::Encode;
 use ssz::{Decode, DecodeError};
 use ssz_derive::{Decode, Encode};
 use std::fmt::{self, Display};
@@ -823,16 +824,32 @@ pub struct LightClientUpdatesQuery {
     pub count: u64,
 }
 
-#[derive(Encode, Decode)]
-pub struct LightClientUpdateResponseChunk {
+pub struct LightClientUpdateResponseChunk<E: EthSpec> {
     pub response_chunk_len: u64,
-    pub response_chunk: LightClientUpdateResponseChunkInner,
+    pub response_chunk: LightClientUpdateResponseChunkInner<E>,
 }
 
-#[derive(Encode, Decode)]
-pub struct LightClientUpdateResponseChunkInner {
+impl<E: EthSpec> Encode for LightClientUpdateResponseChunk<E> {
+    fn is_ssz_fixed_len() -> bool {
+        false
+    }
+
+    fn ssz_bytes_len(&self) -> usize {
+        0_u64.ssz_bytes_len()
+            + self.response_chunk.context.len()
+            + self.response_chunk.payload.ssz_bytes_len()
+    }
+
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        buf.extend_from_slice(&self.response_chunk_len.to_le_bytes());
+        buf.extend_from_slice(&self.response_chunk.context);
+        self.response_chunk.payload.ssz_append(buf);
+    }
+}
+
+pub struct LightClientUpdateResponseChunkInner<E: EthSpec> {
     pub context: [u8; 4],
-    pub payload: Vec<u8>,
+    pub payload: LightClientUpdate<E>,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]

common/malloc_utils/Cargo.toml

@@ -4,10 +4,23 @@ version = "0.1.0"
 authors = ["Paul Hauner <[email protected]>"]
 edition = { workspace = true }
 
+# Features are not rich enough to express the complexity of our defaults, so we choose to just
+# use the jemalloc feature to control whether the dependency is compiled, but avoid using it if
+# the `sysmalloc` feature is set.
+#
+# On Windows, setting the jemalloc feature will result in a compile-time error.
 [features]
+default = []
 mallinfo2 = []
-jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"]
+# The jemalloc feature enables the compilation of jemalloc dependencies. Jemalloc is also the
+# default allocator, unless `sysmalloc` is set.
+#
+# It should be turned off on Windows.
+jemalloc = ["tikv-jemalloc-ctl", "tikv-jemallocator"]
 jemalloc-profiling = ["tikv-jemallocator/profiling"]
+# Force the use of system malloc (or glibc) rather than jemalloc.
+# This is a no-op on Windows where jemalloc is always disabled.
+sysmalloc = []
 
 [dependencies]
 libc = "0.2.79"

common/malloc_utils/src/lib.rs

@@ -25,40 +25,57 @@
 //! functions, then try to compile with the `not_glibc_interface` module.
 
 #[cfg(all(
+    any(feature = "sysmalloc", not(feature = "jemalloc")),
     target_os = "linux",
-    not(target_env = "musl"),
-    not(feature = "jemalloc")
+    not(target_env = "musl")
 ))]
 pub mod glibc;
 
-#[cfg(feature = "jemalloc")]
+#[cfg(all(unix, not(feature = "sysmalloc"), feature = "jemalloc"))]
 pub mod jemalloc;
 
 pub use interface::*;
 
+// Glibc malloc is the default on non-musl Linux if the sysmalloc feature is enabled, or jemalloc
+// is disabled.
 #[cfg(all(
+    any(feature = "sysmalloc", not(feature = "jemalloc")),
     target_os = "linux",
-    not(target_env = "musl"),
-    not(feature = "jemalloc")
+    not(target_env = "musl")
 ))]
 mod interface {
     pub use crate::glibc::configure_glibc_malloc as configure_memory_allocator;
     pub use crate::glibc::scrape_mallinfo_metrics as scrape_allocator_metrics;
+
+    pub fn allocator_name() -> String {
+        "glibc".to_string()
+    }
 }
 
-#[cfg(feature = "jemalloc")]
+// Jemalloc is the default on UNIX (including musl) unless the sysmalloc feature is enabled.
+#[cfg(all(unix, not(feature = "sysmalloc"), feature = "jemalloc"))]
 mod interface {
     #[allow(dead_code)]
     pub fn configure_memory_allocator() -> Result<(), String> {
         Ok(())
     }
 
     pub use crate::jemalloc::scrape_jemalloc_metrics as scrape_allocator_metrics;
+
+    pub fn allocator_name() -> String {
+        match crate::jemalloc::page_size() {
+            Ok(page_size) => format!("jemalloc ({}K)", page_size / 1024),
+            Err(e) => format!("jemalloc (error: {e:?})"),
+        }
+    }
 }
 
-#[cfg(all(
-    any(not(target_os = "linux"), target_env = "musl"),
-    not(feature = "jemalloc")
+#[cfg(any(
+    not(unix),
+    all(
+        any(feature = "sysmalloc", not(feature = "jemalloc")),
+        any(not(target_os = "linux"), target_env = "musl")
+    )
 ))]
 mod interface {
     #[allow(dead_code, clippy::unnecessary_wraps)]
@@ -68,4 +85,8 @@ mod interface {
 
     #[allow(dead_code)]
     pub fn scrape_allocator_metrics() {}
+
+    pub fn allocator_name() -> String {
+        "system".to_string()
+    }
 }

consensus/state_processing/Cargo.toml

@@ -41,5 +41,4 @@ types = { workspace = true }
 
 [dev-dependencies]
 beacon_chain = { workspace = true }
-env_logger = { workspace = true }
 tokio = { workspace = true }