add user agent in http requests

Author: anshalshukla

Cargo.lock

@@ -7,8 +7,8 @@ use eth2::types::{
 };
 use eth2::types::{FullPayloadContents, SignedBlindedBeaconBlock};
 use eth2::{
-    CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, JSON_CONTENT_TYPE_HEADER,
-    SSZ_CONTENT_TYPE_HEADER, StatusCode, ok_or_error,
+    CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, HttpClientBuilderWithUserAgent,
+    JSON_CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER, StatusCode, ok_or_error,
 };
 use reqwest::header::{ACCEPT, HeaderMap, HeaderValue};
 use reqwest::{IntoUrl, Response};
@@ -26,9 +26,6 @@ pub const DEFAULT_TIMEOUT_MILLIS: u64 = 15000;
 /// This timeout is in accordance with v0.2.0 of the [builder specs](https://github.com/flashbots/mev-boost/pull/20).
 pub const DEFAULT_GET_HEADER_TIMEOUT_MILLIS: u64 = 1000;
 
-/// Default user agent for HTTP requests.
-pub const DEFAULT_USER_AGENT: &str = lighthouse_version::VERSION;
-
 /// The value we set on the `ACCEPT` http header to indicate a preference for ssz response.
 pub const PREFERENCE_ACCEPT_VALUE: &str = "application/octet-stream;q=1.0,application/json;q=0.9";
 /// Only accept json responses.
@@ -77,8 +74,8 @@ impl BuilderHttpClient {
         builder_header_timeout: Option<Duration>,
         disable_ssz: bool,
     ) -> Result<Self, Error> {
-        let user_agent = user_agent.unwrap_or(DEFAULT_USER_AGENT.to_string());
-        let client = reqwest::Client::builder().user_agent(&user_agent).build()?;
+        let user_agent = user_agent.unwrap_or(lighthouse_version::user_agent());
+        let client = HttpClientBuilderWithUserAgent::new(Some(user_agent.clone())).build()?;
         Ok(Self {
             client,
             server,

beacon_node/builder_client/src/lib.rs

@@ -19,6 +19,7 @@ ethereum_ssz_derive = { workspace = true }
 futures = { workspace = true }
 futures-util = "0.3.8"
 libp2p-identity = { version = "0.2", features = ["peerid"] }
+lighthouse_version = { workspace = true }
 mediatype = "0.19.13"
 multiaddr = "0.18.2"
 pretty_reqwest_error = { workspace = true }
@@ -37,3 +38,5 @@ zeroize = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true }
+mockito = { workspace = true }
+regex = { workspace = true }

common/eth2/Cargo.toml

@@ -21,8 +21,9 @@ use derivative::Derivative;
 use futures::Stream;
 use futures_util::StreamExt;
 use libp2p_identity::PeerId;
+use lighthouse_version;
 use pretty_reqwest_error::PrettyReqwestError;
-pub use reqwest;
+use reqwest;
 use reqwest::{
     Body, IntoUrl, RequestBuilder, Response,
     header::{HeaderMap, HeaderValue},
@@ -237,8 +238,13 @@ impl AsRef<str> for BeaconNodeHttpClient {
 
 impl BeaconNodeHttpClient {
     pub fn new(server: SensitiveUrl, timeouts: Timeouts) -> Self {
+        let client = reqwest::ClientBuilder::new()
+            .user_agent(lighthouse_version::user_agent())
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new());
+
         Self {
-            client: reqwest::Client::new(),
+            client,
             server,
             timeouts,
         }
@@ -2903,3 +2909,13 @@ pub async fn ok_or_error(response: Response) -> Result<Response, Error> {
         Err(Error::StatusCode(status))
     }
 }
+
+/// A wrapper around `reqwest::ClientBuilder` which adds a user agent for client identification
+pub struct HttpClientBuilderWithUserAgent;
+
+impl HttpClientBuilderWithUserAgent {
+    pub fn new(user_agent: Option<String>) -> reqwest::ClientBuilder {
+        let user_agent = user_agent.unwrap_or(lighthouse_version::user_agent());
+        reqwest::ClientBuilder::new().user_agent(user_agent)
+    }
+}

common/eth2/src/lib.rs

@@ -1,5 +1,6 @@
 use super::types::*;
 use crate::Error;
+use lighthouse_version;
 use reqwest::{
     IntoUrl,
     header::{HeaderMap, HeaderValue},
@@ -45,8 +46,13 @@ impl Display for AuthorizationHeader {
 impl ValidatorClientHttpClient {
     /// Create a new client pre-initialised with an API token.
     pub fn new(server: SensitiveUrl, secret: String) -> Result<Self, Error> {
+        let client = reqwest::ClientBuilder::new()
+            .user_agent(lighthouse_version::user_agent())
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new());
+
         Ok(Self {
-            client: reqwest::Client::new(),
+            client,
             server,
             api_token: Some(secret.into()),
             authorization_header: AuthorizationHeader::Bearer,
@@ -57,8 +63,13 @@ impl ValidatorClientHttpClient {
     ///
     /// A token can be fetched by using `self.get_auth`, and then reading the token from disk.
     pub fn new_unauthenticated(server: SensitiveUrl) -> Result<Self, Error> {
+        let client = reqwest::ClientBuilder::new()
+            .user_agent(lighthouse_version::user_agent())
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new());
+
         Ok(Self {
-            client: reqwest::Client::new(),
+            client,
             server,
             api_token: None,
             authorization_header: AuthorizationHeader::Omit,
@@ -698,3 +709,87 @@ async fn ok_or_error(response: Response) -> Result<Response, Error> {
         Err(Error::StatusCode(status))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use mockito::{Matcher, Server};
+    use std::str::FromStr;
+
+    #[test]
+    fn test_validator_client_creation_with_user_agent() {
+        let server = SensitiveUrl::parse("http://localhost:5062").unwrap();
+        let secret = "test-secret".to_string();
+
+        // Test authenticated client
+        let client = ValidatorClientHttpClient::new(server.clone(), secret.clone()).unwrap();
+        assert!(client.api_token().is_some());
+        assert_eq!(client.authorization_header, AuthorizationHeader::Bearer);
+
+        // Test unauthenticated client
+        let unauth_client = ValidatorClientHttpClient::new_unauthenticated(server).unwrap();
+        assert!(unauth_client.api_token().is_none());
+        assert_eq!(unauth_client.authorization_header, AuthorizationHeader::Omit);
+    }
+
+    #[tokio::test]
+    async fn test_validator_client_user_agent_in_requests() {
+        // Create mock server
+        let mut server = Server::new_async().await;
+        let expected_user_agent = lighthouse_version::user_agent();
+
+        // Mock the auth endpoint with user agent verification
+        let auth_mock = server
+            .mock("GET", "/lighthouse/auth")
+            .match_header("user-agent", expected_user_agent.as_str())
+            .with_status(200)
+            .with_body(r#"{"token_path":"/tmp/test","api_token":"test-token"}"#)
+            .create_async()
+            .await;
+
+        // Create client
+        let server_url = SensitiveUrl::parse(&server.url()).unwrap();
+        let client = ValidatorClientHttpClient::new_unauthenticated(server_url).unwrap();
+
+        // Make request - this should include the user agent header
+        let _result = client.get_auth().await;
+
+        // Verify the mock was called with correct user agent
+        auth_mock.assert_async().await;
+    }
+
+    #[tokio::test]
+    async fn test_validator_client_user_agent_with_auth_token() {
+        // Create mock server
+        let mut server = Server::new_async().await;
+        let expected_user_agent = lighthouse_version::user_agent();
+        let auth_token = "test-bearer-token";
+
+        // Mock the keystores endpoint with both user agent and authorization verification
+        let keystores_mock = server
+            .mock("GET", "/eth/v1/keystores")
+            .match_header("user-agent", expected_user_agent.as_str())
+            .match_header("authorization", format!("Bearer {}", auth_token).as_str())
+            .with_status(200)
+            .with_body(r#"{"data":[]}"#)
+            .create_async()
+            .await;
+
+        // Create authenticated client
+        let server_url = SensitiveUrl::parse(&server.url()).unwrap();
+        let client = ValidatorClientHttpClient::new(server_url, auth_token.to_string()).unwrap();
+
+        // Make request - this should include both user agent and authorization headers
+        let _result = client.get_keystores().await;
+
+        // Verify the mock was called with correct headers
+        keystores_mock.assert_async().await;
+    }
+
+    #[test]
+    fn test_authorization_header_display() {
+        assert_eq!(AuthorizationHeader::Omit.to_string(), "Omit");
+        assert_eq!(AuthorizationHeader::Basic.to_string(), "Basic");
+        assert_eq!(AuthorizationHeader::Bearer.to_string(), "Bearer");
+    }
+}

common/eth2/src/lighthouse_vc/http_client.rs

@@ -48,6 +48,15 @@ pub fn version_with_platform() -> String {
     format!("{}/{}-{}", VERSION, consts::ARCH, consts::OS)
 }
 
+/// Returns `CLIENT_NAME/VERSION` used as default User-Agent for HTTP requests.
+///
+/// ## Example
+///
+/// `Lighthouse/v7.1.0-67da032+`
+pub fn user_agent() -> String {
+    format!("{}", VERSION)
+}
+
 /// Returns semantic versioning information only.
 ///
 /// ## Example
@@ -91,4 +100,34 @@ mod test {
             version()
         );
     }
+
+    #[test]
+    fn user_agent_formatting() {
+        let ua = user_agent();
+
+        // User agent should match the VERSION format
+        let re = Regex::new(
+            r"^Lighthouse/v[0-9]+\.[0-9]+\.[0-9]+(-(rc|beta).[0-9])?(-[[:xdigit:]]{7})?\+?$",
+        )
+        .unwrap();
+
+        assert!(
+            re.is_match(&ua),
+            "user agent doesn't match expected format: {}",
+            ua
+        );
+
+        // User agent should be equal to VERSION
+        assert_eq!(ua, VERSION, "user_agent() should return VERSION");
+    }
+
+    #[test]
+    fn user_agent_non_empty() {
+        let ua = user_agent();
+        assert!(!ua.is_empty(), "user agent should not be empty");
+        assert!(
+            ua.starts_with("Lighthouse/"),
+            "user agent should start with 'Lighthouse/'"
+        );
+    }
 }

common/lighthouse_version/src/lib.rs

@@ -4,7 +4,7 @@
 
 use beacon_node::ProductionBeaconNode;
 use environment::RuntimeContext;
-use eth2::{BeaconNodeHttpClient, Timeouts, reqwest::ClientBuilder};
+use eth2::{BeaconNodeHttpClient, Timeouts, HttpClientBuilderWithUserAgent};
 use sensitive_url::SensitiveUrl;
 use std::path::PathBuf;
 use std::time::Duration;
@@ -81,7 +81,7 @@ impl<E: EthSpec> LocalBeaconNode<E> {
             format!("http://{}:{}", listen_addr.ip(), listen_addr.port()).as_str(),
         )
         .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?;
-        let beacon_node_http_client = ClientBuilder::new()
+        let beacon_node_http_client = HttpClientBuilderWithUserAgent::new(None)
             .timeout(HTTP_TIMEOUT)
             .build()
             .map_err(|e| format!("Unable to build HTTP client: {:?}", e))?;

testing/node_test_rig/src/lib.rs

@@ -17,7 +17,7 @@ use beacon_node_fallback::{
 use clap::ArgMatches;
 use doppelganger_service::DoppelgangerService;
 use environment::RuntimeContext;
-use eth2::{BeaconNodeHttpClient, StatusCode, Timeouts, reqwest::ClientBuilder};
+use eth2::{BeaconNodeHttpClient, HttpClientBuilderWithUserAgent, StatusCode, Timeouts};
 use initialized_validators::Error::UnableToOpenVotingKeystore;
 use lighthouse_validator_store::LighthouseValidatorStore;
 use parking_lot::RwLock;
@@ -272,7 +272,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             let url = x.1;
             let slot_duration = Duration::from_secs(context.eth2_config.spec.seconds_per_slot);
 
-            let mut beacon_node_http_client_builder = ClientBuilder::new();
+            let mut beacon_node_http_client_builder = HttpClientBuilderWithUserAgent::new(None);
 
             // Add new custom root certificates if specified.
             if let Some(certificates) = &config.beacon_nodes_tls_certs {