test: user_agent in http requests

Author: anshalshukla

beacon_node/builder_client/src/lib.rs

@@ -7,9 +7,10 @@ use eth2::types::{
 };
 use eth2::types::{FullPayloadContents, SignedBlindedBeaconBlock};
 use eth2::{
-    CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, HttpClientBuilderWithUserAgent,
-    JSON_CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER, StatusCode, ok_or_error,
+    CONSENSUS_VERSION_HEADER, CONTENT_TYPE_HEADER, JSON_CONTENT_TYPE_HEADER,
+    SSZ_CONTENT_TYPE_HEADER, StatusCode, ok_or_error,
 };
+
 use reqwest::header::{ACCEPT, HeaderMap, HeaderValue};
 use reqwest::{IntoUrl, Response};
 use sensitive_url::SensitiveUrl;
@@ -74,13 +75,18 @@ impl BuilderHttpClient {
         builder_header_timeout: Option<Duration>,
         disable_ssz: bool,
     ) -> Result<Self, Error> {
-        let user_agent = user_agent.unwrap_or(lighthouse_version::user_agent());
-        let client = HttpClientBuilderWithUserAgent::new(Some(user_agent.clone())).build()?;
+        let user_agent = user_agent
+            .as_deref()
+            .unwrap_or(lighthouse_version::DEFAULT_USER_AGENT);
+        let client = eth2::create_client_with_user_agent(user_agent)
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new());
+
         Ok(Self {
             client,
             server,
             timeouts: Timeouts::new(builder_header_timeout),
-            user_agent,
+            user_agent: user_agent.into(),
             disable_ssz,
             ssz_available: Arc::new(false.into()),
         })

beacon_node/http_api/tests/tests.rs

@@ -8,6 +8,7 @@ use eth2::{
     Error::ServerMessage,
     StatusCode, Timeouts,
     mixin::{RequestAccept, ResponseForkName, ResponseOptional},
+    reqwest::Method,
     reqwest::RequestBuilder,
     types::{
         BlockId as CoreBlockId, ForkChoiceNode, ProduceBlockV3Response, StateId as CoreStateId, *,
@@ -35,6 +36,7 @@ use state_processing::per_slot_processing;
 use state_processing::state_advance::partial_state_advance;
 use std::convert::TryInto;
 use std::sync::Arc;
+use std::sync::Mutex;
 use tokio::time::Duration;
 use tree_hash::TreeHash;
 use types::application_domain::ApplicationDomain;
@@ -43,6 +45,7 @@ use types::{
     MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, SingleAttestation, Slot,
     attestation::AttestationBase,
 };
+use warp::Filter;
 
 type E = MainnetEthSpec;
 
@@ -6733,6 +6736,33 @@ impl ApiTester {
         }
         self
     }
+
+    async fn test_user_agent(self) -> Self {
+        let captured: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
+        let captured_filter = captured.clone();
+
+        // Warp route: capture User-Agent header
+        let route = warp::any()
+            .and(warp::header::optional::<String>("user-agent"))
+            .map(move |ua: Option<String>| {
+                *captured_filter.lock().unwrap() = ua;
+                warp::reply()
+            });
+
+        // Start ephemeral server
+        let (addr, server) = warp::serve(route).bind_ephemeral(([127, 0, 0, 1], 0));
+        tokio::spawn(server);
+
+        // Make a request (simulate Lighthouse client)
+        let url = format!("http://{}/", addr);
+        let _client = self.client.create_request(Method::GET, &url).await;
+
+        // Check captured User-Agent
+        let ua = captured.lock().unwrap().clone().unwrap();
+        assert_eq!(ua, lighthouse_version::DEFAULT_USER_AGENT);
+
+        self
+    }
 }
 
 async fn poll_events<S: Stream<Item = Result<EventKind<E>, eth2::Error>> + Unpin, E: EthSpec>(
@@ -7882,3 +7912,8 @@ async fn get_beacon_rewards_blocks_electra() {
         .test_beacon_block_rewards_electra()
         .await;
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_user_agent() {
+    ApiTester::new().await.test_user_agent().await;
+}

common/eth2/src/lib.rs

@@ -23,9 +23,9 @@ use futures_util::StreamExt;
 use libp2p_identity::PeerId;
 use lighthouse_version;
 use pretty_reqwest_error::PrettyReqwestError;
-use reqwest;
+pub use reqwest;
 use reqwest::{
-    Body, IntoUrl, RequestBuilder, Response,
+    Body, IntoUrl, Method, RequestBuilder, Response,
     header::{HeaderMap, HeaderValue},
 };
 pub use reqwest::{StatusCode, Url};
@@ -239,7 +239,7 @@ impl AsRef<str> for BeaconNodeHttpClient {
 impl BeaconNodeHttpClient {
     pub fn new(server: SensitiveUrl, timeouts: Timeouts) -> Self {
         let client = reqwest::ClientBuilder::new()
-            .user_agent(lighthouse_version::user_agent())
+            .user_agent(lighthouse_version::DEFAULT_USER_AGENT)
             .build()
             .unwrap_or_else(|_| reqwest::Client::new());
 
@@ -2891,6 +2891,14 @@ impl BeaconNodeHttpClient {
         self.post_with_timeout_and_response(path, &selections, self.timeouts.sync_aggregators)
             .await
     }
+
+    pub async fn create_request(&self, method: Method, url: &str) -> Result<Response, Error> {
+        self.client
+            .request(method, url)
+            .send()
+            .await
+            .map_err(Error::from)
+    }
 }
 
 /// Returns `Ok(response)` if the response is a `200 OK` response. Otherwise, creates an
@@ -2910,12 +2918,12 @@ pub async fn ok_or_error(response: Response) -> Result<Response, Error> {
     }
 }
 
-/// A wrapper around `reqwest::ClientBuilder` which adds a user agent for client identification
-pub struct HttpClientBuilderWithUserAgent;
-
-impl HttpClientBuilderWithUserAgent {
-    pub fn new(user_agent: Option<String>) -> reqwest::ClientBuilder {
-        let user_agent = user_agent.unwrap_or(lighthouse_version::user_agent());
-        reqwest::ClientBuilder::new().user_agent(user_agent)
-    }
+/// A helper function to create a `reqwest::Client` with a user agent
+pub fn create_client_with_user_agent<'a>(
+    user_agent: impl Into<Option<&'a str>>,
+) -> reqwest::ClientBuilder {
+    let user_agent = user_agent
+        .into()
+        .unwrap_or(lighthouse_version::DEFAULT_USER_AGENT);
+    reqwest::ClientBuilder::new().user_agent(user_agent)
 }

common/eth2/src/lighthouse_vc/http_client.rs

@@ -47,7 +47,7 @@ impl ValidatorClientHttpClient {
     /// Create a new client pre-initialised with an API token.
     pub fn new(server: SensitiveUrl, secret: String) -> Result<Self, Error> {
         let client = reqwest::ClientBuilder::new()
-            .user_agent(lighthouse_version::user_agent())
+            .user_agent(lighthouse_version::DEFAULT_USER_AGENT)
             .build()
             .unwrap_or_else(|_| reqwest::Client::new());
 
@@ -64,7 +64,7 @@ impl ValidatorClientHttpClient {
     /// A token can be fetched by using `self.get_auth`, and then reading the token from disk.
     pub fn new_unauthenticated(server: SensitiveUrl) -> Result<Self, Error> {
         let client = reqwest::ClientBuilder::new()
-            .user_agent(lighthouse_version::user_agent())
+            .user_agent(lighthouse_version::DEFAULT_USER_AGENT)
             .build()
             .unwrap_or_else(|_| reqwest::Client::new());
 
@@ -692,8 +692,17 @@ impl ValidatorClientHttpClient {
         self.delete(url).await
     }
 
-    pub async fn create_request<U: IntoUrl>(&self, url: U) -> Result<Response, Error> {
-        self.client.get(url).send().await.map_err(Error::from)
+    /// Used internally by the `ValidatorClientHttpClient` to create a request.
+    pub async fn create_request<U: IntoUrl>(
+        &self,
+        method: reqwest::Method,
+        url: U,
+    ) -> Result<Response, Error> {
+        self.client
+            .request(method, url)
+            .send()
+            .await
+            .map_err(Error::from)
     }
 }
 

common/lighthouse_version/src/lib.rs

@@ -21,6 +21,9 @@ pub const VERSION: &str = git_version!(
     fallback = "Lighthouse/v7.1.0"
 );
 
+/// Returns `VERSION` used as default User-Agent for HTTP requests.
+pub const DEFAULT_USER_AGENT: &str = VERSION;
+
 /// Returns the first eight characters of the latest commit hash for this build.
 ///
 /// No indication is given if the tree is dirty. This is part of the standard
@@ -48,15 +51,6 @@ pub fn version_with_platform() -> String {
     format!("{}/{}-{}", VERSION, consts::ARCH, consts::OS)
 }
 
-/// Returns `CLIENT_NAME/VERSION` used as default User-Agent for HTTP requests.
-///
-/// ## Example
-///
-/// `Lighthouse/v7.1.0-67da032+`
-pub fn user_agent() -> String {
-    format!("{}", VERSION)
-}
-
 /// Returns semantic versioning information only.
 ///
 /// ## Example
@@ -100,34 +94,4 @@ mod test {
             version()
         );
     }
-
-    #[test]
-    fn user_agent_formatting() {
-        let ua = user_agent();
-
-        // User agent should match the VERSION format
-        let re = Regex::new(
-            r"^Lighthouse/v[0-9]+\.[0-9]+\.[0-9]+(-(rc|beta).[0-9])?(-[[:xdigit:]]{7})?\+?$",
-        )
-        .unwrap();
-
-        assert!(
-            re.is_match(&ua),
-            "user agent doesn't match expected format: {}",
-            ua
-        );
-
-        // User agent should be equal to VERSION
-        assert_eq!(ua, VERSION, "user_agent() should return VERSION");
-    }
-
-    #[test]
-    fn user_agent_non_empty() {
-        let ua = user_agent();
-        assert!(!ua.is_empty(), "user agent should not be empty");
-        assert!(
-            ua.starts_with("Lighthouse/"),
-            "user agent should start with 'Lighthouse/'"
-        );
-    }
 }

testing/node_test_rig/src/lib.rs

@@ -4,7 +4,7 @@
 
 use beacon_node::ProductionBeaconNode;
 use environment::RuntimeContext;
-use eth2::{BeaconNodeHttpClient, Timeouts, HttpClientBuilderWithUserAgent};
+use eth2::{BeaconNodeHttpClient, Timeouts};
 use sensitive_url::SensitiveUrl;
 use std::path::PathBuf;
 use std::time::Duration;
@@ -81,7 +81,7 @@ impl<E: EthSpec> LocalBeaconNode<E> {
             format!("http://{}:{}", listen_addr.ip(), listen_addr.port()).as_str(),
         )
         .map_err(|e| format!("Unable to parse beacon node URL: {:?}", e))?;
-        let beacon_node_http_client = HttpClientBuilderWithUserAgent::new(None)
+        let beacon_node_http_client = eth2::create_client_with_user_agent(None)
             .timeout(HTTP_TIMEOUT)
             .build()
             .map_err(|e| format!("Unable to build HTTP client: {:?}", e))?;

validator_client/http_api/src/test_utils.rs

@@ -6,6 +6,7 @@ use account_utils::{
 };
 use deposit_contract::decode_eth1_tx_data;
 use doppelganger_service::DoppelgangerService;
+use eth2::reqwest::Method;
 use eth2::{
     Error as ApiError,
     lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*},
@@ -456,11 +457,11 @@ impl ApiTester {
 
         // Make a request (simulate Lighthouse client)
         let url = format!("http://{}/", addr);
-        let _client = self.client.create_request(&url).await;
+        let _client = self.client.create_request(Method::GET, &url).await;
 
         // Check captured User-Agent
         let ua = captured.lock().unwrap().clone().unwrap();
-        assert_eq!(ua, lighthouse_version::user_agent());
+        assert_eq!(ua, lighthouse_version::DEFAULT_USER_AGENT);
 
         self
     }

validator_client/http_api/src/tests.rs

@@ -11,6 +11,7 @@ use account_utils::{
     random_password_string, validator_definitions::ValidatorDefinitions,
 };
 use deposit_contract::decode_eth1_tx_data;
+use eth2::reqwest::Method;
 use eth2::{
     Error as ApiError,
     lighthouse_vc::{http_client::ValidatorClientHttpClient, types::*},
@@ -420,11 +421,11 @@ impl ApiTester {
 
         // Make a request (simulate Lighthouse client)
         let url = format!("http://{}/", addr);
-        let _client = self.client.create_request(&url).await;
+        let _client = self.client.create_request(Method::GET, &url).await;
 
         // Check captured User-Agent
         let ua = captured.lock().unwrap().clone().unwrap();
-        assert_eq!(ua, lighthouse_version::user_agent());
+        assert_eq!(ua, lighthouse_version::DEFAULT_USER_AGENT);
 
         self
     }

validator_client/src/lib.rs

@@ -17,7 +17,7 @@ use beacon_node_fallback::{
 use clap::ArgMatches;
 use doppelganger_service::DoppelgangerService;
 use environment::RuntimeContext;
-use eth2::{BeaconNodeHttpClient, HttpClientBuilderWithUserAgent, StatusCode, Timeouts};
+use eth2::{BeaconNodeHttpClient, StatusCode, Timeouts};
 use initialized_validators::Error::UnableToOpenVotingKeystore;
 use lighthouse_validator_store::LighthouseValidatorStore;
 use parking_lot::RwLock;
@@ -272,7 +272,7 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
             let url = x.1;
             let slot_duration = Duration::from_secs(context.eth2_config.spec.seconds_per_slot);
 
-            let mut beacon_node_http_client_builder = HttpClientBuilderWithUserAgent::new(None);
+            let mut beacon_node_http_client_builder = eth2::create_client_with_user_agent(None);
 
             // Add new custom root certificates if specified.
             if let Some(certificates) = &config.beacon_nodes_tls_certs {