diff --git a/eden/fs/cli_rs/edenfs-client/src/client/thrift_client.rs b/eden/fs/cli_rs/edenfs-client/src/client/thrift_client.rs index b9e1bbd4daa75..4fec4737ede32 100644 --- a/eden/fs/cli_rs/edenfs-client/src/client/thrift_client.rs +++ b/eden/fs/cli_rs/edenfs-client/src/client/thrift_client.rs @@ -18,9 +18,13 @@ use edenfs_error::ConnectAndRequestError; use edenfs_error::ErrorHandlingStrategy; use edenfs_error::HasErrorHandlingStrategy; use edenfs_error::Result; +#[cfg(fbcode_build)] use edenfs_telemetry::EdenSample; +#[cfg(fbcode_build)] use edenfs_telemetry::QueueingScubaLogger; +#[cfg(fbcode_build)] use edenfs_telemetry::SampleLogger; +#[cfg(fbcode_build)] use edenfs_telemetry::create_logger; use fbinit::FacebookInit; use futures_stats::TimedTryFutureExt; @@ -41,6 +45,7 @@ use crate::client::connector::StreamingEdenFsThriftClientFuture; use crate::methods::EdenThriftMethod; use crate::use_case::UseCase; +#[cfg(fbcode_build)] lazy_static! { static ref SCUBA_CLIENT: QueueingScubaLogger = QueueingScubaLogger::new(create_logger("edenfs_client".to_string()), 1000); @@ -126,6 +131,7 @@ impl Client for ThriftClient { let mut connection = (*self.connection.lock()).clone(); let mut attempts = 0; let mut retries = 0; + #[cfg(fbcode_build)] let mut sample = EdenSample::new(); loop { attempts += 1; @@ -144,23 +150,30 @@ impl Client for ThriftClient { } .try_timed() .await; - sample.add_int("wall_clock_duration_us", start.elapsed().as_micros() as i64); - sample.add_int("attempts", attempts as i64); - sample.add_int("retries", retries as i64); - sample.add_string("use_case", self.use_case.name()); - sample.add_string("session_id", &self.session_id); - sample.add_string("request_id", generate_id().as_str()); + #[cfg(fbcode_build)] + { + sample.add_int("wall_clock_duration_us", start.elapsed().as_micros() as i64); + sample.add_int("attempts", attempts as i64); + sample.add_int("retries", retries as i64); + sample.add_string("use_case", self.use_case.name()); + sample.add_string("session_id", &self.session_id); + sample.add_string("request_id", generate_id().as_str()); + } let (error, method) = match result { Ok((stats, (result, method))) => { self.stats_handler.on_success(attempts, retries); - sample.add_int("success", true as i64); - sample.add_int("duration_us", stats.completion_time.as_micros() as i64); - sample.add_string("method", method.name()); - let _ = SCUBA_CLIENT.log(sample); // Ideally log should be infalliable, but since its not we don't want to fail the request + #[cfg(fbcode_build)] + { + sample.add_int("success", true as i64); + sample.add_int("duration_us", stats.completion_time.as_micros() as i64); + sample.add_string("method", method.name()); + let _ = SCUBA_CLIENT.log(sample); // Ideally log should be infalliable, but since its not we don't want to fail the request + } break Ok(result); } Err(e) => e, }; + #[cfg(fbcode_build)] sample.add_string("method", method.name()); match error.get_error_handling_strategy() { ErrorHandlingStrategy::Reconnect => { @@ -190,16 +203,22 @@ impl Client for ThriftClient { ); } ErrorHandlingStrategy::Abort => { - sample.fail(format!("{:?}", error).as_str()); - let _ = SCUBA_CLIENT.log(sample); + #[cfg(fbcode_build)] + { + sample.fail(format!("{:?}", error).as_str()); + let _ = SCUBA_CLIENT.log(sample); + } break Err(error); } }; if attempts > MAX_RETRY_ATTEMPTS { - sample.fail(format!("{:?}", error).as_str()); - sample.add_bool("max_retry_reached", true); - let _ = SCUBA_CLIENT.log(sample); + #[cfg(fbcode_build)] + { + sample.fail(format!("{:?}", error).as_str()); + sample.add_bool("max_retry_reached", true); + let _ = SCUBA_CLIENT.log(sample); + } break Err(error); } } diff --git a/eden/fs/cli_rs/edenfs-client/src/use_case/use_case.rs b/eden/fs/cli_rs/edenfs-client/src/use_case/use_case.rs index de7108b674d50..8ce1c318efc51 100644 --- a/eden/fs/cli_rs/edenfs-client/src/use_case/use_case.rs +++ b/eden/fs/cli_rs/edenfs-client/src/use_case/use_case.rs @@ -8,6 +8,7 @@ use std::path::Path; use std::path::PathBuf; +#[cfg(feature = "fb")] use remote_loader::get_remote_configs; use strum::IntoStaticStr; @@ -87,6 +88,12 @@ impl UseCase { }) } + #[cfg(not(feature = "fb"))] + fn get_use_case(&self) -> Option { + None + } + + #[cfg(feature = "fb")] fn get_use_case(&self) -> Option { let is_pub = cpe::x2p::supports_vpnless(); let config_url = helpers::config_url(is_pub); @@ -98,7 +105,8 @@ impl UseCase { }; let http_config = helpers::get_http_config(is_pub, proxy_sock_path).ok()?; let cache_path = self.config_dir.join("scm_use_cases"); - let config: ScmUseCases = get_remote_configs( + + let config = get_remote_configs( is_pub, Some(config_url), 300, // 5 minutes diff --git a/eden/fs/cli_rs/edenfs-commands/src/debug/bench/cmd.rs b/eden/fs/cli_rs/edenfs-commands/src/debug/bench/cmd.rs index de9fdf3bbb530..add0128fbbd0f 100644 --- a/eden/fs/cli_rs/edenfs-commands/src/debug/bench/cmd.rs +++ b/eden/fs/cli_rs/edenfs-commands/src/debug/bench/cmd.rs @@ -133,15 +133,19 @@ impl crate::Subcommand for BenchCmd { random_data.chunk_size as f64 / types::BYTES_IN_KILOBYTE as f64, random_data.total_size() as f64 / types::BYTES_IN_GIGABYTE as f64 ); + #[cfg(fbcode_build)] println!( "{}", dbio::bench_rocksdb_write_mfmd(&test_dir, &random_data)? ); + #[cfg(fbcode_build)] println!( "{}", dbio::bench_rocksdb_read_mfmd(&test_dir, &random_data)? ); + #[cfg(fbcode_build)] println!("{}", dbio::bench_lmdb_write_mfmd(&test_dir, &random_data)?); + #[cfg(fbcode_build)] println!("{}", dbio::bench_lmdb_read_mfmd(&test_dir, &random_data)?); println!( "{}", diff --git a/eden/fs/cli_rs/edenfs-commands/src/debug/bench/dbio.rs b/eden/fs/cli_rs/edenfs-commands/src/debug/bench/dbio.rs index 4a95149822434..ec1f9f2b9e5e7 100644 --- a/eden/fs/cli_rs/edenfs-commands/src/debug/bench/dbio.rs +++ b/eden/fs/cli_rs/edenfs-commands/src/debug/bench/dbio.rs @@ -11,6 +11,7 @@ use std::time::Instant; use anyhow::Result; use anyhow::anyhow; +#[cfg(fbcode_build)] use lmdb::Txn; use super::r#gen::RandomData; @@ -20,6 +21,7 @@ use super::types::Benchmark; use super::types::BenchmarkType; /// Runs the RocksDB write benchmark and returns the benchmark results +#[cfg(fbcode_build)] pub fn bench_rocksdb_write_mfmd(test_dir: &TestDir, random_data: &RandomData) -> Result { let mut agg_write_dur = std::time::Duration::new(0, 0); let db_opts = rocksdb::Options::new().create_if_missing(true); @@ -73,6 +75,7 @@ pub fn bench_rocksdb_write_mfmd(test_dir: &TestDir, random_data: &RandomData) -> } /// Runs the RocksDB read benchmark and returns the benchmark results +#[cfg(fbcode_build)] pub fn bench_rocksdb_read_mfmd(test_dir: &TestDir, random_data: &RandomData) -> Result { let mut agg_read_dur = std::time::Duration::new(0, 0); let mut read_data = vec![0u8; random_data.chunk_size]; @@ -111,6 +114,7 @@ pub fn bench_rocksdb_read_mfmd(test_dir: &TestDir, random_data: &RandomData) -> } /// Runs the LMDB write benchmark and returns the benchmark results +#[cfg(fbcode_build)] pub fn bench_lmdb_write_mfmd(test_dir: &TestDir, random_data: &RandomData) -> Result { let mut agg_write_dur = std::time::Duration::new(0, 0); let env = lmdb::Env::options()? @@ -168,6 +172,7 @@ pub fn bench_lmdb_write_mfmd(test_dir: &TestDir, random_data: &RandomData) -> Re } /// Runs the LMDB read benchmark and returns the benchmark results +#[cfg(fbcode_build)] pub fn bench_lmdb_read_mfmd(test_dir: &TestDir, random_data: &RandomData) -> Result { let mut agg_read_dur = std::time::Duration::new(0, 0); let mut read_data = vec![0u8; random_data.chunk_size]; diff --git a/eden/fs/cli_rs/edenfs-commands/src/glob_and_prefetch/prefetch.rs b/eden/fs/cli_rs/edenfs-commands/src/glob_and_prefetch/prefetch.rs index b94940ec8dabb..9d506f8ac90af 100644 --- a/eden/fs/cli_rs/edenfs-commands/src/glob_and_prefetch/prefetch.rs +++ b/eden/fs/cli_rs/edenfs-commands/src/glob_and_prefetch/prefetch.rs @@ -13,7 +13,9 @@ use std::path::PathBuf; use anyhow::Result; use async_trait::async_trait; use clap::Parser; +#[cfg(fbcode_build)] use edenfs_telemetry::collect_system_info; +#[cfg(fbcode_build)] use edenfs_telemetry::edenfs_events_mapper; use edenfs_utils::path_from_bytes; @@ -49,6 +51,7 @@ pub struct PrefetchCmd { } impl PrefetchCmd { + #[cfg(fbcode_build)] fn new_sample(&self, mount_point: &Path) -> edenfs_telemetry::EdenSample { let mut sample = edenfs_telemetry::EdenSample::new(); collect_system_info(&mut sample, edenfs_events_mapper); @@ -81,6 +84,7 @@ impl crate::Subcommand for PrefetchCmd { let client = instance.get_client(); let (mount_point, _search_root) = self.common.get_mount_point_and_search_root()?; + #[cfg(fbcode_build)] let mut sample = self.new_sample(&mount_point); let patterns = self.common.load_patterns()?; @@ -101,17 +105,21 @@ impl crate::Subcommand for PrefetchCmd { .await { Ok(r) => { + #[cfg(fbcode_build)] sample.add_bool("success", true); Ok(r) } Err(e) => { + #[cfg(fbcode_build)] sample.add_bool("success", false); + #[cfg(fbcode_build)] sample.add_string("error", format!("{:#}", e).as_str()); Err(e) } }?; // NOTE: Is the really still needed? We should not be falling back at all anymore. + #[cfg(fbcode_build)] sample.add_bool("prefetchV2_fallback", false); if return_prefetched_files { @@ -129,6 +137,7 @@ impl crate::Subcommand for PrefetchCmd { } if let Some(prefetched_files) = &result.prefetched_files { + #[cfg(fbcode_build)] sample.add_int( "files_fetched", prefetched_files.matching_files.len() as i64, @@ -142,6 +151,7 @@ impl crate::Subcommand for PrefetchCmd { } } + #[cfg(fbcode_build)] edenfs_telemetry::send(edenfs_telemetry::EDEN_EVENTS_SCUBA.to_string(), sample); Ok(0) } diff --git a/eden/fs/service/EdenServer.cpp b/eden/fs/service/EdenServer.cpp index b5443f87c8f00..50f6eb87b63d7 100644 --- a/eden/fs/service/EdenServer.cpp +++ b/eden/fs/service/EdenServer.cpp @@ -1006,10 +1006,12 @@ void EdenServer::startPeriodicTasks() { } else { entities = {hostname, fmt::format("{}:{}", hostname, reWorkerID)}; } +#ifdef EDEN_HAVE_MONITORING memory_vm_rss_bytes_ = monitoring::OBCAvg( monitoring::OdsCategoryId::ODS_EDEN, fmt::format("eden.{}", kMemoryVmRssBytes), entities); +#endif // EDEN_HAVE_MONITORING // Report memory usage stats once every 60 seconds memoryStatsTask_.updateInterval(60s); } @@ -2850,11 +2852,13 @@ void EdenServer::flushStatsNow() { } void EdenServer::reportMemoryStats() { +#ifdef EDEN_HAVE_MONITORING auto memoryStats = facebook::eden::proc_util::readMemoryStats(); if (memoryStats) { // Bump the OBC counters for the current memory usage memory_vm_rss_bytes_ += memoryStats->resident; } +#endif // EDEN_HAVE_MONITORING } void EdenServer::manageLocalStore() { diff --git a/eden/fs/service/EdenServer.h b/eden/fs/service/EdenServer.h index 0825417717236..95a888afa5dbb 100644 --- a/eden/fs/service/EdenServer.h +++ b/eden/fs/service/EdenServer.h @@ -40,7 +40,9 @@ #include "eden/fs/takeover/TakeoverHandler.h" #include "eden/fs/telemetry/EdenStats.h" #include "eden/fs/telemetry/IActivityRecorder.h" +#ifdef EDEN_HAVE_MONITORING #include "monitoring/obc/OBCAvg.h" +#endif // EDEN_HAVE_MONITORING DECLARE_bool(takeover); @@ -583,8 +585,10 @@ class EdenServer : private TakeoverHandler { void startPeriodicTasks(); void updatePeriodicTaskIntervals(const EdenConfig& config); + #ifdef EDEN_HAVE_MONITORING // calculates eden rss memory usage monitoring::OBCAvg memory_vm_rss_bytes_; + #endif // EDEN_HAVE_MONITORING /** * Schedule a call to unloadInodes() to happen after timeout diff --git a/eden/fs/store/hg/SaplingBackingStore.cpp b/eden/fs/store/hg/SaplingBackingStore.cpp index 1ae1618665b3f..22850413ac154 100644 --- a/eden/fs/store/hg/SaplingBackingStore.cpp +++ b/eden/fs/store/hg/SaplingBackingStore.cpp @@ -158,9 +158,11 @@ SaplingBackingStore::SaplingBackingStore( folly::to("hg-activitybuffer-", getRepoName().value_or("")), [this](const HgImportTraceEvent& event) { this->processHgEvent(event); }); +#ifdef EDEN_HAVE_MONITORING if (config_->getEdenConfig()->enableOBCOnEden.getValue()) { initializeOBCCounters(); } +#endif // EDEN_HAVE_MONITORING } /** @@ -217,9 +219,11 @@ SaplingBackingStore::SaplingBackingStore( folly::to("hg-activitybuffer-", getRepoName().value_or("")), [this](const HgImportTraceEvent& event) { this->processHgEvent(event); }); +#ifdef EDEN_HAVE_MONITORING if (config_->getEdenConfig()->enableOBCOnEden.getValue()) { initializeOBCCounters(); } +#endif // EDEN_HAVE_MONITORING } SaplingBackingStore::~SaplingBackingStore() { @@ -229,6 +233,7 @@ SaplingBackingStore::~SaplingBackingStore() { } } +#ifdef EDEN_HAVE_MONITORING void SaplingBackingStore::initializeOBCCounters() { std::string repoName = store_.getRepoName().data(); // Get the hostname without the ".facebook.com" suffix @@ -243,6 +248,7 @@ void SaplingBackingStore::initializeOBCCounters() { {hostname}); isOBCEnabled_ = true; } +#endif // EDEN_HAVE_MONITORING BackingStore::LocalStoreCachingPolicy SaplingBackingStore::constructLocalStoreCachingPolicy() { @@ -342,9 +348,11 @@ void SaplingBackingStore::setFetchBlobCounters( return; } +#ifdef EDEN_HAVE_MONITORING if (isOBCEnabled_) { getBlobPerRepoLatencies_ += watch.elapsed().count(); } +#endif // EDEN_HAVE_MONITORING stats_->addDuration(&SaplingBackingStoreStats::fetchBlob, watch.elapsed()); if (fetchResult == ObjectFetchContext::FetchResult::Success) { @@ -553,9 +561,11 @@ void SaplingBackingStore::getTreeBatch( } } +#ifdef EDEN_HAVE_MONITORING if (isOBCEnabled_) { getTreePerRepoLatencies_ += batchWatch.elapsed().count(); } +#endif // EDEN_HAVE_MONITORING stats_->addDuration( &SaplingBackingStoreStats::fetchTree, batchWatch.elapsed()); diff --git a/eden/fs/store/hg/SaplingBackingStore.h b/eden/fs/store/hg/SaplingBackingStore.h index 0a68941a1cc9e..db976cb1272c5 100644 --- a/eden/fs/store/hg/SaplingBackingStore.h +++ b/eden/fs/store/hg/SaplingBackingStore.h @@ -28,7 +28,9 @@ #include "eden/fs/store/hg/SaplingImportRequestQueue.h" #include "eden/fs/telemetry/ActivityBuffer.h" #include "eden/scm/lib/backingstore/include/SaplingNativeBackingStore.h" +#ifdef EDEN_HAVE_MONITORING #include "monitoring/obc/OBCPxx.h" +#endif // EDEN_HAVE_MONITORING namespace facebook::eden { @@ -646,6 +648,7 @@ class SaplingBackingStore final : public BackingStore { std::shared_ptr localStore_; EdenStatsPtr stats_; +#ifdef EDEN_HAVE_MONITORING // This is used to avoid reading config in hot path of get request bool isOBCEnabled_ = false; // TODO: this is a prototype to test OBC API on eden @@ -653,6 +656,7 @@ class SaplingBackingStore final : public BackingStore { monitoring::OBCP99P95P50 getBlobPerRepoLatencies_; // calculates p50, p95, p99 monitoring::OBCP99P95P50 getTreePerRepoLatencies_; // calculates p50, p95, p99 void initializeOBCCounters(); +#endif // EDEN_HAVE_MONITORING /** * Reference to the eden config, may be a null pointer in unit tests.