bundle, gui: add tile card + primary metric

jherrera-jump · mmcgee-jump · commit fc840b41421b · 2025-07-31T02:13:35.000Z
diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md
@@ -194,7 +194,7 @@
 | <span class="metrics-name">bundle_&#8203;rtt_&#8203;sample</span> | gauge | Latest RTT sample at scrape time (nanoseconds) |
 | <span class="metrics-name">bundle_&#8203;rtt_&#8203;smoothed</span> | gauge | RTT moving average (nanoseconds) |
 | <span class="metrics-name">bundle_&#8203;rtt_&#8203;var</span> | gauge | RTT variance (nanoseconds) |
-| <span class="metrics-name">bundle_&#8203;message_&#8203;rx_&#8203;delay</span> | histogram | Message receive delay in seconds from bundle server to bundle client |
+| <span class="metrics-name">bundle_&#8203;message_&#8203;rx_&#8203;delay_&#8203;nanos</span> | histogram | Message receive delay in nanoseconds from bundle server to bundle client |
 
 </div>
 
diff --git a/book/api/websocket.md b/book/api/websocket.md
@@ -643,6 +643,8 @@ potential underflow.
         "next_leader_slot": 285228774,
         "tile_primary_metric": {
             "quic": 3,
+            "bundle_rtt_smoothed_millis": 30,
+            "bundle_rx_delay_millis_p90": 101,
             "net_in": 37803082,
             "net_out": 4982399,
             "verify": 0,
@@ -666,15 +668,17 @@ potential underflow.
 | tile_primary_metric | `TilePrimaryMetric` | Per-tile-type primary metrics.  Some of these are point-in-time values (P), and some are 1-second moving window averages (W) |
 
 **`TilePrimaryMetric`**
-| Field   | Type     | Description |
-|---------|----------|-------------|
-| net_in  | `number` | Network ingress bytes per second (W) |
-| quic    | `number` | Active QUIC connections (P) |
-| verify  | `number` | Fraction of transactions that failed sigverify (W) |
-| dedup   | `number` | Fraction of transactions deduplicated (W) |
-| pack    | `number` | Fraction of pack buffer filled (P) |
-| bank    | `number` | Execution TPS (W) |
-| net_out | `number` | Network egress bytes per second (W) |
+| Field                      | Type     | Description |
+|----------------------------|----------|-------------|
+| net_in                     | `number` | Network ingress bytes per second (W) |
+| quic                       | `number` | Active QUIC connections (P) |
+| bundle_rtt_smoothed_millis | `number` | The round-trip time for grpc messages sent to the bundle server.  These are mostly ping messages when the validator is not leader.  An exponential moving average ( avg = 1/8 val + 7/8 avg ) is used to filter the signal |
+| bundle_rx_delay_millis_p90 | `number` | An estimate of the 90th percentile of the one-way delay of a bundle dispatched from the bundle server.  Only samples since the start of the most recent leader rotation for this validator are used to compute the percentile |
+| verify                     | `number` | Fraction of transactions that failed sigverify (W) |
+| dedup                      | `number` | Fraction of transactions deduplicated (W) |
+| pack                       | `number` | Fraction of pack buffer filled (P) |
+| bank                       | `number` | Execution TPS (W) |
+| net_out                    | `number` | Network egress bytes per second (W) |
 
 
 #### `summary.live_tile_timers`
diff --git a/src/disco/bundle/fd_bundle_tile.c b/src/disco/bundle/fd_bundle_tile.c
@@ -59,7 +59,7 @@ metrics_write( fd_bundle_tile_t * ctx ) {
   FD_MGAUGE_SET( BUNDLE, RTT_SMOOTHED, (ulong)ctx->rtt->smoothed_rtt );
   FD_MGAUGE_SET( BUNDLE, RTT_VAR,      (ulong)ctx->rtt->var_rtt      );
 
-  FD_MHIST_COPY( BUNDLE, MESSAGE_RX_DELAY, ctx->metrics.msg_rx_delay );
+  FD_MHIST_COPY( BUNDLE, MESSAGE_RX_DELAY_NANOS, ctx->metrics.msg_rx_delay );
 
   fd_wksp_t * wksp = fd_wksp_containing( ctx );
   fd_wksp_usage_t usage[1];
@@ -565,8 +565,8 @@ unprivileged_init( fd_topo_t *      topo,
   fd_grpc_client_set_authority( ctx->grpc_client, ctx->server_sni, ctx->server_sni_len, ctx->server_tcp_port );
 
   fd_histf_new( ctx->metrics.msg_rx_delay,
-      FD_MHIST_SECONDS_MIN( BUNDLE, MESSAGE_RX_DELAY ),
-      FD_MHIST_SECONDS_MAX( BUNDLE, MESSAGE_RX_DELAY ) );
+      FD_MHIST_MIN( BUNDLE, MESSAGE_RX_DELAY_NANOS ),
+      FD_MHIST_MAX( BUNDLE, MESSAGE_RX_DELAY_NANOS ) );
 }
 
 static ulong
diff --git a/src/disco/gui/fd_gui.c b/src/disco/gui/fd_gui.c
@@ -132,6 +132,9 @@ fd_gui_new( void *             shmem,
   for( ulong i=0UL; i<FD_GUI_SLOTS_CNT; i++ ) gui->slots[ i ]->slot = ULONG_MAX;
   gui->pack_txn_idx = 0UL;
 
+  fd_histf_new( gui->bundle_rx_delay_hist_current,   FD_MHIST_MIN( BUNDLE, MESSAGE_RX_DELAY_NANOS ), FD_MHIST_MAX( BUNDLE, MESSAGE_RX_DELAY_NANOS ) );
+  fd_histf_new( gui->bundle_rx_delay_hist_reference, FD_MHIST_MIN( BUNDLE, MESSAGE_RX_DELAY_NANOS ), FD_MHIST_MAX( BUNDLE, MESSAGE_RX_DELAY_NANOS ) );
+
   return gui;
 }
 
@@ -499,6 +502,13 @@ fd_gui_tile_stats_snap( fd_gui_t *                     gui,
     stats->quic_conn_cnt += quic_metrics[ MIDX( GAUGE, QUIC, CONNECTIONS_ACTIVE ) ];
   }
 
+  fd_topo_tile_t const * bundle = &topo->tiles[ fd_topo_find_tile( topo, "bundle", 0UL ) ];
+  volatile ulong * bundle_metrics = fd_metrics_tile( bundle->metrics );
+  stats->bundle_rtt_smoothed_nanos = bundle_metrics[ MIDX( GAUGE, BUNDLE, RTT_SMOOTHED ) ];
+
+  gui->bundle_rx_delay_hist_current->sum = bundle_metrics[ MIDX( HISTOGRAM, BUNDLE, MESSAGE_RX_DELAY_NANOS ) + FD_HISTF_BUCKET_CNT ];
+  for( ulong b=0; b<FD_HISTF_BUCKET_CNT; b++ ) gui->bundle_rx_delay_hist_current->counts[ b ] = bundle_metrics[ MIDX( HISTOGRAM, BUNDLE, MESSAGE_RX_DELAY_NANOS ) + b ];
+
   stats->verify_drop_cnt = waterfall->out.verify_duplicate +
                            waterfall->out.verify_parse +
                            waterfall->out.verify_failed;
@@ -1731,6 +1741,19 @@ fd_gui_became_leader( fd_gui_t * gui,
   slot->txs.leader_start_time = start_time_nanos;
   slot->txs.leader_end_time   = end_time_nanos;
   if( FD_LIKELY( slot->txs.microblocks_upper_bound==USHORT_MAX ) ) slot->txs.microblocks_upper_bound = (ushort)max_microblocks;
+
+  // snapshot of bundle rx histogram at leader rotation start
+  if( FD_UNLIKELY( _slot % 4 == 0 ) ) {
+    fd_topo_tile_t const * bundle = &gui->topo->tiles[ fd_topo_find_tile( gui->topo, "bundle", 0UL ) ];
+    volatile ulong * bundle_metrics = fd_metrics_tile( bundle->metrics );
+    (void)bundle_metrics;
+
+    gui->bundle_rx_delay_hist_current->sum = bundle_metrics[ MIDX( HISTOGRAM, BUNDLE, MESSAGE_RX_DELAY_NANOS ) + FD_HISTF_BUCKET_CNT ];
+    for( ulong b=0; b<FD_HISTF_BUCKET_CNT; b++ ) gui->bundle_rx_delay_hist_current->counts[ b ] = bundle_metrics[ MIDX( HISTOGRAM, BUNDLE, MESSAGE_RX_DELAY_NANOS ) + b ];
+
+    gui->bundle_rx_delay_hist_reference->sum = bundle_metrics[ MIDX( HISTOGRAM, BUNDLE, MESSAGE_RX_DELAY_NANOS ) + FD_HISTF_BUCKET_CNT ];
+    for( ulong b=0; b<FD_HISTF_BUCKET_CNT; b++ ) gui->bundle_rx_delay_hist_reference->counts[ b ] = bundle_metrics[ MIDX( HISTOGRAM, BUNDLE, MESSAGE_RX_DELAY_NANOS ) + b ];
+  }
 }
 
 void
diff --git a/src/disco/gui/fd_gui.h b/src/disco/gui/fd_gui.h
@@ -6,6 +6,7 @@
 #include "../pack/fd_microblock.h"
 #include "../../waltz/http/fd_http_server.h"
 #include "../../flamenco/leaders/fd_leaders.h"
+#include "../../util/hist/fd_histf.h"
 
 #include "../topo/fd_topo.h"
 
@@ -161,16 +162,17 @@ typedef struct fd_gui_tile_timers fd_gui_tile_timers_t;
 struct fd_gui_tile_stats {
   long  sample_time_nanos;
 
-  ulong net_in_rx_bytes;      /* Number of bytes received by the net or sock tile*/
-  ulong quic_conn_cnt;        /* Number of active QUIC connections */
-  ulong verify_drop_cnt;      /* Number of transactions dropped by verify tiles */
-  ulong verify_total_cnt;     /* Number of transactions received by verify tiles */
-  ulong dedup_drop_cnt;       /* Number of transactions dropped by dedup tile */
-  ulong dedup_total_cnt;      /* Number of transactions received by dedup tile */
-  ulong pack_buffer_cnt;      /* Number of buffered transactions in the pack tile */
-  ulong pack_buffer_capacity; /* Total size of the pack transaction buffer */
-  ulong bank_txn_exec_cnt;    /* Number of transactions processed by the bank tile */
-  ulong net_out_tx_bytes;     /* Number of bytes sent by the net or sock tile */
+  ulong net_in_rx_bytes;           /* Number of bytes received by the net or sock tile*/
+  ulong quic_conn_cnt;             /* Number of active QUIC connections */
+  ulong bundle_rtt_smoothed_nanos; /* RTT (nanoseconds) moving average */
+  ulong verify_drop_cnt;           /* Number of transactions dropped by verify tiles */
+  ulong verify_total_cnt;          /* Number of transactions received by verify tiles */
+  ulong dedup_drop_cnt;            /* Number of transactions dropped by dedup tile */
+  ulong dedup_total_cnt;           /* Number of transactions received by dedup tile */
+  ulong pack_buffer_cnt;           /* Number of buffered transactions in the pack tile */
+  ulong pack_buffer_capacity;      /* Total size of the pack transaction buffer */
+  ulong bank_txn_exec_cnt;         /* Number of transactions processed by the bank tile */
+  ulong net_out_tx_bytes;          /* Number of bytes sent by the net or sock tile */
 };
 
 typedef struct fd_gui_tile_stats fd_gui_tile_stats_t;
@@ -360,6 +362,9 @@ struct fd_gui {
     ulong                tile_timers_leader_history_slot[ FD_GUI_TILE_TIMER_LEADER_CNT ];
   } summary;
 
+  fd_histf_t bundle_rx_delay_hist_reference[ 1 ]; /* histogram snapshot taken at the start of every leader rotation for this validator */
+  fd_histf_t bundle_rx_delay_hist_current[ 1 ]; /* latest histogram snapshot captured from metrics workspace */
+
   fd_gui_slot_t slots[ FD_GUI_SLOTS_CNT ][ 1 ];
 
   ulong pack_txn_idx; /* The pack index of the most recently received transaction */
diff --git a/src/disco/gui/fd_gui_printf.c b/src/disco/gui/fd_gui_printf.c
@@ -611,6 +611,13 @@ fd_gui_printf_tile_stats( fd_gui_t *                  gui,
                           fd_gui_tile_stats_t const * cur ) {
   jsonp_open_object( gui, "tile_primary_metric" );
     jsonp_ulong(  gui, "quic",    cur->quic_conn_cnt );
+    jsonp_double( gui, "bundle_rtt_smoothed_millis", (double)(cur->bundle_rtt_smoothed_nanos) / 1000000.0 );
+
+    fd_histf_t bundle_rx_delay_hist_delta[ 1 ];
+    fd_histf_subtract( gui->bundle_rx_delay_hist_current, gui->bundle_rx_delay_hist_reference, bundle_rx_delay_hist_delta );
+    ulong bundle_rx_delay_nanos_p90 = fd_histf_percentile( bundle_rx_delay_hist_delta, 90U, ULONG_MAX );
+    jsonp_double( gui, "bundle_rx_delay_millis_p90", fd_double_if(bundle_rx_delay_nanos_p90==ULONG_MAX, 0.0, (double)(bundle_rx_delay_nanos_p90) / 1000000.0 ));
+
     if( FD_LIKELY( cur->sample_time_nanos>prev->sample_time_nanos ) ) {
       jsonp_ulong( gui, "net_in",  (ulong)((double)(cur->net_in_rx_bytes - prev->net_in_rx_bytes) * 1000000000.0 / (double)(cur->sample_time_nanos - prev->sample_time_nanos) ));
       jsonp_ulong( gui, "net_out", (ulong)((double)(cur->net_out_tx_bytes - prev->net_out_tx_bytes) * 1000000000.0 / (double)(cur->sample_time_nanos - prev->sample_time_nanos) ));
diff --git a/src/disco/metrics/fd_prometheus.c b/src/disco/metrics/fd_prometheus.c
@@ -71,7 +71,7 @@ render_histogram( fd_prom_render_t *        r,
   for( ulong k=0; k<FD_HISTF_BUCKET_CNT; k++ ) {
     value += *(fd_metrics_tile( tile->metrics ) + metric->offset + k);
 
-    char * le;
+    char * le; /* le here means "less then or equal" not "left edge" */
     char le_str[ 64 ];
     if( FD_UNLIKELY( k==FD_HISTF_BUCKET_CNT-1UL ) ) le = "+Inf";
     else {
diff --git a/src/disco/metrics/generated/fd_metrics_bundle.c b/src/disco/metrics/generated/fd_metrics_bundle.c
@@ -18,5 +18,5 @@ const fd_metrics_meta_t FD_METRICS_BUNDLE[FD_METRICS_BUNDLE_TOTAL] = {
     DECLARE_METRIC( BUNDLE_RTT_SAMPLE, GAUGE ),
     DECLARE_METRIC( BUNDLE_RTT_SMOOTHED, GAUGE ),
     DECLARE_METRIC( BUNDLE_RTT_VAR, GAUGE ),
-    DECLARE_METRIC_HISTOGRAM_SECONDS( BUNDLE_MESSAGE_RX_DELAY ),
+    DECLARE_METRIC_HISTOGRAM_NONE( BUNDLE_MESSAGE_RX_DELAY_NANOS ),
 };
diff --git a/src/disco/metrics/generated/fd_metrics_bundle.h b/src/disco/metrics/generated/fd_metrics_bundle.h
@@ -82,13 +82,13 @@
 #define FD_METRICS_GAUGE_BUNDLE_RTT_VAR_DESC "RTT variance (nanoseconds)"
 #define FD_METRICS_GAUGE_BUNDLE_RTT_VAR_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_OFF  (32UL)
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NAME "bundle_message_rx_delay"
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_TYPE (FD_METRICS_TYPE_HISTOGRAM)
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_DESC "Message receive delay in seconds from bundle server to bundle client"
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_CVT  (FD_METRICS_CONVERTER_SECONDS)
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_MIN  (0.0001)
-#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_MAX  (1.0)
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_OFF  (32UL)
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_NAME "bundle_message_rx_delay_nanos"
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_TYPE (FD_METRICS_TYPE_HISTOGRAM)
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_DESC "Message receive delay in nanoseconds from bundle server to bundle client"
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_CVT  (FD_METRICS_CONVERTER_NONE)
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_MIN  (100000UL)
+#define FD_METRICS_HISTOGRAM_BUNDLE_MESSAGE_RX_DELAY_NANOS_MAX  (1000000000UL)
 
 #define FD_METRICS_BUNDLE_TOTAL (17UL)
 extern const fd_metrics_meta_t FD_METRICS_BUNDLE[FD_METRICS_BUNDLE_TOTAL];
diff --git a/src/disco/metrics/metrics.xml b/src/disco/metrics/metrics.xml
@@ -342,8 +342,8 @@ metric introduced.
     <gauge name="RttSample" summary="Latest RTT sample at scrape time (nanoseconds)" />
     <gauge name="RttSmoothed" summary="RTT moving average (nanoseconds)" />
     <gauge name="RttVar" summary="RTT variance (nanoseconds)" />
-    <histogram name="MessageRxDelay" min="0.0001" max="1.0" converter="seconds">
-        <summary>Message receive delay in seconds from bundle server to bundle client</summary>
+    <histogram name="MessageRxDelayNanos" min="100000" max="1000000000">
+        <summary>Message receive delay in nanoseconds from bundle server to bundle client</summary>
     </histogram>
 </tile>
 
diff --git a/src/util/hist/fd_histf.h b/src/util/hist/fd_histf.h
@@ -156,7 +156,7 @@ fd_histf_sample( fd_histf_t * hist,
 }
 
 /* fd_histf_cnt gets the count of samples in a particular bucket of the
-   historgram.
+   histogram.
 
    fd_histf_{left,right} get the sample values that map to bucket b,
    with a half-open interval [left, right).
@@ -171,6 +171,74 @@ FD_FN_PURE static inline ulong fd_histf_left ( fd_histf_t const * hist, ulong b
 FD_FN_PURE static inline ulong fd_histf_right( fd_histf_t const * hist, ulong b ) { return (ulong)hist->left_edge[ b+1UL ]+(1UL<<63); }
 FD_FN_PURE static inline ulong fd_histf_sum  ( fd_histf_t const * hist          ) { return        hist->sum;                          }
 
+/* fd_histf_percentile computes a percentile estimate.  Note that for
+   tail-end percentiles, its possible that samples will have landed in
+   the overflow/underflow bucket.  Since these are "catch-all" buckets,
+   we don't know how their samples are distributed.  For the underflow
+   bucket, since its bounds are known a linear interpolation is used to
+   estimate percentile.  For the overflow bucket, its bounds are
+   unknown, so the sentinel is returned */
+FD_FN_PURE static inline ulong
+fd_histf_percentile( fd_histf_t const * hist, uchar percentile, ulong sentinel ) {
+   if( FD_UNLIKELY( percentile > 100 ) ) FD_LOG_ERR(( "fd_histf_percentile: percentile must be in [0, 100]. got %u", percentile ));
+
+  ulong total_sample_cnt = 0UL;
+  for( ulong b=0UL; b<FD_HISTF_BUCKET_CNT; b++ )  total_sample_cnt += fd_histf_cnt( hist, b );
+  if( FD_UNLIKELY( !total_sample_cnt ) ) return sentinel;
+
+#define MAP(x, in_min, in_max, out_min, out_max) \
+    ((in_min == in_max) ? ((out_min + out_max) / 2) : (((x) - (in_min)) * ((out_max) - (out_min)) / ((in_max) - (in_min)) + (out_min)))
+
+  ulong rank = MAP((ulong)percentile, 0UL, 100UL, 0UL, total_sample_cnt-1UL);
+  ulong sum  = 0UL;
+
+  for( ulong b=0UL; b<FD_HISTF_BUCKET_CNT; b++ ) {
+    ulong count = fd_histf_cnt  ( hist, b );
+    sum += count;
+    if( sum > rank ) {
+      ulong left  = fd_histf_left ( hist, b );
+      ulong right = fd_histf_right( hist, b );
+      ulong prev  = sum - count; /* the number of samples in previous buckets */
+
+      if( FD_UNLIKELY( b==0 ) ) {
+         /* for the underflow bucket, use linear interpolation */
+         return MAP(rank - prev, 0UL, count-1UL, left, right);
+      } else if( FD_UNLIKELY( b==(FD_HISTF_BUCKET_CNT - 1UL) )) {
+         /* for the overflow bucket, return sentinel */
+         return sentinel;
+      } else {
+         /* max_value is the right value for the bucket before the
+            overflow bucket */
+         ulong max_value = fd_histf_right( hist, FD_HISTF_BUCKET_CNT - 2UL );
+
+         /* interpolate using the same equation used to construct bucket
+            sizes */
+#if FD_HAS_DOUBLE
+         return (ulong)(0.5  + (double)left * pow ( (double)max_value / (double)left, MAP((double)(rank - prev),  0.0, (double)(count-1UL),  0.0,  1.0)/(double)(FD_HISTF_BUCKET_CNT - b - 1UL) ) );
+#else
+         return (ulong)(0.5f + (float )left * powf( (float )max_value / (float )left, MAP((float )(rank - prev), 0.0f, (float )(count-1UL), 0.0f, 1.0f)/(float )(FD_HISTF_BUCKET_CNT - b - 1UL) ) );
+#endif
+      }
+    }
+  }
+
+#undef MAP
+
+  FD_LOG_ERR(( "unreachable" ));
+}
+
+/* fd_histf_subtract subtracts other_hist from hist and stores the
+   resulting histogram in hist.  In order to coherently subtract two
+   histograms, the sample history of other_hist must be a prefix to the
+   sample history of hist.  This effectively "erases" all the samples
+   from hist that were taken at or before the last sample in other_hist. */
+static inline void
+fd_histf_subtract( fd_histf_t const * hist, fd_histf_t const * prefix_hist, fd_histf_t * out ) {
+   out->sum = hist->sum - prefix_hist->sum;
+   for( ulong b=0UL; b<FD_HISTF_BUCKET_CNT; b++ ) out->counts[ b ]    = hist->counts[ b ] - prefix_hist->counts[ b ];
+   for( ulong b=0UL; b<FD_HISTF_BUCKET_CNT; b++ ) out->left_edge[ b ] = hist->left_edge[ b ];
+}
+
 FD_PROTOTYPES_END
 
 #endif /* HEADER_fd_src_util_hist_fd_histf_h */
diff --git a/src/util/hist/test_histf.c b/src/util/hist/test_histf.c
@@ -96,6 +96,37 @@ main( int     argc,
 
   FD_TEST( fd_histf_bucket_cnt( hist )==FD_HISTF_BUCKET_CNT );
 
+  FD_LOG_NOTICE(( "Testing percentile" ));
+
+  hist = fd_histf_join( fd_histf_new( fd_histf_delete( fd_histf_leave( hist ) ), 1U, 100U ) ); /* Reset histogram */
+
+  fd_histf_sample( hist, 0U   );      /* Underflow bucket */
+
+  fd_histf_t snapshot[ 1UL ];
+  memcpy(snapshot, hist, sizeof(fd_histf_t));
+
+  fd_histf_sample( hist, 30U  );      /* Middle bucket */
+  fd_histf_sample( hist, 40U  );      /* Middle bucket */
+  fd_histf_sample( hist, 99U  );      /* End bucket */
+  fd_histf_sample( hist, 101U );      /* Overflow bucket */
+  fd_histf_sample( hist, 110U );      /* Overflow bucket */
+
+  FD_TEST( fd_histf_percentile( hist,   0, ULONG_MAX )==0UL       );
+  FD_TEST( fd_histf_percentile( hist,  20, ULONG_MAX )==30UL      );
+  FD_TEST( fd_histf_percentile( hist,  40, ULONG_MAX )==41UL      );
+  FD_TEST( fd_histf_percentile( hist,  60, ULONG_MAX )==86UL      ); /* end bucket. midpoint between 74 and 99 */
+  FD_TEST( fd_histf_percentile( hist,  80, ULONG_MAX )==ULONG_MAX ); /* overflow bucket */
+  FD_TEST( fd_histf_percentile( hist, 100, ULONG_MAX )==ULONG_MAX ); /* overflow bucket */
+  FD_TEST( fd_histf_percentile( hist,  60, 42UL      )==86UL      ); /* Sentinel unused */
+
+  fd_histf_t delta[ 1UL ];
+  fd_histf_subtract( hist, snapshot, delta );
+  FD_TEST( fd_histf_percentile( delta, 0, ULONG_MAX )==30UL ); /* "30" should now have sample_rank=0 */
+
+  hist = fd_histf_join( fd_histf_new( fd_histf_delete( fd_histf_leave( hist ) ), 1U, 100U ) );
+
+  FD_TEST( fd_histf_percentile( hist, 50, 42UL )==42UL );      /* Empty histogram returns sentinel */
+
   FD_LOG_NOTICE(( "Testing performance" ));
   fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, 0U, 0UL ) );
   long overhead = -fd_log_wallclock();
