Repair & Sign: async repair requests

Author: npatel-jump

src/app/firedancer/config/default.toml

@@ -689,10 +689,10 @@ user = ""
     shred_tile_count = 1
 
     # How many sign tiles to run. Should be set to 2. This is
-    # configurable and designed to scale for the repair tile.  
+    # configurable and designed to scale for the repair tile.
     # The 0th tile gets used by other tiles for signing messages.  While
     # the remaining tiles distribute the workload of signing repair
-    # requests. 
+    # requests.
     sign_tile_count = 2
 
 # All memory that will be used in Firedancer is pre-allocated in two

src/disco/shred/fd_shred_tile.c

@@ -857,7 +857,6 @@ after_frag( fd_shred_ctx_t *    ctx,
         ulong sz = fd_shred_header_sz( shred->variant );
         fd_memcpy( fd_chunk_to_laddr( ctx->repair_out_mem, ctx->repair_out_chunk ), shred, sz );
         ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
-        // FD_LOG_INFO(("Published shred %lu %u %u, time: %ld", shred->slot, shred->idx, shred->fec_set_idx, fd_log_wallclock()));
         fd_stem_publish( stem, ctx->repair_out_idx, sig, ctx->repair_out_chunk, sz, 0UL, ctx->tsorig, tspub );
         ctx->repair_out_chunk = fd_dcache_compact_next( ctx->repair_out_chunk, sz, ctx->repair_out_chunk0, ctx->repair_out_wmark );
       }

src/disco/sign/fd_sign_tile.c

@@ -123,7 +123,8 @@ during_frag_sensitive( void * _ctx,
   ulong mtu  = ctx->in[ in_idx ].mtu;
 
   if( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>mtu ) {
-    FD_LOG_EMERG(( "oversz or out of bounds signing request (role=%d chunk=%lu sz=%lu mtu=%lu, chunk0=%lu, wmark=%lu)", role, chunk, sz, mtu, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
+    FD_LOG_EMERG(( "oversz or out of bounds signing request (role=%d chunk=%lu sz=%lu mtu=%lu, chunk0=%lu, wmark=%lu)",
+      role, chunk, sz, mtu, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
   }
 
   void * src = fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk );

src/discof/repair/fd_repair_tile.c

@@ -130,6 +130,8 @@ struct fd_repair_tile_ctx {
 
   fd_wksp_t * wksp;
 
+  fd_stem_context_t * stem;
+
   uchar              in_kind[ MAX_IN_LINKS ];
   fd_repair_in_ctx_t in_links[ MAX_IN_LINKS ];
 
@@ -424,7 +426,7 @@ fd_repair_sign_and_send( fd_repair_tile_ctx_t *  repair_tile_ctx,
   }
 }
 
-/*  REPAIR TILE REQUEST HANDLING ARCHITECTURE
+/*  REQUEST HANDLING ARCHITECTURE
     =========================================
 
     The repair tile implements two distinct request handling patterns
@@ -440,11 +442,7 @@ fd_repair_sign_and_send( fd_repair_tile_ctx_t *  repair_tile_ctx,
     - PINGs & PONGs: Handles peer connectivity and liveness with simple
       round-trip messages.
 
-    - PEER WARM UPs: On receiving peer information in
-      handle_new_cluster_contact_info, we prepay the RTT cost by sending
-      a placeholder Repair request immediately.
-
-    2. ASYNCHRONOUS REQUEST HANDLING  
+    2. ASYNCHRONOUS REQUEST HANDLING
     --------------------------------
     Used strictly for repair requests. These requests are sent to the
     sign tile, and the repair tile continues handling other operations
@@ -464,6 +462,10 @@ fd_repair_sign_and_send( fd_repair_tile_ctx_t *  repair_tile_ctx,
     - ORPHAN: Requests for the highest shred in the parent slot of an
       orphaned slot. Used to establish the chain of slot ancestry when a
       slot's parent is missing.
+    
+    - PEER WARM UPs: On receiving peer information in
+      handle_new_cluster_contact_info, we prepay the RTT cost by sending
+      a placeholder Repair request immediately.
 
     Async requests can be distributed across multiple sign tiles using
     round-robin based on the request nonce. This provides load balancing
@@ -533,11 +535,8 @@ fd_repair_send_request_async( fd_repair_tile_ctx_t *   ctx,
 
     fd_repair_pending_sign_req_t * pending = fd_repair_pending_sign_req_deque_push_tail_nocopy( ctx->pending_sign_req_deque );
 
-    /* Use the unified sign_and_send function in async mode */
     fd_repair_sign_and_send( ctx, &protocol, &peer->addr, pending->buf, sizeof(pending->buf), 1, nonce );
 
-    /* Since async requests don't complete the buffer with signature,
-       we need to track the encoded length */
     fd_bincode_encode_ctx_t encode_ctx = { .data = pending->buf, .dataend = pending->buf + sizeof(pending->buf) };
     if( FD_UNLIKELY( fd_repair_protocol_encode( &protocol, &encode_ctx ) != FD_BINCODE_SUCCESS ) ) {
         FD_LOG_CRIT(( "Failed to encode repair message (type %#x)", protocol.discriminant ));
@@ -593,12 +592,12 @@ handle_new_cluster_contact_info( fd_repair_tile_ctx_t * ctx,
     };
     int dup = fd_repair_add_active_peer( ctx->repair, &repair_peer, in_dests[i].pubkey );
     if( !dup ) {
-/* The repair process uses a Ping-Pong protocol that incurs one
-   round-trip time (RTT) for the initial repair request. To optimize
-   this, we proactively send a placeholder Repair request as soon as we
-   receive a peer's contact information for the first time, effectively
-   prepaying the RTT cost. */
-      fd_repair_send_request(ctx, ctx->stem, ctx->repair, 0, 0, 0, in_dests[i].pubkey, fd_log_wallclock());
+    /* The repair process uses a Ping-Pong protocol that incurs one
+      round-trip time (RTT) for the initial repair request. To optimize
+      this, we proactively send a placeholder Repair request as soon as we
+      receive a peer's contact information for the first time, effectively
+      prepaying the RTT cost. */
+      fd_repair_send_request_async(ctx, ctx->stem, ctx->repair, 0, 0, 0, in_dests[i].pubkey, fd_log_wallclock());
       ulong hash_src = 0xfffffUL & fd_ulong_hash( (ulong)in_dests[i].ip4_addr | ((ulong)repair_peer.port<<32) );
       FD_LOG_INFO(( "Added repair peer: pubkey %s hash_src %lu", FD_BASE58_ENC_32_ALLOCA(in_dests[i].pubkey), hash_src ));
     }
@@ -670,7 +669,7 @@ during_frag( fd_repair_tile_ctx_t * ctx,
   fd_memcpy( ctx->buffer, dcache_entry, dcache_entry_sz );
 }
 
-static ulong
+static ulong FD_FN_UNUSED
 fd_repair_send_ping( fd_repair_tile_ctx_t        * repair_tile_ctx,
                      fd_repair_t                 * glob,
                      fd_pinged_elem_t            * val,
@@ -697,7 +696,7 @@ fd_repair_send_ping( fd_repair_tile_ctx_t        * repair_tile_ctx,
   return (ulong)((uchar*)ctx.data - buf);
 }
 
-static void
+static void FD_FN_UNUSED
 fd_repair_recv_pong(fd_repair_t * glob, fd_gossip_ping_t const * pong, fd_gossip_peer_addr_t const * from) {
   fd_pinged_elem_t * val = fd_pinged_table_query(glob->pinged, from, NULL);
   if( val == NULL || !fd_pubkey_eq( &val->id, &pong->from ) )
@@ -747,6 +746,8 @@ after_frag( fd_repair_tile_ctx_t * ctx,
 
   if( FD_UNLIKELY( ctx->skip_frag ) ) return;
 
+  ctx->stem = stem;
+
   uint in_kind = ctx->in_kind[ in_idx ];
   // FD_LOG_INFO(( "in_idx: %lu, in_kind: %u", in_idx, in_kind ));
   if( FD_UNLIKELY( in_kind==IN_KIND_CONTACT ) ) {
@@ -768,7 +769,7 @@ after_frag( fd_repair_tile_ctx_t * ctx,
        sign tile will be returned. Since the repair_sign links are
        reliable, the incoming sign_repair fragments represent a complete
        set of the previously sent outgoing messages. However, with
-       multiple sign tiles, the responses may not arrive in order. But, 
+       multiple sign tiles, the responses may not arrive in order. But,
        we can safely process them sequentially as we encounter them in
        the deque. */
     while( !fd_repair_pending_sign_req_deque_empty( ctx->pending_sign_req_deque ) ) {
@@ -1195,7 +1196,7 @@ unprivileged_init( fd_topo_t *      topo,
 
     if( ctx->in_kind[ in_idx ] == IN_KIND_SIGN ) {
       // fd_wksp_t * wksp = fd_wksp_containing( link->dcache );
-      // FD_LOG_NOTICE(( "repair tile: link %s[%lu] mem=%p, dcache=%p, wksp=%p, wksp_name=%s, in_idx=%u", 
+      // FD_LOG_NOTICE(( "repair tile: link %s[%lu] mem=%p, dcache=%p, wksp=%p, wksp_name=%s, in_idx=%u",
                       // link->name, link->kind_id,
                       // (void*)ctx->in_links[ in_idx ].mem,
                       // link->dcache,

src/discof/send/fd_send_tile.c

@@ -2,6 +2,7 @@
 #include "../../disco/topo/fd_topo.h"
 #include "../../disco/keyguard/fd_keyload.h"
 #include "../../disco/keyguard/fd_keyguard.h"
+#include "../../disco/fd_txn_m_t.h"
 #include "generated/fd_send_tile_seccomp.h"
 
 

src/util/hist/fd_histf.h

@@ -7,7 +7,6 @@
 
 #include <math.h> /* FIXME: HMMM */
 #include "../log/fd_log.h"
-
 #if FD_HAS_AVX
 #include "../simd/fd_avx.h"
 #endif