Async Sign Updt: map, flow control

Author: npatel-jump

src/app/firedancer/config/default.toml

@@ -688,10 +688,10 @@ user = ""
     # very high TPS rates because the cluster size will be very small.
     shred_tile_count = 1
 
-    # How many sign tiles to run. Should be set to 2. This is
-    # configurable and designed to scale for the repair tile.
-    # The 0th tile gets used by other tiles for signing messages.  While
-    # the remaining tiles distribute the workload of signing repair
+    # How many sign tiles to run. Should be set >= 2. This is
+    # configurable and horizontally scales repair request signing.
+    # One tile is reserved for synchronous signing across all tiles.
+    # The remaining tiles distribute the workload of signing repair
     # requests.
     sign_tile_count = 2
 

src/app/firedancer/topology.c

@@ -490,7 +490,13 @@ fd_config_validate( fd_config_t const * config ) {
   CFG_HAS_NON_ZERO ( layout.verify_tile_count );
   CFG_HAS_NON_ZERO ( layout.bank_tile_count  );
   CFG_HAS_NON_ZERO ( layout.shred_tile_count );
-  CFG_HAS_NON_ZERO ( firedancer.layout.sign_tile_count );
+
+  if( FD_UNLIKELY( config->is_firedancer ) ) {
+    CFG_HAS_NON_ZERO( firedancer.layout.sign_tile_count );
+    if( FD_UNLIKELY( config->firedancer.layout.sign_tile_count < 2 ) ) {
+      FD_LOG_ERR(( "firedancer.layout.sign_tile_count must be >= 2" ));
+    }
+  }
 
   if( 0U!=config->firedancer.layout.writer_tile_count ) {
     if( FD_UNLIKELY( config->firedancer.layout.writer_tile_count>config->firedancer.layout.exec_tile_count ) ) {

src/app/shared/fd_config.c

@@ -129,8 +129,7 @@ during_frag_sensitive( void * _ctx,
   ulong mtu  = ctx->in[ in_idx ].mtu;
 
   if( chunk<ctx->in[ in_idx ].chunk0 || chunk>ctx->in[ in_idx ].wmark || sz>mtu ) {
-    FD_LOG_EMERG(( "oversz or out of bounds signing request (role=%d chunk=%lu sz=%lu mtu=%lu, chunk0=%lu, wmark=%lu)",
-      role, chunk, sz, mtu, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
+    FD_LOG_EMERG(( "oversz or out of bounds signing request (role=%d chunk=%lu sz=%lu mtu=%lu, chunk0=%lu, wmark=%lu)", role, chunk, sz, mtu, ctx->in[ in_idx ].chunk0, ctx->in[ in_idx ].wmark ));
   }
 
   void * src = fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk );
@@ -172,6 +171,7 @@ after_frag_sensitive( void *              _ctx,
   FD_TEST( in_idx<MAX_IN );
 
   int role = ctx->in[ in_idx ].role;
+
   fd_keyguard_authority_t authority = {0};
   memcpy( authority.identity_pubkey, ctx->public_key, 32 );
 

src/disco/sign/fd_sign_tile.c

@@ -1,8 +1,8 @@
 #include "fd_send_tile.h"
 #include "../../disco/topo/fd_topo.h"
 #include "../../disco/keyguard/fd_keyload.h"
-#include "../../disco/keyguard/fd_keyguard.h"
 #include "../../disco/fd_txn_m_t.h"
+#include "../../disco/keyguard/fd_keyguard.h"
 #include "generated/fd_send_tile_seccomp.h"
 
 

src/discof/repair/fd_repair_tile.c

@@ -3,5 +3,7 @@ $(call add-hdrs,fd_repair.h)
 $(call add-objs,fd_repair,fd_flamenco)
 ifdef FD_HAS_HOSTED
 #$(call make-bin,fd_repair_tool,fd_repair_tool,fd_flamenco fd_ballet fd_util)
+$(call make-unit-test,test_repair,test_repair,fd_flamenco fd_ballet fd_util)
+$(call run-unit-test,test_repair)
 endif
 endif

src/discof/send/fd_send_tile.c

@@ -40,6 +40,12 @@ fd_repair_new ( void * shmem, ulong seed ) {
   glob->peer_idx   = 0;
   glob->actives_random_seed  = 0;
 
+  /* Initialize pending sign request pool and map */
+  shm = FD_SCRATCH_ALLOC_APPEND( l, fd_repair_pending_sign_req_pool_align(), fd_repair_pending_sign_req_pool_footprint( FD_REPAIR_PENDING_SIGN_REQ_MAX ) );
+  glob->pending_sign_req_pool = fd_repair_pending_sign_req_pool_join( fd_repair_pending_sign_req_pool_new( shm, FD_REPAIR_PENDING_SIGN_REQ_MAX ) );
+  shm = FD_SCRATCH_ALLOC_APPEND( l, fd_repair_pending_sign_req_map_align(), fd_repair_pending_sign_req_map_footprint( FD_REPAIR_PENDING_SIGN_REQ_MAX ) );
+  glob->pending_sign_req_map = fd_repair_pending_sign_req_map_join( fd_repair_pending_sign_req_map_new( shm, FD_REPAIR_PENDING_SIGN_REQ_MAX, seed ) );
+
   ulong scratch_top = FD_SCRATCH_ALLOC_FINI(l, 1UL);
   if ( scratch_top > (ulong)shmem + fd_repair_footprint() ) {
     FD_LOG_ERR(("Enough space not allocated for repair"));
@@ -60,6 +66,8 @@ fd_repair_delete ( void * shmap ) {
   fd_active_table_delete( fd_active_table_leave( glob->actives ) );
   fd_inflight_table_delete( fd_inflight_table_leave( glob->dupdetect ) );
   fd_pinged_table_delete( fd_pinged_table_leave( glob->pinged ) );
+  fd_repair_pending_sign_req_pool_delete( fd_repair_pending_sign_req_pool_leave( glob->pending_sign_req_pool ) );
+  fd_repair_pending_sign_req_map_delete( fd_repair_pending_sign_req_map_leave( glob->pending_sign_req_map ) );
   return glob;
 }
 
@@ -366,7 +374,7 @@ fd_repair_create_inflight_request( fd_repair_t * glob, int type, ulong slot, uin
     dupelem->last_send_time = 0L;
   }
 
-  if( FD_LIKELY( dupelem->last_send_time+(long)40e6  < now ) ) { /* 40ms */
+  if( FD_LIKELY( dupelem->last_send_time+(long)80e6  < now ) ) { /* 80ms */
     dupelem->last_send_time = now;
     dupelem->req_cnt        = FD_REPAIR_NUM_NEEDED_PEERS;
     return 1;
@@ -525,3 +533,75 @@ fd_repair_metrics_t *
 fd_repair_get_metrics( fd_repair_t * repair ) {
   return &repair->metrics;
 }
+
+/* Pending Sign Request API
+
+   These functions manage the pool and map of pending sign requests in
+   the repair module. Each request is identified by a unique nonce,
+   allowing for nonce to be used as a key in the map.
+
+  fd_repair_pending_sign_req_t * fd_repair_acquire_pending_request(...);
+    Acquires an empty pending sign request from the pool. Returns
+    pointer or NULL if pool is full. Caller is responsible for setting
+    all fields before adding to map.
+
+  int fd_repair_add_pending_to_map(...);
+    Adds a pending sign request to the map. Returns 0 on success, -1 on
+    failure. The pending request must be previously acquired from the
+    pool.
+
+  fd_repair_pending_sign_req_t * fd_repair_find_pending_request(...);
+    Finds a pending sign request by nonce. Returns pointer or NULL.
+
+  int fd_repair_remove_pending_request(...);
+    Removes a pending sign request by nonce. Returns 0 on success, -1
+    if not found.
+
+   All functions assume the repair context is valid and not used concurrently.
+*/
+
+int
+fd_repair_pending_request_pool_free( fd_repair_t * repair ) {
+  return (int)fd_repair_pending_sign_req_pool_free( repair->pending_sign_req_pool );
+}
+
+fd_repair_pending_sign_req_t *
+fd_repair_acquire_pending_request( fd_repair_t * repair ) {
+  /* Check if there is any space for a new pending sign request */
+  if( FD_UNLIKELY( fd_repair_pending_request_pool_free( repair ) == 0 ) ) {
+    return NULL;
+  }
+
+  fd_repair_pending_sign_req_t * pending = fd_repair_pending_sign_req_pool_ele_acquire( repair->pending_sign_req_pool );
+  return pending;
+}
+
+int
+fd_repair_add_pending_to_map( fd_repair_t * repair,
+                              fd_repair_pending_sign_req_t * pending ) {
+  if( FD_UNLIKELY( !pending ) ) {
+    return -1;
+  }
+
+  fd_repair_pending_sign_req_map_ele_insert( repair->pending_sign_req_map, pending, repair->pending_sign_req_pool );
+  return 0;
+}
+
+fd_repair_pending_sign_req_t *
+fd_repair_find_pending_request( fd_repair_t * repair,
+                                ulong         nonce ) {
+  return fd_repair_pending_sign_req_map_ele_query( repair->pending_sign_req_map, &nonce, NULL, repair->pending_sign_req_pool );
+}
+
+int
+fd_repair_remove_pending_request( fd_repair_t * repair,
+                                  ulong         nonce ) {
+  fd_repair_pending_sign_req_t * pending = fd_repair_pending_sign_req_map_ele_query( repair->pending_sign_req_map, &nonce, NULL, repair->pending_sign_req_pool );
+  if( FD_UNLIKELY( !pending ) ) {
+    return -1;
+  }
+
+  fd_repair_pending_sign_req_map_ele_remove( repair->pending_sign_req_map, &nonce, NULL, repair->pending_sign_req_pool );
+  fd_repair_pending_sign_req_pool_ele_release( repair->pending_sign_req_pool, pending );
+  return 0;
+}

src/flamenco/repair/Local.mk

@@ -34,6 +34,10 @@
 #define FD_PING_PRE_IMAGE_SZ (48UL)
 /* Number of peers to send requests to. */
 #define FD_REPAIR_NUM_NEEDED_PEERS (1)
+/* Max number of pending sign requests */
+#define FD_REPAIR_PENDING_SIGN_REQ_MAX (1<<10)
+/* Maximum size for sign buffer, typically <= 160 bytes (e.g., pings, repairs) */
+#define FD_REPAIR_MAX_SIGN_BUF_SIZE (256UL)
 
 typedef fd_gossip_peer_addr_t fd_repair_peer_addr_t;
 
@@ -165,6 +169,28 @@ typedef struct fd_pinged_elem fd_pinged_elem_t;
 #define MAP_T        fd_pinged_elem_t
 #include "../../util/tmpl/fd_map_giant.c"
 
+/* Pending sign request structure for async request handling */
+struct fd_repair_pending_sign_req {
+  ulong       nonce;        /* map key, unique nonce */
+  ulong       next;         /* used internally by fd_map_chain */
+  uchar       buf[FD_REPAIR_MAX_SIGN_BUF_SIZE];
+  ulong       buflen;
+  ulong       sig_offset;
+  uint        dst_ip_addr;
+  ushort      dst_port;
+  fd_pubkey_t recipient;
+};
+typedef struct fd_repair_pending_sign_req fd_repair_pending_sign_req_t;
+
+#define POOL_NAME   fd_repair_pending_sign_req_pool
+#define POOL_T      fd_repair_pending_sign_req_t
+#include "../../util/tmpl/fd_pool.c"
+
+#define MAP_NAME     fd_repair_pending_sign_req_map
+#define MAP_KEY      nonce
+#define MAP_ELE_T    fd_repair_pending_sign_req_t
+#include "../../util/tmpl/fd_map_chain.c"
+
 struct fd_peer {
   fd_pubkey_t   key;
   fd_ip4_port_t ip4;
@@ -238,6 +264,9 @@ struct fd_repair {
     fd_vote_stake_weight_t * stake_weights_temp;
     /* Path to the file where we write the cache of known good repair peers, to make cold booting faster */
     int good_peer_cache_file_fd;
+    /* Pending sign requests for async operations */
+    fd_repair_pending_sign_req_t      * pending_sign_req_pool;
+    fd_repair_pending_sign_req_map_t  * pending_sign_req_map;
     /* Metrics */
     fd_repair_metrics_t metrics;
 };
@@ -249,13 +278,16 @@ fd_repair_align ( void ) { return 128UL; }
 FD_FN_CONST static inline ulong
 fd_repair_footprint( void ) {
   ulong l = FD_LAYOUT_INIT;
-  l = FD_LAYOUT_APPEND( l, alignof(fd_repair_t), sizeof(fd_repair_t) );
-  l = FD_LAYOUT_APPEND( l, fd_active_table_align(), fd_active_table_footprint(FD_ACTIVE_KEY_MAX) );
-  l = FD_LAYOUT_APPEND( l, fd_inflight_table_align(), fd_inflight_table_footprint(FD_NEEDED_KEY_MAX) );
-  l = FD_LAYOUT_APPEND( l, fd_pinged_table_align(), fd_pinged_table_footprint(FD_REPAIR_PINGED_MAX) );
+  l = FD_LAYOUT_APPEND( l, alignof(fd_repair_t),                    sizeof(fd_repair_t) );
+  l = FD_LAYOUT_APPEND( l, fd_active_table_align(),                 fd_active_table_footprint(FD_ACTIVE_KEY_MAX) );
+  l = FD_LAYOUT_APPEND( l, fd_inflight_table_align(),               fd_inflight_table_footprint(FD_NEEDED_KEY_MAX) );
+  l = FD_LAYOUT_APPEND( l, fd_pinged_table_align(),                 fd_pinged_table_footprint(FD_REPAIR_PINGED_MAX) );
   /* regular and temp stake weights */
-  l = FD_LAYOUT_APPEND( l, alignof(fd_vote_stake_weight_t), FD_STAKE_WEIGHTS_MAX * sizeof(fd_vote_stake_weight_t) );
-  l = FD_LAYOUT_APPEND( l, alignof(fd_vote_stake_weight_t), FD_STAKE_WEIGHTS_MAX * sizeof(fd_vote_stake_weight_t) );
+  l = FD_LAYOUT_APPEND( l, alignof(fd_vote_stake_weight_t),         FD_STAKE_WEIGHTS_MAX * sizeof(fd_vote_stake_weight_t) );
+  l = FD_LAYOUT_APPEND( l, alignof(fd_vote_stake_weight_t),         FD_STAKE_WEIGHTS_MAX * sizeof(fd_vote_stake_weight_t) );
+  /* pending sign request structures */
+  l = FD_LAYOUT_APPEND( l, fd_repair_pending_sign_req_pool_align(), fd_repair_pending_sign_req_pool_footprint( FD_REPAIR_PENDING_SIGN_REQ_MAX ) );
+  l = FD_LAYOUT_APPEND( l, fd_repair_pending_sign_req_map_align(),  fd_repair_pending_sign_req_map_footprint( FD_REPAIR_PENDING_SIGN_REQ_MAX ) );
   return FD_LAYOUT_FINI(l, fd_repair_align() );
 }
 
@@ -330,5 +362,24 @@ void fd_repair_set_stake_weights_fini( fd_repair_t * repair );
 fd_repair_metrics_t *
 fd_repair_get_metrics( fd_repair_t * repair );
 
+/* Pending sign request operations */
+fd_repair_pending_sign_req_t *
+fd_repair_acquire_pending_request( fd_repair_t * repair );
+
+int
+fd_repair_add_pending_to_map( fd_repair_t * repair,
+                              fd_repair_pending_sign_req_t * pending );
+
+fd_repair_pending_sign_req_t *
+fd_repair_find_pending_request( fd_repair_t * repair,
+                                ulong         nonce );
+
+int
+fd_repair_remove_pending_request( fd_repair_t * repair,
+                                  ulong         nonce );
+
+int
+fd_repair_pending_request_pool_free( fd_repair_t * repair );
+
 
 #endif /* HEADER_fd_src_flamenco_repair_fd_repair_h */