Skip to content

Commit 8ca1386

Browse files
npatel-jumpnpatel-jump
committed
repair: worker pool async sign and ping-pong opt
Enable asynchronous signing requests in the repair flow. Cleanup and document async repair signing logic. Update for flow control and mapping improvements.
1 parent 315e29b commit 8ca1386

File tree

12 files changed

+1101
-154
lines changed

12 files changed

+1101
-154
lines changed

‎src/app/firedancer-dev/commands/repair.c‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ repair_topo( config_t * config ) {
6161
ulong net_tile_cnt = config->layout.net_tile_count;
6262
ulong shred_tile_cnt = config->layout.shred_tile_count;
6363
ulong quic_tile_cnt = config->layout.quic_tile_count;
64+
ulong sign_tile_cnt = config->firedancer.layout.sign_tile_count;
6465

6566
fd_topo_t * topo = { fd_topob_new( &config->topo, config->name ) };
6667
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
@@ -131,11 +132,16 @@ repair_topo( config_t * config ) {
131132
/**/ fd_topob_link( topo, "gossip_net", "net_gossip", config->net.ingress_buffer_size, FD_NET_MTU, 1UL );
132133

133134
/**/ fd_topob_link( topo, "repair_net", "net_repair", config->net.ingress_buffer_size, FD_NET_MTU, 1UL );
134-
/**/ fd_topob_link( topo, "repair_sign", "repair_sign", 128UL, 2048UL, 1UL );
135+
135136
FOR(shred_tile_cnt) fd_topob_link( topo, "shred_repair", "shred_repair", pending_fec_shreds_depth, FD_SHRED_REPAIR_MTU, 2UL /* at most 2 msgs per after_frag */ );
136137

137138
FOR(shred_tile_cnt) fd_topob_link( topo, "repair_shred", "shred_repair", pending_fec_shreds_depth, sizeof(fd_ed25519_sig_t), 1UL );
138-
/**/ fd_topob_link( topo, "sign_repair", "sign_repair", 128UL, 64UL, 1UL );
139+
140+
/**/ fd_topob_link( topo, "ping_sign", "repair_sign", 128UL, 2048UL, 1UL );
141+
/**/ fd_topob_link( topo, "sign_ping", "sign_repair", 128UL, sizeof(fd_ed25519_sig_t), 1UL );
142+
FOR(sign_tile_cnt-1) fd_topob_link( topo, "repair_sign", "repair_sign", 128UL, 2048UL, 1UL );
143+
FOR(sign_tile_cnt-1) fd_topob_link( topo, "sign_repair", "sign_repair", 1024UL, sizeof(fd_ed25519_sig_t), 1UL );
144+
139145
/**/ fd_topob_link( topo, "repair_repla", "repair_repla", 65536UL, sizeof(fd_fec_out_t), 1UL );
140146
/**/ fd_topob_link( topo, "poh_shred", "poh_shred", 16384UL, USHORT_MAX, 1UL );
141147

@@ -177,7 +183,7 @@ repair_topo( config_t * config ) {
177183

178184
/* topo, tile_name, tile_wksp, metrics_wksp, cpu_idx, is_agave, uses_keyswitch */
179185
FOR(shred_tile_cnt) fd_topob_tile( topo, "shred", "shred", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
180-
/**/ fd_topob_tile( topo, "sign", "sign", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
186+
FOR(sign_tile_cnt) fd_topob_tile( topo, "sign", "sign", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
181187
/**/ fd_topob_tile( topo, "metric", "metric", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
182188
/**/ fd_topob_tile( topo, "gossip", "gossip", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
183189
fd_topo_tile_t * repair_tile = fd_topob_tile( topo, "repair", "repair", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
@@ -298,12 +304,17 @@ repair_topo( config_t * config ) {
298304
/**/ fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "stake_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
299305
FOR(shred_tile_cnt) fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "shred_repair", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
300306
301-
/**/ fd_topob_tile_in( topo, "sign", 0UL, "metric_in", "repair_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
302-
/**/ fd_topob_tile_out( topo, "repair", 0UL, "repair_sign", 0UL );
303-
/**/ fd_topob_tile_in( topo, "repair", 0UL, "metric_in", "sign_repair", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
307+
/**/ fd_topob_tile_in( topo, "sign", 0UL, "metric_in", "ping_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
308+
/**/ fd_topob_tile_out( topo, "repair", 0UL, "ping_sign", 0UL );
304309
/**/ fd_topob_tile_out( topo, "repair", 0UL, "repair_repla", 0UL );
305310
FOR(shred_tile_cnt) fd_topob_tile_out( topo, "repair", 0UL, "repair_shred", i );
306-
/**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_repair", 0UL );
311+
/**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_ping", 0UL );
312+
313+
FOR(sign_tile_cnt-1) fd_topob_tile_out( topo, "repair", 0UL, "repair_sign", i );
314+
FOR(sign_tile_cnt-1) fd_topob_tile_in ( topo, "sign", i+1, "metric_in", "repair_sign", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
315+
FOR(sign_tile_cnt-1) fd_topob_tile_out( topo, "sign", i+1, "sign_repair", i );
316+
FOR(sign_tile_cnt-1) fd_topob_tile_in ( topo, "repair", 0UL, "metric_in", "sign_repair", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
317+
/**/ fd_topob_tile_in ( topo, "repair", 0UL, "metric_in", "sign_ping", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED );
307318

308319
if( 1 ) {
309320
fd_topob_wksp( topo, "scap" );

‎src/app/firedancer/config/default.toml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,13 @@ user = ""
688688
# very high TPS rates because the cluster size will be very small.
689689
shred_tile_count = 1
690690

691+
# How many sign tiles to run. Should be set >= 2. This is
692+
# configurable and horizontally scales repair request signing.
693+
# One tile is reserved for synchronous signing across all tiles.
694+
# The remaining tiles distribute the workload of signing repair
695+
# requests.
696+
sign_tile_count = 2
697+
691698
# All memory that will be used in Firedancer is pre-allocated in two
692699
# kinds of pages: huge and gigantic. Huge pages are 2 MiB and gigantic
693700
# pages are 1 GiB. This is done to prevent TLB misses which can have a

0 commit comments

Comments
 (0)