snapshots: lthash

Author: cali-jumptrading

book/api/metrics-generated.md

@@ -916,6 +916,41 @@
 
 </div>
 
+## Snapwr Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">snapwr_&#8203;state</span> | gauge | State of the tile. 0=IDLE, 1=PROCESSING, 4=SHUTDOWN |
+| <span class="metrics-name">snapwr_&#8203;vinyl_&#8203;bytes_&#8203;written</span> | gauge | Number of bytes written so far to the vinyl snapshot file. Might decrease if snapshot creation is aborted and restarted |
+
+</div>
+
+## Snaplta Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">snaplta_&#8203;state</span> | gauge | State of the tile. 0=hashing, 1=done, 2=shutdown |
+| <span class="metrics-name">snaplta_&#8203;full_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the full snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
+| <span class="metrics-name">snaplta_&#8203;incremental_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the incremental snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
+
+</div>
+
+## Snaplts Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">snaplts_&#8203;state</span> | gauge | State of the tile. 0=hashing, 1=done, 2=shutdown |
+| <span class="metrics-name">snaplts_&#8203;full_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the full snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
+| <span class="metrics-name">snaplts_&#8203;incremental_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the incremental snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
+
+</div>
+
 ## Ipecho Tile
 
 <div class="metrics">
@@ -1096,14 +1131,3 @@
 | <span class="metrics-name">exec_&#8203;progcache_&#8203;invalidations</span> | counter | Number of program cache invalidations |
 
 </div>
-
-## Snapwr Tile
-
-<div class="metrics">
-
-| Metric | Type | Description |
-|--------|------|-------------|
-| <span class="metrics-name">snapwr_&#8203;state</span> | gauge | State of the tile. 0=IDLE, 1=PROCESSING, 4=SHUTDOWN |
-| <span class="metrics-name">snapwr_&#8203;vinyl_&#8203;bytes_&#8203;written</span> | gauge | Number of bytes written so far to the vinyl snapshot file. Might decrease if snapshot creation is aborted and restarted |
-
-</div>

src/app/firedancer-dev/commands/backtest.c

@@ -47,10 +47,12 @@ backtest_topo( config_t * config ) {
   config->development.no_clone = 1;
 
   ulong exec_tile_cnt   = config->firedancer.layout.exec_tile_count;
+  ulong lta_tile_cnt    = config->firedancer.layout.snaplta_tile_count;
 
-  int disable_snap_loader = !config->gossip.entrypoints_cnt;
-  int snap_vinyl          = !!config->firedancer.vinyl.enabled;
-  int solcap_enabled      = strlen( config->capture.solcap_capture )>0;
+  int disable_snap_loader      = !config->gossip.entrypoints_cnt;
+  int snap_vinyl               = !!config->firedancer.vinyl.enabled;
+  int solcap_enabled           = strlen( config->capture.solcap_capture )>0;
+  int snapshot_lthash_disabled = config->development.snapshots.disable_lthash_verification;
 
   fd_topo_t * topo = { fd_topob_new( &config->topo, config->name ) };
   topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
@@ -109,10 +111,20 @@ backtest_topo( config_t * config ) {
     fd_topob_wksp( topo, "snapld" );
     fd_topob_wksp( topo, "snapdc" );
     fd_topob_wksp( topo, "snapin" );
+
+    if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
+      fd_topob_wksp( topo, "snaplta" );
+      fd_topob_wksp( topo, "snaplts" );
+    }
+
     fd_topo_tile_t * snapct_tile = fd_topob_tile( topo, "snapct",  "snapct",  "metric_in",  cpu_idx++, 0, 0 );
     fd_topo_tile_t * snapld_tile = fd_topob_tile( topo, "snapld",  "snapld",  "metric_in",  cpu_idx++, 0, 0 );
     fd_topo_tile_t * snapdc_tile = fd_topob_tile( topo, "snapdc",  "snapdc",  "metric_in",  cpu_idx++, 0, 0 );
                      snapin_tile = fd_topob_tile( topo, "snapin",  "snapin",  "metric_in",  cpu_idx++, 0, 0 );
+    if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
+      FOR(lta_tile_cnt)  fd_topob_tile( topo, "snaplta", "snaplta", "metric_in", cpu_idx++,  0, 0 )->allow_shutdown = 1;
+      /**/               fd_topob_tile( topo, "snaplts", "snaplts", "metric_in", cpu_idx++,  0, 0 )->allow_shutdown = 1;
+    }
     snapct_tile->allow_shutdown = 1;
     snapld_tile->allow_shutdown = 1;
     snapdc_tile->allow_shutdown = 1;
@@ -143,18 +155,40 @@ backtest_topo( config_t * config ) {
     fd_topob_wksp( topo, "snapct_ld"    );
     fd_topob_wksp( topo, "snapld_dc"    );
     fd_topob_wksp( topo, "snapdc_in"    );
-    fd_topob_wksp( topo, "snapin_ct"    );
+    if( FD_UNLIKELY( snapshot_lthash_disabled ) ) {
+      fd_topob_wksp( topo, "snapin_ct"   );
+    } else {
+      fd_topob_wksp( topo, "snaplts_ct" );
+    }
+
     fd_topob_wksp( topo, "snapin_manif" );
     fd_topob_wksp( topo, "snapct_repr"  );
 
+    if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
+      fd_topob_wksp( topo, "snaplta_lts" );
+      fd_topob_wksp( topo, "snapin_lts" );
+    }
+
     fd_topob_link( topo, "snapct_ld",    "snapct_ld",    128UL,   sizeof(fd_ssctrl_init_t),       1UL );
     fd_topob_link( topo, "snapld_dc",    "snapld_dc",    16384UL, USHORT_MAX,                     1UL );
     fd_topob_link( topo, "snapdc_in",    "snapdc_in",    16384UL, USHORT_MAX,                     1UL );
-    fd_topob_link( topo, "snapin_ct",    "snapin_ct",    128UL,   0UL,                            1UL );
+    if( FD_UNLIKELY( snapshot_lthash_disabled ) ) {
+      fd_topob_link( topo, "snapin_ct",  "snapin_ct",    128UL,   0UL,                            1UL );
+    }
     fd_topob_link( topo, "snapin_manif", "snapin_manif", 4UL,     sizeof(fd_snapshot_manifest_t), 1UL ); /* TODO: Should be depth 1 or 2 but replay backpressures */
     fd_topob_link( topo, "snapct_repr",  "snapct_repr",  128UL,   0UL,                            1UL )->permit_no_consumers = 1;
 
-    fd_topob_tile_in ( topo, "snapct",  0UL, "metric_in", "snapin_ct",    0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+    if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
+      FOR(lta_tile_cnt) fd_topob_link( topo, "snaplta_lts", "snaplta_lts",  128UL,  sizeof(fd_lthash_value_t),              1UL );
+      /**/              fd_topob_link( topo, "snapin_lts",  "snapin_lts",   1024UL, sizeof(fd_snapshot_existing_account_t), 1UL );
+      /**/              fd_topob_link( topo, "snaplts_ct",  "snaplts_ct",   128UL,  0UL,                                    1UL );
+      }
+
+    if( FD_UNLIKELY( snapshot_lthash_disabled ) ) {
+      fd_topob_tile_in ( topo, "snapct",  0UL, "metric_in", "snapin_ct",    0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+    } else {
+      fd_topob_tile_in ( topo, "snapct",  0UL, "metric_in", "snaplts_ct",   0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+    }
     fd_topob_tile_in ( topo, "snapct",  0UL, "metric_in", "snapld_dc",    0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
     fd_topob_tile_out( topo, "snapct",  0UL,              "snapct_ld",    0UL                                       );
     fd_topob_tile_out( topo, "snapct",  0UL,              "snapct_repr",  0UL                                       );
@@ -163,9 +197,22 @@ backtest_topo( config_t * config ) {
     fd_topob_tile_in ( topo, "snapdc",  0UL, "metric_in", "snapld_dc",    0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
     fd_topob_tile_out( topo, "snapdc",  0UL,              "snapdc_in",    0UL                                       );
     fd_topob_tile_in ( topo, "snapin",  0UL, "metric_in", "snapdc_in",    0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
-    fd_topob_tile_out( topo, "snapin",  0UL,              "snapin_ct",    0UL                                       );
+    if( FD_UNLIKELY( snapshot_lthash_disabled ) ) {
+      fd_topob_tile_out( topo, "snapin",  0UL, "snapin_ct",  0UL );
+    } else {
+      fd_topob_tile_out( topo, "snapin",  0UL, "snapin_lts", 0UL );
+    }
+
     fd_topob_tile_out( topo, "snapin",  0UL,              "snapin_manif", 0UL                                       );
     fd_topob_tile_in ( topo, "replay",  0UL, "metric_in", "snapin_manif", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED   );
+
+    if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
+      FOR(lta_tile_cnt) fd_topob_tile_in(     topo, "snaplta", i,   "metric_in", "snapdc_in",    0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+      FOR(lta_tile_cnt) fd_topob_tile_out(    topo, "snaplta", i,                "snaplta_lts",  i                                         );
+      /**/              fd_topob_tile_in(     topo, "snaplts", 0UL, "metric_in", "snapin_lts",   0UL, FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+      FOR(lta_tile_cnt) fd_topob_tile_in(     topo, "snaplts", 0UL, "metric_in", "snaplta_lts",  i,   FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+      /**/              fd_topob_tile_out(    topo, "snaplts", 0UL,              "snaplts_ct",   0UL                                       );
+    }
   } else {
     fd_topob_wksp( topo, "genesi_out" );
     fd_topob_link( topo, "genesi_out", "genesi_out", 2UL, 10UL*1024UL*1024UL+32UL+sizeof(fd_lthash_value_t), 1UL );