Certificate benchmark as a function of number of voters (#192)

bwbush · web-flow · commit 8bcf660d02df · 2025-02-13T12:59:29.000-07:00
* Added committee-size benchmarks for certificate operations
* Documented certificate benchmarks as a function of committee size
* Updated logbook
* Relocated function for realistic stake distribution
* Removed lint
diff --git a/Logbook.md b/Logbook.md
@@ -1,5 +1,24 @@
 # Leios logbook
 
+## 2025-02-13
+
+### Certificate CPU benchmarks as a function of number of voters
+
+In support of the Haskell and Rust simulations, we've benchmarked certificate operations as a function of the number of voters. A realistic distribution of stake is used in these measurements.
+
+| Number of pools | Number of committee seats | Generate certificate | Verify certificate | Weigh certificate |
+|----------------:|--------------------------:|---------------------:|-------------------:|------------------:|
+|            2500 |                       500 |              63.4 ms |           104.8 ms |           10.6 ms |
+|            2500 |                       600 |              71.1 ms |           116.9 ms |           12.0 ms |
+|            2500 |                       700 |              77.4 ms |           125.5 ms |           12.3 ms |
+|            2500 |                       800 |              83.5 ms |           134.4 ms |           12.8 ms |
+|            2500 |                       900 |              88.2 ms |           141.1 ms |           12.4 ms |
+|            2500 |                      1000 |              92.5 ms |           144.9 ms |           12.3 ms |
+
+Serialization and deserialization likely also exhibit the same trend.
+
+A recipe for parallelizing parts of the certificate operations has been added to the [Specification for BLS certificates](crypto-benchmarks.rs/Specification.md).
+
 ## 2025-02-12
 
 ### Added BLS crypto to CI
diff --git a/crypto-benchmarks.rs/Specification.md b/crypto-benchmarks.rs/Specification.md
@@ -142,6 +142,17 @@ but not including any minor overhead arising from CBOR serialization. As noted p
 
 As a general rule of thumb, assume that 80% of votes are persistent and 20% are non-persistent.
 
+Here are details for how certificate operations vary with committee size.
+
+| Number of pools | Number of committee seats | Generate certificate | Verify certificate | Weigh certificate |
+|----------------:|--------------------------:|---------------------:|-------------------:|------------------:|
+|            2500 |                       500 |              63.4 ms |           104.8 ms |           10.6 ms |
+|            2500 |                       600 |              71.1 ms |           116.9 ms |           12.0 ms |
+|            2500 |                       700 |              77.4 ms |           125.5 ms |           12.3 ms |
+|            2500 |                       800 |              83.5 ms |           134.4 ms |           12.8 ms |
+|            2500 |                       900 |              88.2 ms |           141.1 ms |           12.4 ms |
+|            2500 |                      1000 |              92.5 ms |           144.9 ms |           12.3 ms |
+
 
 ## Certificate size for realistic stake distributions
 
diff --git a/crypto-benchmarks.rs/benches/vote_bench.rs b/crypto-benchmarks.rs/benches/vote_bench.rs
@@ -2,6 +2,7 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use leios_crypto_benchmarks::cert::*;
 use leios_crypto_benchmarks::registry::{arbitrary_pools, PersistentId, Registry};
 use quickcheck::{Arbitrary, Gen};
+use std::env;
 
 use leios_crypto_benchmarks::key::{check_pop, key_gen, SecKey};
 use leios_crypto_benchmarks::primitive::{
@@ -101,14 +102,22 @@ fn benchmark_verify_vote_nonpersistent(c: &mut Criterion) {
     });
 }
 
-fn benchmark_gen_cert(c: &mut Criterion) {
+fn benchmark_gen_cert_impl(c: &mut Criterion, name: &'static str, n_pools: usize, n_voters: usize) {
     let g = &mut Gen::new(10);
-    c.bench_function("cert::gen_cert", |b| {
+    c.bench_function(name, |b| {
         b.iter_batched(
             || {
                 let total = realistic_total_stake(g);
-                let n = realistic_pool_count(g);
-                let voters = realistic_voters(g, n);
+                let n = if n_pools > 0 {
+                    n_pools
+                } else {
+                    realistic_pool_count(g)
+                };
+                let voters = if n_voters > 0 {
+                    n_voters
+                } else {
+                    realistic_voters(g, n)
+                };
                 let stake = arbitrary_stake_distribution(g, total, n, 11., 1.);
                 let pools = arbitrary_pools(g, &stake);
                 let reg = Registry::make(&pools, voters);
@@ -121,14 +130,69 @@ fn benchmark_gen_cert(c: &mut Criterion) {
     });
 }
 
-fn benchmark_verify_cert(c: &mut Criterion) {
+fn benchmark_gen_cert(c: &mut Criterion) {
+    benchmark_gen_cert_impl(c, "cert::gen_cert", 0, 0);
+    if env::var("BENCHMARK_CERT_EXTRA").map(|x| x == "1") == Result::Ok(true) {
+        benchmark_gen_cert_impl(
+            c,
+            "cert::gen_cert, n_pools = 2500, n_voters = 500",
+            2500,
+            500,
+        );
+        benchmark_gen_cert_impl(
+            c,
+            "cert::gen_cert, n_pools = 2500, n_voters = 600",
+            2500,
+            600,
+        );
+        benchmark_gen_cert_impl(
+            c,
+            "cert::gen_cert, n_pools = 2500, n_voters = 700",
+            2500,
+            700,
+        );
+        benchmark_gen_cert_impl(
+            c,
+            "cert::gen_cert, n_pools = 2500, n_voters = 800",
+            2500,
+            800,
+        );
+        benchmark_gen_cert_impl(
+            c,
+            "cert::gen_cert, n_pools = 2500, n_voters = 900",
+            2500,
+            900,
+        );
+        benchmark_gen_cert_impl(
+            c,
+            "cert::gen_cert, n_pools = 2500, n_voters = 1000",
+            2500,
+            1000,
+        );
+    }
+}
+
+fn benchmark_verify_cert_impl(
+    c: &mut Criterion,
+    name: &'static str,
+    n_pools: usize,
+    n_voters: usize,
+) {
     let g = &mut Gen::new(10);
-    c.bench_function("cert::verify_cert", |b| {
+    c.bench_function(name, |b| {
         b.iter_batched(
             || {
                 let total = realistic_total_stake(g);
-                let n = realistic_pool_count(g);
-                let voters = realistic_voters(g, n);
+                let n = if n_pools > 0 {
+                    n_pools
+                } else {
+                    realistic_pool_count(g)
+                };
+                let voters = if n_voters > 0 {
+                    n_voters
+                } else {
+                    realistic_voters(g, n)
+                };
                 let stake = arbitrary_stake_distribution(g, total, n, 11., 1.);
                 let pools = arbitrary_pools(g, &stake);
                 let reg = Registry::make(&pools, voters);
@@ -142,14 +206,69 @@ fn benchmark_verify_cert(c: &mut Criterion) {
     });
 }
 
-fn benchmark_weigh_cert(c: &mut Criterion) {
+fn benchmark_verify_cert(c: &mut Criterion) {
+    benchmark_verify_cert_impl(c, "cert::verify_cert", 0, 0);
+    if env::var("BENCHMARK_CERT_EXTRA").map(|x| x == "1") == Result::Ok(true) {
+        benchmark_verify_cert_impl(
+            c,
+            "cert::verify_cert, n_pools = 2500, n_voters = 500",
+            2500,
+            500,
+        );
+        benchmark_verify_cert_impl(
+            c,
+            "cert::verify_cert, n_pools = 2500, n_voters = 600",
+            2500,
+            600,
+        );
+        benchmark_verify_cert_impl(
+            c,
+            "cert::verify_cert, n_pools = 2500, n_voters = 700",
+            2500,
+            700,
+        );
+        benchmark_verify_cert_impl(
+            c,
+            "cert::verify_cert, n_pools = 2500, n_voters = 800",
+            2500,
+            800,
+        );
+        benchmark_verify_cert_impl(
+            c,
+            "cert::verify_cert, n_pools = 2500, n_voters = 900",
+            2500,
+            900,
+        );
+        benchmark_verify_cert_impl(
+            c,
+            "cert::verify_cert, n_pools = 2500, n_voters = 1000",
+            2500,
+            1000,
+        );
+    }
+}
+
+fn benchmark_weigh_cert_impl(
+    c: &mut Criterion,
+    name: &'static str,
+    n_pools: usize,
+    n_voters: usize,
+) {
     let g = &mut Gen::new(10);
-    c.bench_function("cert::weigh_cert", |b| {
+    c.bench_function(name, |b| {
         b.iter_batched(
             || {
                 let total = realistic_total_stake(g);
-                let n = realistic_pool_count(g);
-                let voters = realistic_voters(g, n);
+                let n = if n_pools > 0 {
+                    n_pools
+                } else {
+                    realistic_pool_count(g)
+                };
+                let voters = if n_voters > 0 {
+                    n_voters
+                } else {
+                    realistic_voters(g, n)
+                };
                 let stake = arbitrary_stake_distribution(g, total, n, 11., 1.);
                 let pools = arbitrary_pools(g, &stake);
                 let reg = Registry::make(&pools, voters);
@@ -162,6 +281,47 @@ fn benchmark_weigh_cert(c: &mut Criterion) {
         )
     });
 }
+fn benchmark_weigh_cert(c: &mut Criterion) {
+    benchmark_weigh_cert_impl(c, "cert::weigh_cert", 0, 0);
+    if env::var("BENCHMARK_CERT_EXTRA").map(|x| x == "1") == Result::Ok(true) {
+        benchmark_weigh_cert_impl(
+            c,
+            "cert::weigh_cert, n_pools = 2500, n_voters = 500",
+            2500,
+            500,
+        );
+        benchmark_weigh_cert_impl(
+            c,
+            "cert::weigh_cert, n_pools = 2500, n_voters = 600",
+            2500,
+            600,
+        );
+        benchmark_weigh_cert_impl(
+            c,
+            "cert::weigh_cert, n_pools = 2500, n_voters = 700",
+            2500,
+            700,
+        );
+        benchmark_weigh_cert_impl(
+            c,
+            "cert::weigh_cert, n_pools = 2500, n_voters = 800",
+            2500,
+            800,
+        );
+        benchmark_weigh_cert_impl(
+            c,
+            "cert::weigh_cert, n_pools = 2500, n_voters = 900",
+            2500,
+            900,
+        );
+        benchmark_weigh_cert_impl(
+            c,
+            "cert::weigh_cert, n_pools = 2500, n_voters = 1000",
+            2500,
+            1000,
+        );
+    }
+}
 
 criterion_group!(
     benches,
diff --git a/crypto-benchmarks.rs/src/main.rs b/crypto-benchmarks.rs/src/main.rs
@@ -15,8 +15,7 @@ use leios_crypto_benchmarks::key::{
     check_pop, key_gen, sign_message, verify_message, PoP, PubKey, Reg, SecKey, Sig,
 };
 use leios_crypto_benchmarks::primitive::{
-    arbitrary_poolkeyhash, arbitrary_stake_distribution, Coin, EbHash, Eid, KesSig,
-    PoolKeyhash,
+    arbitrary_poolkeyhash, arbitrary_stake_distribution, Coin, EbHash, Eid, KesSig, PoolKeyhash,
 };
 use leios_crypto_benchmarks::registry::{arbitrary_pools, PoolInfo, Registry};
 use leios_crypto_benchmarks::sortition::voter_check;
diff --git a/crypto-benchmarks.rs/src/primitive.rs b/crypto-benchmarks.rs/src/primitive.rs
@@ -5,14 +5,10 @@ use num_traits::FromPrimitive;
 use pallas::ledger::primitives::{byron::Blake2b256, Hash};
 use pallas::ledger::traverse::time::Slot;
 use quickcheck::{Arbitrary, Gen};
-use rand::prelude::Distribution;
-use rand::rngs::StdRng;
-use rand::SeedableRng;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
-use statrs::distribution::ContinuousCDF;
-use statrs::distribution::{Beta, Uniform};
 use std::collections::BTreeMap;
 
+use crate::realism::realistic_stake_dist;
 use crate::util::{arbitrary_fixed_bytes, deserialize_fixed_bytes, serialize_fixed_bytes};
 
 pub use pallas::ledger::primitives::PoolKeyhash;
@@ -27,28 +23,6 @@ pub fn arbitrary_coin(g: &mut Gen) -> Coin {
     u64::arbitrary(g) % 999999 + 1
 }
 
-pub(crate) fn realistic_stake_dist(
-    g: &mut Gen,
-    total: u64,
-    n: usize,
-    alpha: f64,
-    beta: f64,
-) -> Vec<Coin> {
-    let rng = &mut StdRng::seed_from_u64(u64::arbitrary(g));
-    let noise = Uniform::new(0.75, 1.25).unwrap();
-    let curve = Beta::new(alpha, beta).unwrap();
-    let cum: Vec<f64> = (0..n)
-        .map(|i| curve.cdf((i as f64) / (total as f64)))
-        .collect();
-    let dif: Vec<f64> = (1..n)
-        .map(|i| (cum[i] - cum[i - 1]) * noise.sample(rng))
-        .collect();
-    let scale: f64 = (total as f64) / dif.iter().sum::<f64>();
-    dif.iter()
-        .map(|coin| (scale * *coin).round() as Coin)
-        .collect()
-}
-
 pub fn arbitrary_stake_distribution(
     g: &mut Gen,
     total: u64,
diff --git a/crypto-benchmarks.rs/src/realism.rs b/crypto-benchmarks.rs/src/realism.rs
@@ -1,6 +1,10 @@
+use pallas::ledger::primitives::Coin;
 use quickcheck::{Arbitrary, Gen};
-
-use crate::primitive::Coin;
+use rand::prelude::Distribution;
+use rand::rngs::StdRng;
+use rand::SeedableRng;
+use statrs::distribution::ContinuousCDF;
+use statrs::distribution::{Beta, Uniform};
 
 pub fn realistic_pool_count(g: &mut Gen) -> usize {
     usize::arbitrary(g) % 1500 + 1500
@@ -13,3 +17,25 @@ pub fn realistic_total_stake(g: &mut Gen) -> Coin {
 pub fn realistic_voters(g: &mut Gen, pools: usize) -> usize {
     pools * (usize::arbitrary(g) % 500 + 500) / 1000
 }
+
+pub(crate) fn realistic_stake_dist(
+    g: &mut Gen,
+    total: u64,
+    n: usize,
+    alpha: f64,
+    beta: f64,
+) -> Vec<Coin> {
+    let rng = &mut StdRng::seed_from_u64(u64::arbitrary(g));
+    let noise = Uniform::new(0.75, 1.25).unwrap();
+    let curve = Beta::new(alpha, beta).unwrap();
+    let cum: Vec<f64> = (0..n)
+        .map(|i| curve.cdf((i as f64) / (total as f64)))
+        .collect();
+    let dif: Vec<f64> = (1..n)
+        .map(|i| (cum[i] - cum[i - 1]) * noise.sample(rng))
+        .collect();
+    let scale: f64 = (total as f64) / dif.iter().sum::<f64>();
+    dif.iter()
+        .map(|coin| (scale * *coin).round() as Coin)
+        .collect()
+}
