diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index a56863d96..5e1b2dfde 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -23,3 +23,8 @@ serde = ["dep:serde", "bitcoin/serde", "hashbrown?/serde"] [dev-dependencies] bdk_chain = { path = "../chain" } bdk_testenv = { path = "../testenv", default-features = false } +criterion = { version = "0.2" } + +[[bench]] +name = "checkpoint_skiplist" +harness = false diff --git a/crates/core/benches/checkpoint_skiplist.rs b/crates/core/benches/checkpoint_skiplist.rs new file mode 100644 index 000000000..aba5113be --- /dev/null +++ b/crates/core/benches/checkpoint_skiplist.rs @@ -0,0 +1,235 @@ +use bdk_core::CheckPoint; +use bitcoin::hashes::Hash; +use bitcoin::BlockHash; +use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; + +/// Create a checkpoint chain with the given length +fn create_checkpoint_chain(length: u32) -> CheckPoint { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + for height in 1..=length { + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + cp +} + +/// Benchmark get() operations at various depths +fn bench_checkpoint_get(c: &mut Criterion) { + // Small chain - get near start + c.bench_function("get_100_near_start", |b: &mut Bencher| { + let cp = create_checkpoint_chain(100); + let target = 10; + b.iter(|| { + black_box(cp.get(target)); + }); + }); + + // Medium chain - get middle + c.bench_function("get_1000_middle", |b: &mut Bencher| { + let cp = create_checkpoint_chain(1000); + let target = 500; + b.iter(|| { + black_box(cp.get(target)); + }); + }); + + // Large chain - get near end + c.bench_function("get_10000_near_end", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 9000; + b.iter(|| { + black_box(cp.get(target)); + }); + }); + + // Large chain - get near start (best case for skiplist) + c.bench_function("get_10000_near_start", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; + b.iter(|| { + black_box(cp.get(target)); + }); + }); +} + +/// Benchmark floor_at() operations +fn bench_checkpoint_floor_at(c: &mut Criterion) { + c.bench_function("floor_at_1000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(1000); + let target = 750; // Target that might not exist exactly + b.iter(|| { + black_box(cp.floor_at(target)); + }); + }); + + c.bench_function("floor_at_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 7500; + b.iter(|| { + black_box(cp.floor_at(target)); + }); + }); +} + +/// Benchmark range() iteration +fn bench_checkpoint_range(c: &mut Criterion) { + // Small range in middle (tests skip pointer efficiency) + c.bench_function("range_1000_middle_10pct", |b: &mut Bencher| { + let cp = create_checkpoint_chain(1000); + b.iter(|| { + let range: Vec<_> = cp.range(450..=550).collect(); + black_box(range); + }); + }); + + // Large range (tests iteration performance) + c.bench_function("range_10000_large_50pct", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + b.iter(|| { + let range: Vec<_> = cp.range(2500..=7500).collect(); + black_box(range); + }); + }); + + // Range from start (tests early termination) + c.bench_function("range_10000_from_start", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + b.iter(|| { + let range: Vec<_> = cp.range(..=100).collect(); + black_box(range); + }); + }); + + // Range near tip (minimal skip pointer usage) + c.bench_function("range_10000_near_tip", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + b.iter(|| { + let range: Vec<_> = cp.range(9900..).collect(); + black_box(range); + }); + }); + + // Single element range (edge case) + c.bench_function("range_single_element", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + b.iter(|| { + let range: Vec<_> = cp.range(5000..=5000).collect(); + black_box(range); + }); + }); +} + +/// Benchmark insert() operations +fn bench_checkpoint_insert(c: &mut Criterion) { + c.bench_function("insert_sparse_1000", |b: &mut Bencher| { + // Create a sparse chain + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + for i in 1..=100 { + let height = i * 10; + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + let insert_height = 505; + let insert_hash = BlockHash::from_byte_array([255; 32]); + + b.iter(|| { + let result = cp.clone().insert(insert_height, insert_hash); + black_box(result); + }); + }); +} + +/// Compare linear traversal vs skiplist-enhanced get() +fn bench_traversal_comparison(c: &mut Criterion) { + // Linear traversal benchmark + c.bench_function("linear_traversal_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; // Near the beginning + + b.iter(|| { + let mut current = cp.clone(); + while current.height() > target { + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + black_box(current); + }); + }); + + // Skiplist-enhanced get() for comparison + c.bench_function("skiplist_get_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; // Same target + + b.iter(|| { + black_box(cp.get(target)); + }); + }); +} + +/// Analyze skip pointer distribution and usage +fn bench_skip_pointer_analysis(c: &mut Criterion) { + c.bench_function("count_skip_pointers_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + + b.iter(|| { + let mut count = 0; + let mut current = cp.clone(); + loop { + if current.skip().is_some() { + count += 1; + } + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + black_box(count); + }); + }); + + // Measure actual skip pointer usage during traversal + c.bench_function("skip_usage_in_traversal", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; + + b.iter(|| { + let mut current = cp.clone(); + let mut skips_used = 0; + + while current.height() > target { + if let Some(skip_cp) = current.skip() { + if skip_cp.height() >= target { + current = skip_cp; + skips_used += 1; + continue; + } + } + + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + black_box((current, skips_used)); + }); + }); +} + +criterion_group!( + benches, + bench_checkpoint_get, + bench_checkpoint_floor_at, + bench_checkpoint_range, + bench_checkpoint_insert, + bench_traversal_comparison, + bench_skip_pointer_analysis +); + +criterion_main!(benches); diff --git a/crates/core/src/checkpoint.rs b/crates/core/src/checkpoint.rs index 5f0ef3e20..bf54731ad 100644 --- a/crates/core/src/checkpoint.rs +++ b/crates/core/src/checkpoint.rs @@ -6,6 +6,9 @@ use bitcoin::{block::Header, BlockHash}; use crate::BlockId; +/// Interval for skiplist pointers based on checkpoint index. +const CHECKPOINT_SKIP_INTERVAL: u32 = 100; + /// A checkpoint is a node of a reference-counted linked list of [`BlockId`]s. /// /// Checkpoints are cheaply cloneable and are useful to find the agreement point between two sparse @@ -28,6 +31,10 @@ struct CPInner { data: D, /// Previous checkpoint (if any). prev: Option>>, + /// Skip pointer for fast traversals. + skip: Option>>, + /// Index of this checkpoint (number of checkpoints from the first). + index: u32, } /// When a `CPInner` is dropped we need to go back down the chain and manually remove any @@ -121,6 +128,16 @@ impl CheckPoint { self.0.prev.clone().map(CheckPoint) } + /// Get the index of this checkpoint (number of checkpoints from the first). + pub fn index(&self) -> u32 { + self.0.index + } + + /// Get the skip pointer checkpoint if it exists. + pub fn skip(&self) -> Option> { + self.0.skip.clone().map(CheckPoint) + } + /// Iterate from this checkpoint in descending height. pub fn iter(&self) -> CheckPointIter { self.clone().into_iter() @@ -130,7 +147,37 @@ impl CheckPoint { /// /// Returns `None` if checkpoint at `height` does not exist`. pub fn get(&self, height: u32) -> Option { - self.range(height..=height).next() + let mut current = self.clone(); + + if current.height() == height { + return Some(current); + } + + // Use skip pointers to jump close to target + while current.height() > height { + match current.skip() { + Some(skip_cp) => match skip_cp.height().cmp(&height) { + core::cmp::Ordering::Greater => current = skip_cp, + core::cmp::Ordering::Equal => return Some(skip_cp), + core::cmp::Ordering::Less => break, // Skip would undershoot + }, + None => break, // No more skip pointers + } + } + + // Linear search for exact height + while current.height() > height { + match current.prev() { + Some(prev_cp) => match prev_cp.height().cmp(&height) { + core::cmp::Ordering::Greater => current = prev_cp, + core::cmp::Ordering::Equal => return Some(prev_cp), + core::cmp::Ordering::Less => break, // Height doesn't exist + }, + None => break, // End of chain + } + } + + None } /// Iterate checkpoints over a height range. @@ -143,17 +190,39 @@ impl CheckPoint { { let start_bound = range.start_bound().cloned(); let end_bound = range.end_bound().cloned(); - self.iter() - .skip_while(move |cp| match end_bound { - core::ops::Bound::Included(inc_bound) => cp.height() > inc_bound, - core::ops::Bound::Excluded(exc_bound) => cp.height() >= exc_bound, - core::ops::Bound::Unbounded => false, - }) - .take_while(move |cp| match start_bound { - core::ops::Bound::Included(inc_bound) => cp.height() >= inc_bound, - core::ops::Bound::Excluded(exc_bound) => cp.height() > exc_bound, - core::ops::Bound::Unbounded => true, - }) + + let is_above_bound = |height: u32| match end_bound { + core::ops::Bound::Included(inc_bound) => height > inc_bound, + core::ops::Bound::Excluded(exc_bound) => height >= exc_bound, + core::ops::Bound::Unbounded => false, + }; + + let mut current = self.clone(); + + // Use skip pointers to jump close to target + while is_above_bound(current.height()) { + match current.skip() { + Some(skip_cp) if is_above_bound(skip_cp.height()) => { + current = skip_cp; + } + _ => break, // Skip would undershoot or doesn't exist + } + } + + // Linear search to exact position + while is_above_bound(current.height()) { + match current.prev() { + Some(prev) => current = prev, + None => break, + } + } + + // Iterate from start point + current.into_iter().take_while(move |cp| match start_bound { + core::ops::Bound::Included(inc_bound) => cp.height() >= inc_bound, + core::ops::Bound::Excluded(exc_bound) => cp.height() > exc_bound, + core::ops::Bound::Unbounded => true, + }) } /// Returns the checkpoint at `height` if one exists, otherwise the nearest checkpoint at a @@ -201,6 +270,8 @@ where }, data, prev: None, + skip: None, + index: 0, })) } @@ -265,6 +336,7 @@ where cp = cp.prev().expect("will break before genesis block"); }; + // Rebuild the chain: base -> new block -> tail base.extend(core::iter::once((height, data)).chain(tail.into_iter().rev())) .expect("tail is in order") } @@ -274,18 +346,42 @@ where /// Returns an `Err(self)` if the block you are pushing on is not at a greater height that the /// one you are pushing on to. pub fn push(self, height: u32, data: D) -> Result { - if self.height() < height { - Ok(Self(Arc::new(CPInner { - block_id: BlockId { - height, - hash: data.to_blockhash(), - }, - data, - prev: Some(self.0), - }))) - } else { - Err(self) + if self.height() >= height { + return Err(self); } + + let new_index = self.0.index + 1; + + // Skip pointers are added every CHECKPOINT_SKIP_INTERVAL (100) checkpoints + // e.g., checkpoints at index 100, 200, 300, etc. have skip pointers + let needs_skip_pointer = + new_index >= CHECKPOINT_SKIP_INTERVAL && new_index % CHECKPOINT_SKIP_INTERVAL == 0; + + let skip = if needs_skip_pointer { + // Skip pointer points back CHECKPOINT_SKIP_INTERVAL positions + // e.g., checkpoint at index 200 points to checkpoint at index 100 + // We walk back CHECKPOINT_SKIP_INTERVAL - 1 steps since we start from self (index + // new_index - 1) + let mut current = self.0.clone(); + for _ in 0..(CHECKPOINT_SKIP_INTERVAL - 1) { + // This is safe: if we're at index >= 100, we must have at least 99 predecessors + current = current.prev.clone().expect("chain has enough checkpoints"); + } + Some(current) + } else { + None + }; + + Ok(Self(Arc::new(CPInner { + block_id: BlockId { + height, + hash: data.to_blockhash(), + }, + data, + prev: Some(self.0), + skip, + index: new_index, + }))) } } @@ -353,12 +449,15 @@ mod tests { let genesis = cp.get(0).expect("genesis exists"); let weak = Arc::downgrade(&genesis.0); - // At this point there should be exactly two strong references to the - // genesis checkpoint: the variable `genesis` and the chain `cp`. + // At this point there should be exactly three strong references to the + // genesis checkpoint: + // 1. The variable `genesis` + // 2. The chain `cp` through checkpoint 1's prev pointer + // 3. Checkpoint at index 100's skip pointer (points to index 0) assert_eq!( Arc::strong_count(&genesis.0), - 2, - "`cp` and `genesis` should be the only strong references", + 3, + "`cp`, `genesis`, and checkpoint 100's skip pointer should be the only strong references", ); // Dropping the chain should remove one strong reference. diff --git a/crates/core/tests/test_checkpoint_skiplist.rs b/crates/core/tests/test_checkpoint_skiplist.rs new file mode 100644 index 000000000..28da9aa15 --- /dev/null +++ b/crates/core/tests/test_checkpoint_skiplist.rs @@ -0,0 +1,245 @@ +use bdk_core::CheckPoint; +use bitcoin::hashes::Hash; +use bitcoin::BlockHash; + +#[test] +fn test_skiplist_indices() { + // Create a long chain to test skiplist + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + assert_eq!(cp.index(), 0); + + for height in 1..=500 { + let hash = BlockHash::from_byte_array([height as u8; 32]); + cp = cp.push(height, hash).unwrap(); + assert_eq!(cp.index(), height); + } + + // Test that skip pointers are set correctly + // At index 100, 200, 300, 400, 500 we should have skip pointers + assert_eq!(cp.index(), 500); + + // Navigate to index 400 and check skip pointer + let mut current = cp.clone(); + for _ in 0..100 { + current = current.prev().unwrap(); + } + assert_eq!(current.index(), 400); + + // Check that skip pointer exists at index 400 + if let Some(skip) = current.skip() { + assert_eq!(skip.index(), 300); + } else { + panic!("Expected skip pointer at index 400"); + } + + // Navigate to index 300 and check skip pointer + for _ in 0..100 { + current = current.prev().unwrap(); + } + assert_eq!(current.index(), 300); + + if let Some(skip) = current.skip() { + assert_eq!(skip.index(), 200); + } else { + panic!("Expected skip pointer at index 300"); + } + + // Navigate to index 100 and check skip pointer + for _ in 0..200 { + current = current.prev().unwrap(); + } + assert_eq!(current.index(), 100); + + if let Some(skip) = current.skip() { + assert_eq!(skip.index(), 0); + } else { + panic!("Expected skip pointer at index 100"); + } +} + +#[test] +fn test_skiplist_get_performance() { + // Create a very long chain + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + for height in 1..=1000 { + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Test that get() can find checkpoints efficiently + // This should use skip pointers to navigate quickly + + // Verify the chain was built correctly + assert_eq!(cp.height(), 1000); + assert_eq!(cp.index(), 1000); + + // Find checkpoint near the beginning + if let Some(found) = cp.get(50) { + assert_eq!(found.height(), 50); + assert_eq!(found.index(), 50); + } else { + // Debug: print the first few checkpoints + let mut current = cp.clone(); + println!("First 10 checkpoints:"); + for _ in 0..10 { + println!("Height: {}, Index: {}", current.height(), current.index()); + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + panic!("Could not find checkpoint at height 50"); + } + + // Find checkpoint in the middle + if let Some(found) = cp.get(500) { + assert_eq!(found.height(), 500); + assert_eq!(found.index(), 500); + } else { + panic!("Could not find checkpoint at height 500"); + } + + // Find checkpoint near the end + if let Some(found) = cp.get(950) { + assert_eq!(found.height(), 950); + assert_eq!(found.index(), 950); + } else { + panic!("Could not find checkpoint at height 950"); + } + + // Test non-existent checkpoint + assert!(cp.get(1001).is_none()); +} + +#[test] +fn test_skiplist_floor_at() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Create sparse chain with gaps + for height in [10, 50, 100, 150, 200, 300, 400, 500] { + let hash = BlockHash::from_byte_array([height as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Test floor_at with skip pointers + let floor = cp.floor_at(250).unwrap(); + assert_eq!(floor.height(), 200); + + let floor = cp.floor_at(99).unwrap(); + assert_eq!(floor.height(), 50); + + let floor = cp.floor_at(500).unwrap(); + assert_eq!(floor.height(), 500); + + let floor = cp.floor_at(600).unwrap(); + assert_eq!(floor.height(), 500); +} + +#[test] +fn test_skiplist_insert_maintains_indices() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Build initial chain + for height in [10, 20, 30, 40, 50] { + let hash = BlockHash::from_byte_array([height as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Insert a block in the middle + let hash = BlockHash::from_byte_array([25; 32]); + cp = cp.insert(25, hash); + + // Check that indices are maintained correctly + let check = cp.get(50).unwrap(); + assert_eq!(check.index(), 6); // 0, 10, 20, 25, 30, 40, 50 + + let check = cp.get(25).unwrap(); + assert_eq!(check.index(), 3); + + // Check the full chain has correct indices + let mut current = cp.clone(); + let expected_heights = [50, 40, 30, 25, 20, 10, 0]; + let expected_indices = [6, 5, 4, 3, 2, 1, 0]; + + for (expected_height, expected_index) in expected_heights.iter().zip(expected_indices.iter()) { + assert_eq!(current.height(), *expected_height); + assert_eq!(current.index(), *expected_index); + if *expected_height > 0 { + current = current.prev().unwrap(); + } + } +} + +#[test] +fn test_skiplist_range_uses_skip_pointers() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Create a chain with 500 checkpoints + for height in 1..=500 { + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Test range iteration + let range_items: Vec<_> = cp.range(100..=200).collect(); + assert_eq!(range_items.len(), 101); + assert_eq!(range_items.first().unwrap().height(), 200); + assert_eq!(range_items.last().unwrap().height(), 100); + + // Test open range + let range_items: Vec<_> = cp.range(450..).collect(); + assert_eq!(range_items.len(), 51); + assert_eq!(range_items.first().unwrap().height(), 500); + assert_eq!(range_items.last().unwrap().height(), 450); +} + +#[test] +fn test_range_edge_cases() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Create sparse chain: 0, 100, 200, 300, 400, 500 + for i in 1..=5 { + let height = i * 100; + let hash = BlockHash::from_byte_array([i as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Empty range (start > end) + #[allow(clippy::reversed_empty_ranges)] + let empty: Vec<_> = cp.range(300..200).collect(); + assert!(empty.is_empty()); + + // Single element range + let single: Vec<_> = cp.range(300..=300).collect(); + assert_eq!(single.len(), 1); + assert_eq!(single[0].height(), 300); + + // Range with non-existent bounds (150..250) + let partial: Vec<_> = cp.range(150..250).collect(); + assert_eq!(partial.len(), 1); + assert_eq!(partial[0].height(), 200); + + // Exclusive end bound (100..300 includes 100 and 200, but not 300) + let exclusive: Vec<_> = cp.range(100..300).collect(); + assert_eq!(exclusive.len(), 2); + assert_eq!(exclusive[0].height(), 200); + assert_eq!(exclusive[1].height(), 100); + + // Unbounded range (..) + let all: Vec<_> = cp.range(..).collect(); + assert_eq!(all.len(), 6); + assert_eq!(all.first().unwrap().height(), 500); + assert_eq!(all.last().unwrap().height(), 0); + + // Range beyond chain bounds + let beyond: Vec<_> = cp.range(600..700).collect(); + assert!(beyond.is_empty()); + + // Range from genesis + let from_genesis: Vec<_> = cp.range(0..=200).collect(); + assert_eq!(from_genesis.len(), 3); + assert_eq!(from_genesis[0].height(), 200); + assert_eq!(from_genesis[2].height(), 0); +}