The real v2 - new database and new data model + gRPC

Dockerfile

@@ -6,7 +6,7 @@ WORKDIR /app
 
 
 RUN go mod download
-RUN env CGO_ENABLED=0 go build -o main ./src
+RUN env CGO_ENABLED=0 go build -o main ./cmd/blindbit-oracle
 
 FROM busybox
 COPY --from=buildstage /app/main .

Makefile

@@ -0,0 +1,32 @@
+# Benchmark targets
+.PHONY: benchmark
+benchmark: build-benchmark
+	@echo "Running benchmark..."
+	./bin/benchmark $(ARGS)
+
+.PHONY: benchmark-v1
+benchmark-v1: build-benchmark
+	@echo "Running v1 HTTP benchmark only..."
+	./bin/benchmark -v1 -v2=false $(ARGS)
+
+.PHONY: benchmark-v2
+benchmark-v2: build-benchmark
+	@echo "Running v2 gRPC benchmark only..."
+	./bin/benchmark -v1=false -v2 $(ARGS)
+
+.PHONY: compare
+compare: build-benchmark
+	@echo "Comparing v1 and v2 data..."
+	./bin/benchmark -compare $(ARGS)
+
+.PHONY: build-benchmark
+build-benchmark:
+	@echo "Building benchmark tool..."
+	@mkdir -p bin
+	go build -o bin/benchmark ./cmd/benchmark
+
+# Example usage:
+# make benchmark ARGS="-startheight=100 -endheight=200"
+# make benchmark-v1 ARGS="-startheight=100 -endheight=200"
+# make benchmark-v2 ARGS="-startheight=100 -endheight=200"
+# make compare ARGS="-startheight=100 -endheight=200"

blindbit.example.toml

@@ -1,13 +1,26 @@
+# possible values: trace, debug, info, warn, error
+log_level = "debug"
+
 # 0.0.0.0:8000 to expose outside of localhost
 # default: "127.0.0.1:8000"
-host = "127.0.0.1:8000"
+http_host = "127.0.0.1:8000"
+
+# defines where the gRPC host is exposed
+# default: "" gRPC is not exposed by default
+grpc_host = "127.0.0.1:8001"
 
 # Defines on which chain the wallet runs. Allowed values: main, testnet, signet, regtest.
 # default: signet
 chain = "signet"
 
+# bitcoin core rpc endpoint
 # default: http://127.0.0.1:8332
-rpc_endpoint = "http://127.0.0.1:18443"
+core_rpc_endpoint = "http://127.0.0.1:18443"
+
+# bitcoin core rest endpoint
+# note: requires bitcoin core PR #32540 (already merged to master but in a release yet)
+# default: ""
+core_rest_endpoint = "http://127.0.01:38332"
 
 # required, unless rpc_user and rpc_pass are set
 cookie_path = ""

cmd/benchmark/README.md

@@ -0,0 +1,71 @@
+# Oracle Benchmark Tool
+
+This tool benchmarks the performance difference between v1 (HTTP) and v2 (gRPC streaming) APIs for fetching block data.
+
+## Building
+
+```bash
+make build-benchmark
+```
+
+## Usage
+
+### Run both benchmarks
+```bash
+./bin/benchmark -startheight=100 -endheight=200
+```
+
+### Run only v1 HTTP benchmark
+```bash
+./bin/benchmark -v1 -v2=false -startheight=100 -endheight=200
+```
+
+### Run only v2 gRPC benchmark
+```bash
+./bin/benchmark -v1=false -v2 -startheight=100 -endheight=200
+```
+
+### Compare v1 and v2 data (validation)
+```bash
+./bin/benchmark -compare -startheight=100 -endheight=200
+```
+
+### Using Makefile
+```bash
+# Run both
+make benchmark ARGS="-startheight=100 -endheight=200"
+
+# Run only v1
+make benchmark-v1 ARGS="-startheight=100 -endheight=200"
+
+# Run only v2
+make benchmark-v2 ARGS="-startheight=100 -endheight=200"
+
+# Compare data
+make benchmark ARGS="-compare -startheight=264100 -endheight=264200"
+```
+
+## Command Line Flags
+
+- `-startheight`: Start block height (default: 1)
+- `-endheight`: End block height (default: 10)
+- `-http`: HTTP API base URL (default: "http://127.0.0.1:8000")
+- `-grpc`: gRPC server host:port (default: "127.0.0.1:50051")
+- `-v1`: Run v1 HTTP benchmark (default: true)
+- `-v2`: Run v2 gRPC benchmark (default: true)
+- `-compare`: Compare v1 and v2 data instead of benchmarking (default: false)
+
+## What it measures
+
+The benchmark fetches block data (tweaks, filters) for each block height in the range and measures:
+
+- Total time to fetch all blocks
+- Blocks processed per second
+- Individual block fetch times
+
+## Expected Results
+
+- **v1 (HTTP)**: Makes individual HTTP requests for each block, good for small ranges
+- **v2 (gRPC)**: Uses streaming to fetch all blocks in one connection, better for large ranges
+
+The gRPC streaming approach should show better performance for larger block ranges due to reduced connection overhead and better batching.

cmd/benchmark/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/rs/zerolog"
+	"github.com/setavenger/blindbit-lib/logging"
+	"github.com/setavenger/blindbit-oracle/internal/benchmark"
+)
+
+func main() {
+	var (
+		startHeight = flag.Uint64("startheight", 1, "Start block height")
+		endHeight   = flag.Uint64("endheight", 10, "End block height")
+		httpURL     = flag.String("http", "http://127.0.0.1:8000", "HTTP API base URL")
+		grpcHost    = flag.String("grpc", "127.0.0.1:50051", "gRPC server host:port")
+		runV1       = flag.Bool("v1", true, "Run v1 HTTP benchmark")
+		runV2       = flag.Bool("v2", true, "Run v2 gRPC benchmark")
+		compare     = flag.Bool("compare", false, "Compare v1 and v2 data instead of benchmarking")
+	)
+	flag.Parse()
+
+	// Setup logging
+	logging.SetLogLevel(zerolog.InfoLevel)
+
+	if *compare {
+		logging.L.Info().
+			Uint64("start_height", *startHeight).
+			Uint64("end_height", *endHeight).
+			Msg("Starting data comparison")
+
+		benchmark.CompareV1V2Results(*startHeight, *endHeight, *httpURL, *grpcHost)
+		return
+	}
+
+	logging.L.Info().
+		Uint64("start_height", *startHeight).
+		Uint64("end_height", *endHeight).
+		Msg("Starting benchmark")
+
+	// heat up cache or whatever to keep it somewhat fair
+	benchmark.BenchmarkV2(*startHeight, *endHeight, *grpcHost)
+
+	if *runV1 {
+		logging.L.Info().Msg("=== Running V1 HTTP Benchmark ===")
+		benchmark.BenchmarkV1(*startHeight, *endHeight, *httpURL)
+	}
+
+	if *runV2 {
+		logging.L.Info().Msg("=== Running V2 gRPC Streaming Benchmark ===")
+		benchmark.BenchmarkV2(*startHeight, *endHeight, *grpcHost)
+	}
+
+	logging.L.Info().Msg("Benchmark completed")
+}