Add badgerdb-sampler tool for sampling BadgerDB databases used by Oasis nodes

badgerdb-sampler/Makefile

@@ -0,0 +1,43 @@
+# Makefile for badgerdb-sampler
+
+.PHONY: all build-all build-v2 build-v3 build-v4 clean tidy-all
+
+# Default target
+all: build-all
+
+# Build all versions
+build-all: build-v2 build-v3 build-v4
+
+# Build BadgerDB v2 version
+build-v2:
+	@echo "Building badgerdb-sampler-v2..."
+	go build -modfile=go_v2.mod -tags badgerv2 -o bin/badgerdb-sampler-v2
+	@echo "✓ Built: bin/badgerdb-sampler-v2"
+
+# Build BadgerDB v3 version
+build-v3:
+	@echo "Building badgerdb-sampler-v3..."
+	go build -modfile=go_v3.mod -tags badgerv3 -o bin/badgerdb-sampler-v3
+	@echo "✓ Built: bin/badgerdb-sampler-v3"
+
+# Build BadgerDB v4 version
+build-v4:
+	@echo "Building badgerdb-sampler-v4..."
+	go build -modfile=go_v4.mod -tags badgerv4 -o bin/badgerdb-sampler-v4
+	@echo "✓ Built: bin/badgerdb-sampler-v4"
+
+# Clean build artifacts
+clean:
+	@echo "Cleaning bin/..."
+	rm -f bin/badgerdb-sampler-v2 bin/badgerdb-sampler-v3 bin/badgerdb-sampler-v4
+	@echo "Cleaning outputs/..."
+	rm -rf outputs/*
+	@echo "✓ Cleaned"
+
+# Tidy all module files
+tidy-all:
+	@echo "Tidying all module files..."
+	GOFLAGS="-tags=badgerv2" go mod tidy -modfile=go_v2.mod
+	GOFLAGS="-tags=badgerv3" go mod tidy -modfile=go_v3.mod
+	GOFLAGS="-tags=badgerv4" go mod tidy -modfile=go_v4.mod
+	@echo "✓ All module files tidied"

badgerdb-sampler/README.md

@@ -0,0 +1,123 @@
+# BadgerDB Sampler
+
+This **badgerdb-sampler** tool implements a comprehensive data extraction and decoding logic for BadgerDB databases used in Oasis node snapshots. This tool can extract and decode most data structures from consensus and runtime databases, even EVM data. It handles multiple BadgerDB versions (v2-v4) and data representations use by Oasis nodes (v20.x-v25.x). The decoding logic handles various data formats and reports decoding issues.
+
+**Warning:** For experimental purposes only. Decoded data might be incorrect.
+
+## Features
+
+- **Multi-version BadgerDB Support**: Compatible with BadgerDB v2, v3, and v4
+- **Comprehensive Decoding**: Handles consensus state, runtime state, and EVM-specific data
+- **Error Tracking**: Collects and reports decoding errors with detailed error counts
+- **Statistics Generation**: Provides key type distributions, database size, and sample counts
+- **Module-aware Parsing**: Recognizes and routes data to appropriate decoders (evm, accounts, contracts, core)
+- **EVM Event Decoding**: Includes event signature database for human-readable EVM event names
+- **FUSE Filesystem Support**: Works with databases on FUSE mounts using intelligent fallback strategies
+- **Read-only Access**: Can analyze databases currently in use by nodes via `BypassLockGuard`
+
+## Supported Database Types
+
+- `consensus-blockstore` - Block metadata and commit info
+- `consensus-evidence` - Byzantine validator evidence
+- `consensus-mkvs` - Consensus state Merkle tree
+- `consensus-state` - Tendermint/CometBFT consensus state
+- `runtime-history` - Runtime block history with CBOR-encoded data (includes EVM events/transactions)
+- `runtime-mkvs` - Runtime state Merkle tree (includes EVM storage)
+
+## Building
+
+```bash
+# Build all versions (recommended)
+make build-all
+
+# Clean build artifacts
+make clean
+```
+
+## Usage
+
+### Prerequisites
+
+- Go 1.21 or higher
+- BadgerDB databases from Oasis nodes (see https://snapshots.oasis.io/)
+
+### Command Syntax
+
+```bash
+./bin/badgerdb-sampler-v{2,3,4} <database-type> <path-to-db> [output-json] [max-samples]
+```
+
+**Parameters:**
+- `database-type`: One of the supported database types (see above)
+- `path-to-db`: Path to the BadgerDB database directory
+- `output-json`: Optional path to save JSON results (default: stdout only)
+- `max-samples`: Optional maximum number of samples to collect (default: 1000)
+
+### Examples
+
+```bash
+# Analyze v2 consensus blockstore (default 1000 samples, stdout only)
+./bin/badgerdb-sampler-v2 consensus-blockstore /path/to/blockstore.badger.db
+
+# Analyze v3 runtime MKVS with JSON output
+./bin/badgerdb-sampler-v3 runtime-mkvs /path/to/mkvs_storage.badger.db ./outputs/testnet-20220303/emerald-runtime-mkvs.json
+
+# Analyze runtime history with custom sample limit
+./bin/badgerdb-sampler-v3 runtime-history /path/to/history.badger.db ./outputs/runtime-history.json 500
+
+# Analyze currently running node's database (read-only)
+./bin/badgerdb-sampler-v3 consensus-state /var/lib/oasis/node/consensus/state.badger.db ./outputs/consensus-state.json
+```
+
+## Output Format
+
+The tool outputs JSON with comprehensive statistics and samples:
+
+```json
+{
+  "database_path": "/path/to/mkvs_storage.badger.db",
+  "database_type": "runtime-mkvs",
+  "badgerdb_version": "v3",
+  "database_size_bytes": 1234567890,
+  "sample_count": 1000,
+  "key_type_counts": {
+    "mkvs:node": 850,
+    "mkvs:root": 150
+  },
+  "error_counts": {
+    "failed to decode CBOR value: unexpected EOF": 5,
+    "unknown module prefix": 2
+  },
+  "samples": [
+    {
+      "key_type": "mkvs:node",
+      "key": { /* decoded key structure */ },
+      "value": { /* decoded value structure */ },
+    }
+  ]
+}
+```
+
+**Key Fields:**
+- `key_type_counts`: Distribution of different key types in the database
+- `error_counts`: Aggregated decoding errors across all samples
+- `samples`: Array of individual key-value pairs with full decoding details
+
+## Architecture
+
+The tool uses a three-layer design:
+
+1. **Database Access** (`db_v*.go`, `db_common.go`)
+   - Version-specific BadgerDB initialization via build tags
+   - FUSE workaround with temp directory and symlinks
+   - Read-only mode with lock bypass for in-use databases
+
+2. **Decoding Logic** (`decode_*.go`)
+   - `decode_consensus.go`: Tendermint protobuf parsing
+   - `decode_runtime.go`: CBOR/MKVS parsing with module routing
+   - `decode_evm.go`: EVM-specific parsing (contracts, events, transactions)
+
+3. **Type System** (`types_*.go`)
+   - Separated deserialization and output types
+   - EVM-specific output structures
+   - Event signature database for human-readable names

badgerdb-sampler/db_common.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// createLocalMirror creates a temporary local directory with database files copied/symlinked
+// to avoid mmap issues on FUSE filesystems. Returns temp path and cleanup function.
+func createLocalMirror(fusePath string) (string, func(), error) {
+	// Create temp directory on local filesystem (not FUSE)
+	tmpDir, err := os.MkdirTemp("", "badgerdb-temp-*")
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create temp directory: %w", err)
+	}
+
+	cleanup := func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: Failed to clean up temp directory %s: %v\n", tmpDir, err)
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "  Creating local mirror in: %s\n", tmpDir)
+
+	entries, err := os.ReadDir(fusePath)
+	if err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("failed to read database directory: %w", err)
+	}
+
+	copiedCount := 0
+	linkedCount := 0
+	skippedCount := 0
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		name := entry.Name()
+		srcPath := filepath.Join(fusePath, name)
+		dstPath := filepath.Join(tmpDir, name)
+
+		// Skip memtable files - these will be created fresh
+		if strings.HasSuffix(name, ".mem") {
+			fmt.Fprintf(os.Stderr, "  Skipping memtable file: %s\n", name)
+			skippedCount++
+			continue
+		}
+
+		// Copy small metadata files (MANIFEST, DISCARD, etc.)
+		// Symlink large data files (*.sst, *.vlog)
+		if strings.HasSuffix(name, ".sst") || strings.HasSuffix(name, ".vlog") {
+			// Symlink large data files
+			if err := os.Symlink(srcPath, dstPath); err != nil {
+				cleanup()
+				return "", nil, fmt.Errorf("failed to symlink %s: %w", name, err)
+			}
+			linkedCount++
+		} else {
+			// Copy small metadata files
+			if err := copyFile(srcPath, dstPath); err != nil {
+				cleanup()
+				return "", nil, fmt.Errorf("failed to copy %s: %w", name, err)
+			}
+			copiedCount++
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "  Mirror created: %d files copied, %d files symlinked, %d files skipped\n",
+		copiedCount, linkedCount, skippedCount)
+
+	return tmpDir, cleanup, nil
+}
+
+// copyFile copies a file from src to dst
+func copyFile(src, dst string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer dstFile.Close()
+
+	if _, err := io.Copy(dstFile, srcFile); err != nil {
+		return err
+	}
+
+	return dstFile.Sync()
+}
+
+// removeMemtableFiles renames .mem files by appending .bak suffix and returns a restore function.
+// This allows opening databases with corrupted memtables by letting BadgerDB create fresh memtable files.
+func removeMemtableFiles(dbPath string) (backupDir string, restore func() error, err error) {
+	entries, err := os.ReadDir(dbPath)
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to read database directory: %w", err)
+	}
+
+	// Rename all .mem files to .mem.bak
+	count := 0
+	for _, entry := range entries {
+		if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mem") {
+			oldPath := filepath.Join(dbPath, entry.Name())
+			newPath := oldPath + ".bak"
+			if err := os.Rename(oldPath, newPath); err != nil {
+				return "", nil, fmt.Errorf("failed to rename %s: %w", entry.Name(), err)
+			}
+			count++
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "  Renamed %d memtable file(s) to *.mem.bak\n", count)
+
+	// Restore function renames .mem.bak files back to .mem
+	restore = func() error {
+		entries, err := os.ReadDir(dbPath)
+		if err != nil {
+			return fmt.Errorf("failed to read database directory: %w", err)
+		}
+		count := 0
+		for _, entry := range entries {
+			if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mem.bak") {
+				oldPath := filepath.Join(dbPath, entry.Name())
+				newPath := strings.TrimSuffix(oldPath, ".bak")
+				if err := os.Rename(oldPath, newPath); err != nil {
+					return fmt.Errorf("failed to restore %s: %w", entry.Name(), err)
+				}
+				count++
+			}
+		}
+		fmt.Fprintf(os.Stderr, "  Restored %d memtable file(s) from *.mem.bak\n", count)
+		return nil
+	}
+
+	return dbPath, restore, nil
+}