From ff1aec3c3f72de4439af0f39d320e1fcd26274e2 Mon Sep 17 00:00:00 2001 From: glyh Date: Thu, 16 Oct 2025 15:11:27 +0800 Subject: [PATCH 01/23] implement RocksDB compatibility test in Python --- .../scripts/rocksdb-compatibility/.gitignore | 2 + .../rocksdb-compatibility/install-rocksdb.sh | 19 +++++ .../rocksdb-compatibility/requirements.txt | 4 + .../scripts/rocksdb-compatibility/rocksdb.py | 71 ++++++++++++++++ .../scripts/rocksdb-compatibility/test.py | 84 +++++++++++++++++++ 5 files changed, 180 insertions(+) create mode 100644 buildkite/scripts/rocksdb-compatibility/.gitignore create mode 100755 buildkite/scripts/rocksdb-compatibility/install-rocksdb.sh create mode 100644 buildkite/scripts/rocksdb-compatibility/requirements.txt create mode 100644 buildkite/scripts/rocksdb-compatibility/rocksdb.py create mode 100644 buildkite/scripts/rocksdb-compatibility/test.py diff --git a/buildkite/scripts/rocksdb-compatibility/.gitignore b/buildkite/scripts/rocksdb-compatibility/.gitignore new file mode 100644 index 00000000000..13d5ab46f9f --- /dev/null +++ b/buildkite/scripts/rocksdb-compatibility/.gitignore @@ -0,0 +1,2 @@ +/.venv +/__pycache__ diff --git a/buildkite/scripts/rocksdb-compatibility/install-rocksdb.sh b/buildkite/scripts/rocksdb-compatibility/install-rocksdb.sh new file mode 100755 index 00000000000..289317ebfc9 --- /dev/null +++ b/buildkite/scripts/rocksdb-compatibility/install-rocksdb.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -euox pipefail + +ROCKSDB_VERSION=10.5.1 + +ROCKSDB_SOURCE=`mktemp -d --tmpdir rocksdb-$ROCKSDB_VERSION.XXXXXX` +trap "rm -rf $ROCKSDB_SOURCE" EXIT + +curl -L https://github.com/facebook/rocksdb/archive/refs/tags/v${ROCKSDB_VERSION}.tar.gz | tar xz -C $ROCKSDB_SOURCE + +cd $ROCKSDB_SOURCE/rocksdb-${ROCKSDB_VERSION} + +# NOTE: +# `-Wno-unused-parameter` is to fix this error: +# util/compression.cc:684:40: error: unused parameter ‘args’ [-Werror=unused-parameter] +# 684 | Status ExtractUncompressedSize(Args& args) override { +# | ~~~~~~^~~~ +EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j$(nproc) install diff --git a/buildkite/scripts/rocksdb-compatibility/requirements.txt b/buildkite/scripts/rocksdb-compatibility/requirements.txt new file mode 100644 index 00000000000..2f2609dd589 --- /dev/null +++ b/buildkite/scripts/rocksdb-compatibility/requirements.txt @@ -0,0 +1,4 @@ +cffi==2.0.0 +requests==2.31 +tqdm==4.65 +pycurl==7.45.7 diff --git a/buildkite/scripts/rocksdb-compatibility/rocksdb.py b/buildkite/scripts/rocksdb-compatibility/rocksdb.py new file mode 100644 index 00000000000..7ceca6f1c25 --- /dev/null +++ b/buildkite/scripts/rocksdb-compatibility/rocksdb.py @@ -0,0 +1,71 @@ +from cffi import FFI + +ffi = FFI() + +ffi.cdef(""" +typedef struct rocksdb_t rocksdb_t; +typedef struct rocksdb_options_t rocksdb_options_t; +typedef struct rocksdb_readoptions_t rocksdb_readoptions_t; +typedef struct rocksdb_iterator_t rocksdb_iterator_t; + +rocksdb_options_t* rocksdb_options_create(void); +void rocksdb_options_destroy(rocksdb_options_t*); +void rocksdb_options_set_create_if_missing(rocksdb_options_t*, unsigned char); + +rocksdb_t* rocksdb_open(const rocksdb_options_t* options, const char* name, char** errptr); +void rocksdb_close(rocksdb_t* db); + +rocksdb_readoptions_t* rocksdb_readoptions_create(void); +void rocksdb_readoptions_destroy(rocksdb_readoptions_t*); + +rocksdb_iterator_t* rocksdb_create_iterator(rocksdb_t* db, const rocksdb_readoptions_t* options); +void rocksdb_iter_destroy(rocksdb_iterator_t* iter); +void rocksdb_iter_seek_to_first(rocksdb_iterator_t* iter); +unsigned char rocksdb_iter_valid(const rocksdb_iterator_t* iter); +void rocksdb_iter_next(rocksdb_iterator_t* iter); +const char* rocksdb_iter_key(const rocksdb_iterator_t* iter, size_t* klen); +const char* rocksdb_iter_value(const rocksdb_iterator_t* iter, size_t* vlen); +""") + +# Load the library +rocksdb = ffi.dlopen("librocksdb.so") + + +def test(path): + # --- Open existing DB --- + options = rocksdb.rocksdb_options_create() + rocksdb.rocksdb_options_set_create_if_missing(options, 0) # do not create new DB + err_ptr = ffi.new("char**") + db = rocksdb.rocksdb_open(options, path, err_ptr) + if err_ptr[0] != ffi.NULL: + raise RuntimeError("Open error: " + ffi.string(err_ptr[0]).decode()) + rocksdb.rocksdb_options_destroy(options) + + # --- Create read options and iterator --- + ropts = rocksdb.rocksdb_readoptions_create() + iter_ = rocksdb.rocksdb_create_iterator(db, ropts) + + count = 0 + # --- Iterate over all keys --- + rocksdb.rocksdb_iter_seek_to_first(iter_) + while count < 10 and rocksdb.rocksdb_iter_valid(iter_): + klen = ffi.new("size_t*") + vlen = ffi.new("size_t*") + + key_ptr = rocksdb.rocksdb_iter_key(iter_, klen) + val_ptr = rocksdb.rocksdb_iter_value(iter_, vlen) + + # Create buffer views without copying + key_buf = ffi.buffer(key_ptr, klen[0]) + val_buf = ffi.buffer(val_ptr, vlen[0]) + + # key_buf and val_buf behave like bytes, e.g., + print(f"Found KV-pair: {key_buf[:].hex()} -> {val_buf[:].hex()}") + + rocksdb.rocksdb_iter_next(iter_) + count += 1 + + # --- Cleanup --- + rocksdb.rocksdb_iter_destroy(iter_) + rocksdb.rocksdb_readoptions_destroy(ropts) + rocksdb.rocksdb_close(db) diff --git a/buildkite/scripts/rocksdb-compatibility/test.py b/buildkite/scripts/rocksdb-compatibility/test.py new file mode 100644 index 00000000000..dd93b1a4bcb --- /dev/null +++ b/buildkite/scripts/rocksdb-compatibility/test.py @@ -0,0 +1,84 @@ +import base64 +import os +import random +import shutil +import tarfile +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path +from urllib.parse import urljoin +from tqdm import tqdm + +import requests +import rocksdb +import pycurl + +# Match keys starting with "genesis_ledger" or "epoch_ledger" and ending with ".tar.gz" +def matches_pattern(key: str) -> bool: + return (key.startswith("genesis_ledger") or key.startswith("epoch_ledger")) and key.endswith(".tar.gz") + + +def download_file(url: str, dest_path: str) -> None: + with open(dest_path, "wb") as f: + # Create a progress bar (tqdm) + pbar = tqdm(unit="B", unit_scale=True, unit_divisor=1024, ncols=80) + + def progress(download_t, download_d, upload_t, upload_d): + if download_t > 0: + pbar.total = download_t + pbar.update(download_d - pbar.n) + + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, f) + c.setopt(c.FOLLOWLOCATION, True) + c.setopt(c.NOPROGRESS, False) + c.setopt(c.XFERINFOFUNCTION, progress) + c.perform() + c.close() + + pbar.close() + + +def extract_tar_gz(tar_path: str, target_dir: str) -> None: + with tarfile.open(tar_path, "r:gz") as tar: + tar.extractall(path=target_dir) + +def main(): + # Fetch S3 XML listing + resp = requests.get("https://snark-keys.o1test.net.s3.amazonaws.com/", verify=False) + if not resp.ok: + raise RuntimeError(f"Failed to list S3 bucket: {resp.status_code} {resp.reason}") + + + ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"} + + root = ET.fromstring(resp.text) + tar_keys = [elem.text for elem in root.findall(".//s3:Contents/s3:Key", ns) if matches_pattern(elem.text) ] + + if not tar_keys: + raise RuntimeError("No ledger tar files found.") + + for tar_key in random.sample(tar_keys, min(5, len(tar_keys))): + tar_uri = urljoin("https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net/", tar_key) + print(f"Testing RocksDB compatibility on {tar_uri}") + + with tempfile.TemporaryDirectory() as tmpdir: + tar_path = os.path.join(tmpdir, os.path.basename(tar_key)) + print(f" Downloading to {tar_path}...") + download_file(tar_uri, tar_path) + + db_path = os.path.join(tmpdir, "extracted") + os.makedirs(db_path, exist_ok=True) + print(f" Extracting to {db_path}...") + extract_tar_gz(tar_path, db_path) + + print(f" Testing extracted RocksDB at {db_path}") + try: + rocksdb.test(db_path.encode('utf-8')) + except Exception as e: + raise RuntimeError(f"Failed to open RocksDB at {db_path}: {type(e).__name__}: {e}") + + +if __name__ == "__main__": + main() From 71a950ace6e6df3486c844a47f6c1160ff53319a Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:26:48 +0800 Subject: [PATCH 02/23] move RocksDB test from `/buildkite/scripts` -> `scripts` --- {buildkite/scripts => scripts}/rocksdb-compatibility/.gitignore | 0 .../scripts => scripts}/rocksdb-compatibility/install-rocksdb.sh | 0 .../scripts => scripts}/rocksdb-compatibility/requirements.txt | 0 {buildkite/scripts => scripts}/rocksdb-compatibility/rocksdb.py | 0 {buildkite/scripts => scripts}/rocksdb-compatibility/test.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {buildkite/scripts => scripts}/rocksdb-compatibility/.gitignore (100%) rename {buildkite/scripts => scripts}/rocksdb-compatibility/install-rocksdb.sh (100%) rename {buildkite/scripts => scripts}/rocksdb-compatibility/requirements.txt (100%) rename {buildkite/scripts => scripts}/rocksdb-compatibility/rocksdb.py (100%) rename {buildkite/scripts => scripts}/rocksdb-compatibility/test.py (100%) diff --git a/buildkite/scripts/rocksdb-compatibility/.gitignore b/scripts/rocksdb-compatibility/.gitignore similarity index 100% rename from buildkite/scripts/rocksdb-compatibility/.gitignore rename to scripts/rocksdb-compatibility/.gitignore diff --git a/buildkite/scripts/rocksdb-compatibility/install-rocksdb.sh b/scripts/rocksdb-compatibility/install-rocksdb.sh similarity index 100% rename from buildkite/scripts/rocksdb-compatibility/install-rocksdb.sh rename to scripts/rocksdb-compatibility/install-rocksdb.sh diff --git a/buildkite/scripts/rocksdb-compatibility/requirements.txt b/scripts/rocksdb-compatibility/requirements.txt similarity index 100% rename from buildkite/scripts/rocksdb-compatibility/requirements.txt rename to scripts/rocksdb-compatibility/requirements.txt diff --git a/buildkite/scripts/rocksdb-compatibility/rocksdb.py b/scripts/rocksdb-compatibility/rocksdb.py similarity index 100% rename from buildkite/scripts/rocksdb-compatibility/rocksdb.py rename to scripts/rocksdb-compatibility/rocksdb.py diff --git a/buildkite/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py similarity index 100% rename from buildkite/scripts/rocksdb-compatibility/test.py rename to scripts/rocksdb-compatibility/test.py From d21421272ea2e590f50958c075e3efe0a8140e84 Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:29:11 +0800 Subject: [PATCH 03/23] Don't use counter when looping --- scripts/rocksdb-compatibility/rocksdb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/rocksdb-compatibility/rocksdb.py b/scripts/rocksdb-compatibility/rocksdb.py index 7ceca6f1c25..1e0398bfd9a 100644 --- a/scripts/rocksdb-compatibility/rocksdb.py +++ b/scripts/rocksdb-compatibility/rocksdb.py @@ -45,10 +45,10 @@ def test(path): ropts = rocksdb.rocksdb_readoptions_create() iter_ = rocksdb.rocksdb_create_iterator(db, ropts) - count = 0 # --- Iterate over all keys --- rocksdb.rocksdb_iter_seek_to_first(iter_) - while count < 10 and rocksdb.rocksdb_iter_valid(iter_): + for _ in range(10): + if not rocksdb.rocksdb_iter_valid(iter_) klen = ffi.new("size_t*") vlen = ffi.new("size_t*") @@ -63,7 +63,6 @@ def test(path): print(f"Found KV-pair: {key_buf[:].hex()} -> {val_buf[:].hex()}") rocksdb.rocksdb_iter_next(iter_) - count += 1 # --- Cleanup --- rocksdb.rocksdb_iter_destroy(iter_) From 418986ba7cc4ecdff98af3c891b5d4600047b940 Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:33:52 +0800 Subject: [PATCH 04/23] Refactor RocksDB binding using context manager --- scripts/rocksdb-compatibility/rocksdb.py | 72 ++++++++++++++---------- scripts/rocksdb-compatibility/test.py | 2 +- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/scripts/rocksdb-compatibility/rocksdb.py b/scripts/rocksdb-compatibility/rocksdb.py index 1e0398bfd9a..0ef94d5a357 100644 --- a/scripts/rocksdb-compatibility/rocksdb.py +++ b/scripts/rocksdb-compatibility/rocksdb.py @@ -1,4 +1,5 @@ from cffi import FFI +from contextlib import contextmanager ffi = FFI() @@ -30,41 +31,50 @@ # Load the library rocksdb = ffi.dlopen("librocksdb.so") - -def test(path): - # --- Open existing DB --- - options = rocksdb.rocksdb_options_create() - rocksdb.rocksdb_options_set_create_if_missing(options, 0) # do not create new DB +@contextmanager +def rocksdb_options(create_if_missing=False): + opts = rocksdb.rocksdb_options_create() + rocksdb.rocksdb_options_set_create_if_missing(opts, int(create_if_missing)) + try: + yield opts + finally: + rocksdb.rocksdb_options_destroy(opts) + +@contextmanager +def open_db(path, options): err_ptr = ffi.new("char**") - db = rocksdb.rocksdb_open(options, path, err_ptr) + db = rocksdb.rocksdb_open(options, path.encode('utf-8'), err_ptr) if err_ptr[0] != ffi.NULL: raise RuntimeError("Open error: " + ffi.string(err_ptr[0]).decode()) - rocksdb.rocksdb_options_destroy(options) + try: + yield db + finally: + rocksdb.rocksdb_close(db) - # --- Create read options and iterator --- +@contextmanager +def read_iter(db): ropts = rocksdb.rocksdb_readoptions_create() iter_ = rocksdb.rocksdb_create_iterator(db, ropts) + try: + yield iter_ + finally: + rocksdb.rocksdb_iter_destroy(iter_) + rocksdb.rocksdb_readoptions_destroy(ropts) - # --- Iterate over all keys --- - rocksdb.rocksdb_iter_seek_to_first(iter_) - for _ in range(10): - if not rocksdb.rocksdb_iter_valid(iter_) - klen = ffi.new("size_t*") - vlen = ffi.new("size_t*") - - key_ptr = rocksdb.rocksdb_iter_key(iter_, klen) - val_ptr = rocksdb.rocksdb_iter_value(iter_, vlen) - - # Create buffer views without copying - key_buf = ffi.buffer(key_ptr, klen[0]) - val_buf = ffi.buffer(val_ptr, vlen[0]) - - # key_buf and val_buf behave like bytes, e.g., - print(f"Found KV-pair: {key_buf[:].hex()} -> {val_buf[:].hex()}") - - rocksdb.rocksdb_iter_next(iter_) - - # --- Cleanup --- - rocksdb.rocksdb_iter_destroy(iter_) - rocksdb.rocksdb_readoptions_destroy(ropts) - rocksdb.rocksdb_close(db) +def test(path): + with rocksdb_options(create_if_missing=False) as opts, open_db(path, opts) as db, read_iter(db) as it: + rocksdb.rocksdb_iter_seek_to_first(it) + for _ in range(10): + if not rocksdb.rocksdb_iter_valid(it): + break + + klen = ffi.new("size_t*") + vlen = ffi.new("size_t*") + key_ptr = rocksdb.rocksdb_iter_key(it, klen) + val_ptr = rocksdb.rocksdb_iter_value(it, vlen) + + key_buf = ffi.buffer(key_ptr, klen[0]) + val_buf = ffi.buffer(val_ptr, vlen[0]) + print(f"Found KV-pair: {key_buf[:].hex()} -> {val_buf[:].hex()}") + + rocksdb.rocksdb_iter_next(it) diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index dd93b1a4bcb..64ace6c7f4c 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -75,7 +75,7 @@ def main(): print(f" Testing extracted RocksDB at {db_path}") try: - rocksdb.test(db_path.encode('utf-8')) + rocksdb.test(db_path) except Exception as e: raise RuntimeError(f"Failed to open RocksDB at {db_path}: {type(e).__name__}: {e}") From 4e97d2daa78301f26a75bf4f52f9dc9635841bed Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:38:20 +0800 Subject: [PATCH 05/23] Remove requests as dependency --- .../rocksdb-compatibility/requirements.txt | 1 - scripts/rocksdb-compatibility/test.py | 39 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/scripts/rocksdb-compatibility/requirements.txt b/scripts/rocksdb-compatibility/requirements.txt index 2f2609dd589..40fdc62a1d5 100644 --- a/scripts/rocksdb-compatibility/requirements.txt +++ b/scripts/rocksdb-compatibility/requirements.txt @@ -1,4 +1,3 @@ cffi==2.0.0 -requests==2.31 tqdm==4.65 pycurl==7.45.7 diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index 64ace6c7f4c..1088f0b7718 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -5,13 +5,14 @@ import tarfile import tempfile import xml.etree.ElementTree as ET +from io import BytesIO from pathlib import Path from urllib.parse import urljoin + +import pycurl from tqdm import tqdm -import requests import rocksdb -import pycurl # Match keys starting with "genesis_ledger" or "epoch_ledger" and ending with ".tar.gz" def matches_pattern(key: str) -> bool: @@ -44,17 +45,33 @@ def extract_tar_gz(tar_path: str, target_dir: str) -> None: with tarfile.open(tar_path, "r:gz") as tar: tar.extractall(path=target_dir) -def main(): - # Fetch S3 XML listing - resp = requests.get("https://snark-keys.o1test.net.s3.amazonaws.com/", verify=False) - if not resp.ok: - raise RuntimeError(f"Failed to list S3 bucket: {resp.status_code} {resp.reason}") - - +def list_s3_keys(url, matches_pattern): + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, buffer) + c.setopt(c.FOLLOWLOCATION, True) + c.setopt(c.SSL_VERIFYPEER, False) + c.setopt(c.SSL_VERIFYHOST, 0) + c.perform() + status_code = c.getinfo(c.RESPONSE_CODE) + c.close() + + if status_code != 200: + raise RuntimeError(f"Failed to list S3 bucket: {status_code}") + + data = buffer.getvalue() + root = ET.fromstring(data) ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"} + tar_keys = [ + elem.text + for elem in root.findall(".//s3:Contents/s3:Key", ns) + if matches_pattern(elem.text) + ] + return tar_keys - root = ET.fromstring(resp.text) - tar_keys = [elem.text for elem in root.findall(".//s3:Contents/s3:Key", ns) if matches_pattern(elem.text) ] +def main(): + tar_keys = list_s3_keys("https://snark-keys.o1test.net.s3.amazonaws.com/", matches_pattern) if not tar_keys: raise RuntimeError("No ledger tar files found.") From 92820f7a1a4471a6e4f4be7129cc08618ded05f9 Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:41:00 +0800 Subject: [PATCH 06/23] Add document for rocksDB compatibility test in code --- scripts/rocksdb-compatibility/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/rocksdb-compatibility/README.md diff --git a/scripts/rocksdb-compatibility/README.md b/scripts/rocksdb-compatibility/README.md new file mode 100644 index 00000000000..3c78a798df1 --- /dev/null +++ b/scripts/rocksdb-compatibility/README.md @@ -0,0 +1,3 @@ +To run this test: +1. Run `install-rocksdb.sh` (preferably in a docker because it installs to system library) to ensure rocksdb dyn lib are installed +2. Run `test.py` inside a venv where everything in `requirements.txt` is installed From ac79ed8e1e2e0409262a58684c009118c546f591 Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:43:29 +0800 Subject: [PATCH 07/23] make test round configurable --- scripts/rocksdb-compatibility/rocksdb.py | 17 +++++++++++++++-- scripts/rocksdb-compatibility/test.py | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/rocksdb-compatibility/rocksdb.py b/scripts/rocksdb-compatibility/rocksdb.py index 0ef94d5a357..0469029fc96 100644 --- a/scripts/rocksdb-compatibility/rocksdb.py +++ b/scripts/rocksdb-compatibility/rocksdb.py @@ -61,10 +61,23 @@ def read_iter(db): rocksdb.rocksdb_iter_destroy(iter_) rocksdb.rocksdb_readoptions_destroy(ropts) -def test(path): +def test(path, rounds): + """ + Iterate over a RocksDB database and print key-value pairs in hexadecimal. + + Args: + path (str): Path to the RocksDB database. + rounds (int): Number of key-value pairs to read from the start of the database. + + Behavior: + - Opens the database in read-only mode (does not create a new DB). + - Uses a RocksDB iterator to traverse from the first key. + - Prints each key-value pair as hexadecimal strings. + - Stops early if the iterator reaches the end of the DB before 'rounds' entries. + """ with rocksdb_options(create_if_missing=False) as opts, open_db(path, opts) as db, read_iter(db) as it: rocksdb.rocksdb_iter_seek_to_first(it) - for _ in range(10): + for _ in range(rounds): if not rocksdb.rocksdb_iter_valid(it): break diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index 1088f0b7718..d39cf295b01 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -92,7 +92,7 @@ def main(): print(f" Testing extracted RocksDB at {db_path}") try: - rocksdb.test(db_path) + rocksdb.test(db_path, 10) except Exception as e: raise RuntimeError(f"Failed to open RocksDB at {db_path}: {type(e).__name__}: {e}") From 7a4e749743f4c8474dc46efd2d8b486f3170024c Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 12:44:48 +0800 Subject: [PATCH 08/23] Lift magic constants to top level in test --- scripts/rocksdb-compatibility/test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index d39cf295b01..fe0bc2b0aeb 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -14,6 +14,9 @@ import rocksdb +NUM_LEDGER_TARS = 5 +NUM_KV_PER_LEDGER = 10 + # Match keys starting with "genesis_ledger" or "epoch_ledger" and ending with ".tar.gz" def matches_pattern(key: str) -> bool: return (key.startswith("genesis_ledger") or key.startswith("epoch_ledger")) and key.endswith(".tar.gz") @@ -76,7 +79,7 @@ def main(): if not tar_keys: raise RuntimeError("No ledger tar files found.") - for tar_key in random.sample(tar_keys, min(5, len(tar_keys))): + for tar_key in random.sample(tar_keys, min(NUM_LEDGER_TARS, len(tar_keys))): tar_uri = urljoin("https://s3-us-west-2.amazonaws.com/snark-keys.o1test.net/", tar_key) print(f"Testing RocksDB compatibility on {tar_uri}") @@ -92,7 +95,7 @@ def main(): print(f" Testing extracted RocksDB at {db_path}") try: - rocksdb.test(db_path, 10) + rocksdb.test(db_path, NUM_KV_PER_LEDGER) except Exception as e: raise RuntimeError(f"Failed to open RocksDB at {db_path}: {type(e).__name__}: {e}") From ff523ceb25d3d1d2e1838d5ced0689277c1cb4fc Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 13:17:25 +0800 Subject: [PATCH 09/23] RocksDB Compatibility: Make pyright happier on test.py --- scripts/rocksdb-compatibility/test.py | 35 +++++++++++++-------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index fe0bc2b0aeb..44971eec5e1 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -1,12 +1,10 @@ -import base64 import os import random -import shutil import tarfile import tempfile import xml.etree.ElementTree as ET from io import BytesIO -from pathlib import Path +from typing import List from urllib.parse import urljoin import pycurl @@ -27,17 +25,18 @@ def download_file(url: str, dest_path: str) -> None: # Create a progress bar (tqdm) pbar = tqdm(unit="B", unit_scale=True, unit_divisor=1024, ncols=80) - def progress(download_t, download_d, upload_t, upload_d): + def progress(download_t, download_d, _upload_t, _upload_d): + _ = (_upload_t, _upload_d) # Make pyright happier if download_t > 0: pbar.total = download_t pbar.update(download_d - pbar.n) c = pycurl.Curl() - c.setopt(c.URL, url) - c.setopt(c.WRITEDATA, f) - c.setopt(c.FOLLOWLOCATION, True) - c.setopt(c.NOPROGRESS, False) - c.setopt(c.XFERINFOFUNCTION, progress) + c.setopt(pycurl.URL, url) + c.setopt(pycurl.WRITEDATA, f) + c.setopt(pycurl.FOLLOWLOCATION, True) + c.setopt(pycurl.NOPROGRESS, False) + c.setopt(pycurl.XFERINFOFUNCTION, progress) c.perform() c.close() @@ -48,16 +47,16 @@ def extract_tar_gz(tar_path: str, target_dir: str) -> None: with tarfile.open(tar_path, "r:gz") as tar: tar.extractall(path=target_dir) -def list_s3_keys(url, matches_pattern): +def list_s3_keys(url, matches_pattern) -> List[str] : buffer = BytesIO() c = pycurl.Curl() - c.setopt(c.URL, url) - c.setopt(c.WRITEDATA, buffer) - c.setopt(c.FOLLOWLOCATION, True) - c.setopt(c.SSL_VERIFYPEER, False) - c.setopt(c.SSL_VERIFYHOST, 0) + c.setopt(pycurl.URL, url) + c.setopt(pycurl.WRITEDATA, buffer) + c.setopt(pycurl.FOLLOWLOCATION, True) + c.setopt(pycurl.SSL_VERIFYPEER, False) + c.setopt(pycurl.SSL_VERIFYHOST, 0) c.perform() - status_code = c.getinfo(c.RESPONSE_CODE) + status_code = c.getinfo(pycurl.RESPONSE_CODE) c.close() if status_code != 200: @@ -67,9 +66,9 @@ def list_s3_keys(url, matches_pattern): root = ET.fromstring(data) ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"} tar_keys = [ - elem.text + text for elem in root.findall(".//s3:Contents/s3:Key", ns) - if matches_pattern(elem.text) + if (text := elem.text) is not None and matches_pattern(text) ] return tar_keys From b56a2ce696ea6b8f6d471d54e40bf812093ddb4d Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 13:21:26 +0800 Subject: [PATCH 10/23] RocksDB test: don't catch and rethrow --- scripts/rocksdb-compatibility/test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index 44971eec5e1..10beac23b7d 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -93,10 +93,7 @@ def main(): extract_tar_gz(tar_path, db_path) print(f" Testing extracted RocksDB at {db_path}") - try: - rocksdb.test(db_path, NUM_KV_PER_LEDGER) - except Exception as e: - raise RuntimeError(f"Failed to open RocksDB at {db_path}: {type(e).__name__}: {e}") + rocksdb.test(db_path, NUM_KV_PER_LEDGER) if __name__ == "__main__": From b65b62e93b2b92252e5d28fd9ffd55a7e5607c59 Mon Sep 17 00:00:00 2001 From: glyh Date: Fri, 17 Oct 2025 13:23:21 +0800 Subject: [PATCH 11/23] Add a TODO to figure out how to enable SSL ceritiifacte when listing from S3 bucket --- scripts/rocksdb-compatibility/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/rocksdb-compatibility/test.py b/scripts/rocksdb-compatibility/test.py index 10beac23b7d..eaeca114301 100644 --- a/scripts/rocksdb-compatibility/test.py +++ b/scripts/rocksdb-compatibility/test.py @@ -47,6 +47,7 @@ def extract_tar_gz(tar_path: str, target_dir: str) -> None: with tarfile.open(tar_path, "r:gz") as tar: tar.extractall(path=target_dir) +# TODO: figure out how to enable SSL here def list_s3_keys(url, matches_pattern) -> List[str] : buffer = BytesIO() c = pycurl.Curl() From 30768eac1114978975d43c5c4de36fd027500cb7 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 17 Oct 2025 13:50:01 +0200 Subject: [PATCH 12/23] Ci test for rocksdb --- buildkite/src/Jobs/Test/RocksdbTest.dhall | 48 +++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 buildkite/src/Jobs/Test/RocksdbTest.dhall diff --git a/buildkite/src/Jobs/Test/RocksdbTest.dhall b/buildkite/src/Jobs/Test/RocksdbTest.dhall new file mode 100644 index 00000000000..80ad6413249 --- /dev/null +++ b/buildkite/src/Jobs/Test/RocksdbTest.dhall @@ -0,0 +1,48 @@ +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall + +let Size = ../../Command/Size.dhall + +let RunInToolchain = ../../Command/RunInToolchain.dhall + +let key = "archive-hardfork-toolbox-test" + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart + (S.contains "buildkite/scripts/rocksdb-compatibility") + , S.exactly "buildkite/src/Jobs/Test/RocksdbTest" "dhall" + ] + , path = "Test" + , name = "RocksdbTest" + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + ([] : List Text) + ( "./buildkite/scripts/rocksdb-compatibility/install-rocks.sh" + ++ " && pip install -r buildkite/scripts/rocksdb-compatibility/requirements.txt" + ++ " && python3 ./buildkite/scripts/rocksdb-compatibility/rocksdb.py" + ) + , label = "Rocksdb Test" + , key = key + , target = Size.Large + } + ] + } From 8de71d5e481f34be9bc4c3e55390b9a66e6d4d6b Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 17 Oct 2025 14:44:37 +0200 Subject: [PATCH 13/23] fix dirty when --- buildkite/src/Jobs/Test/RocksdbTest.dhall | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/RocksdbTest.dhall b/buildkite/src/Jobs/Test/RocksdbTest.dhall index 80ad6413249..e0eca172825 100644 --- a/buildkite/src/Jobs/Test/RocksdbTest.dhall +++ b/buildkite/src/Jobs/Test/RocksdbTest.dhall @@ -18,8 +18,7 @@ in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/scripts/rocksdb-compatibility") + [ S.strictlyStart (S.contains "scripts/rocksdb-compatibility") , S.exactly "buildkite/src/Jobs/Test/RocksdbTest" "dhall" ] , path = "Test" From ddbbd69a47c157525d085edeb48202e5cfa7eb95 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 17 Oct 2025 20:23:03 +0200 Subject: [PATCH 14/23] update name and fix paths --- ...dhall => RocksDBLedgerTarCompatibilityTest.dhall} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename buildkite/src/Jobs/Test/{RocksdbTest.dhall => RocksDBLedgerTarCompatibilityTest.dhall} (72%) diff --git a/buildkite/src/Jobs/Test/RocksdbTest.dhall b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall similarity index 72% rename from buildkite/src/Jobs/Test/RocksdbTest.dhall rename to buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall index e0eca172825..9c24dafa7ac 100644 --- a/buildkite/src/Jobs/Test/RocksdbTest.dhall +++ b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall @@ -12,7 +12,7 @@ let Size = ../../Command/Size.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall -let key = "archive-hardfork-toolbox-test" +let key = "test" in Pipeline.build Pipeline.Config::{ @@ -22,7 +22,7 @@ in Pipeline.build , S.exactly "buildkite/src/Jobs/Test/RocksdbTest" "dhall" ] , path = "Test" - , name = "RocksdbTest" + , name = "RocksDBLedgerTarCompatibilityTest" , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Test @@ -35,11 +35,11 @@ in Pipeline.build , commands = RunInToolchain.runInToolchain ([] : List Text) - ( "./buildkite/scripts/rocksdb-compatibility/install-rocks.sh" - ++ " && pip install -r buildkite/scripts/rocksdb-compatibility/requirements.txt" - ++ " && python3 ./buildkite/scripts/rocksdb-compatibility/rocksdb.py" + ( "./scripts/rocksdb-compatibility/install-rocks.sh" + ++ " && pip install -r ./scripts/rocksdb-compatibility/requirements.txt" + ++ " && python3 ./scripts/rocksdb-compatibility/rocksdb.py" ) - , label = "Rocksdb Test" + , label = "RocksDB: Ledger Tar Compatibility Test" , key = key , target = Size.Large } From c7ef3b4f425ac843c99280aac4850dd4d9323016 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 20 Oct 2025 12:18:25 +0200 Subject: [PATCH 15/23] fix bash checks --- scripts/rocksdb-compatibility/install-rocksdb.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/rocksdb-compatibility/install-rocksdb.sh b/scripts/rocksdb-compatibility/install-rocksdb.sh index 289317ebfc9..22f1d623e75 100755 --- a/scripts/rocksdb-compatibility/install-rocksdb.sh +++ b/scripts/rocksdb-compatibility/install-rocksdb.sh @@ -4,7 +4,9 @@ set -euox pipefail ROCKSDB_VERSION=10.5.1 -ROCKSDB_SOURCE=`mktemp -d --tmpdir rocksdb-$ROCKSDB_VERSION.XXXXXX` +ROCKSDB_SOURCE=$(mktemp -d --tmpdir rocksdb-$ROCKSDB_VERSION.XXXXXX) + +# shellcheck disable=SC2064 trap "rm -rf $ROCKSDB_SOURCE" EXIT curl -L https://github.com/facebook/rocksdb/archive/refs/tags/v${ROCKSDB_VERSION}.tar.gz | tar xz -C $ROCKSDB_SOURCE @@ -16,4 +18,4 @@ cd $ROCKSDB_SOURCE/rocksdb-${ROCKSDB_VERSION} # util/compression.cc:684:40: error: unused parameter ‘args’ [-Werror=unused-parameter] # 684 | Status ExtractUncompressedSize(Args& args) override { # | ~~~~~~^~~~ -EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j$(nproc) install +EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j"$(nproc)" install From fac15ec4db9f44cc6c1bf73785ae74c12224023d Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 20 Oct 2025 12:18:30 +0200 Subject: [PATCH 16/23] fix dirtyWhen --- .../src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall index 9c24dafa7ac..34c93c5ac36 100644 --- a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall +++ b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall @@ -19,7 +19,7 @@ in Pipeline.build , spec = JobSpec::{ , dirtyWhen = [ S.strictlyStart (S.contains "scripts/rocksdb-compatibility") - , S.exactly "buildkite/src/Jobs/Test/RocksdbTest" "dhall" + , S.exactly "buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest" "dhall" ] , path = "Test" , name = "RocksDBLedgerTarCompatibilityTest" @@ -35,7 +35,7 @@ in Pipeline.build , commands = RunInToolchain.runInToolchain ([] : List Text) - ( "./scripts/rocksdb-compatibility/install-rocks.sh" + ( "./scripts/rocksdb-compatibility/install-rocksdb.sh" ++ " && pip install -r ./scripts/rocksdb-compatibility/requirements.txt" ++ " && python3 ./scripts/rocksdb-compatibility/rocksdb.py" ) From 2397895f9bc20f48482c88f0afa2af7a1158e4ab Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 20 Oct 2025 12:51:38 +0200 Subject: [PATCH 17/23] lint --- .../src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall index 34c93c5ac36..eeae64fa489 100644 --- a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall +++ b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall @@ -19,7 +19,9 @@ in Pipeline.build , spec = JobSpec::{ , dirtyWhen = [ S.strictlyStart (S.contains "scripts/rocksdb-compatibility") - , S.exactly "buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest" "dhall" + , S.exactly + "buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest" + "dhall" ] , path = "Test" , name = "RocksDBLedgerTarCompatibilityTest" From b67bb48f76235a3ea047e1e0921240db3e761274 Mon Sep 17 00:00:00 2001 From: glyh Date: Tue, 21 Oct 2025 10:52:57 +0800 Subject: [PATCH 18/23] Use sudo when installing RocksDB on CI --- scripts/rocksdb-compatibility/install-rocksdb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rocksdb-compatibility/install-rocksdb.sh b/scripts/rocksdb-compatibility/install-rocksdb.sh index 22f1d623e75..7893df8f5e6 100755 --- a/scripts/rocksdb-compatibility/install-rocksdb.sh +++ b/scripts/rocksdb-compatibility/install-rocksdb.sh @@ -18,4 +18,4 @@ cd $ROCKSDB_SOURCE/rocksdb-${ROCKSDB_VERSION} # util/compression.cc:684:40: error: unused parameter ‘args’ [-Werror=unused-parameter] # 684 | Status ExtractUncompressedSize(Args& args) override { # | ~~~~~~^~~~ -EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j"$(nproc)" install +sudo EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j"$(nproc)" install From b578311488457379f0dd659e07e00abe0f0472f1 Mon Sep 17 00:00:00 2001 From: glyh Date: Tue, 21 Oct 2025 11:13:42 +0800 Subject: [PATCH 19/23] Refactor RocksDB binding to use generator for `read_iter` --- scripts/rocksdb-compatibility/rocksdb.py | 45 ++++++++++++++---------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/scripts/rocksdb-compatibility/rocksdb.py b/scripts/rocksdb-compatibility/rocksdb.py index 0469029fc96..6097b85aa6c 100644 --- a/scripts/rocksdb-compatibility/rocksdb.py +++ b/scripts/rocksdb-compatibility/rocksdb.py @@ -51,14 +51,32 @@ def open_db(path, options): finally: rocksdb.rocksdb_close(db) -@contextmanager def read_iter(db): + """ + Generator that yields (key, value) pairs from a RocksDB database. + + Args: + db (rocksdb_t*): A RocksDB database handle. + + Yields: + tuple[bytes, bytes]: The (key, value) pairs from the database. + """ ropts = rocksdb.rocksdb_readoptions_create() - iter_ = rocksdb.rocksdb_create_iterator(db, ropts) + it = rocksdb.rocksdb_create_iterator(db, ropts) try: - yield iter_ + rocksdb.rocksdb_iter_seek_to_first(it) + while rocksdb.rocksdb_iter_valid(it): + klen = ffi.new("size_t*") + vlen = ffi.new("size_t*") + key_ptr = rocksdb.rocksdb_iter_key(it, klen) + val_ptr = rocksdb.rocksdb_iter_value(it, vlen) + yield ( + bytes(ffi.buffer(key_ptr, klen[0])), + bytes(ffi.buffer(val_ptr, vlen[0])), + ) + rocksdb.rocksdb_iter_next(it) finally: - rocksdb.rocksdb_iter_destroy(iter_) + rocksdb.rocksdb_iter_destroy(it) rocksdb.rocksdb_readoptions_destroy(ropts) def test(path, rounds): @@ -75,19 +93,8 @@ def test(path, rounds): - Prints each key-value pair as hexadecimal strings. - Stops early if the iterator reaches the end of the DB before 'rounds' entries. """ - with rocksdb_options(create_if_missing=False) as opts, open_db(path, opts) as db, read_iter(db) as it: - rocksdb.rocksdb_iter_seek_to_first(it) - for _ in range(rounds): - if not rocksdb.rocksdb_iter_valid(it): + with rocksdb_options(create_if_missing=False) as opts, open_db(path, opts) as db: + for i, (key, val) in enumerate(read_iter(db)): + print(f"Found KV-pair: {key.hex()} -> {val.hex()}") + if i + 1 >= rounds: break - - klen = ffi.new("size_t*") - vlen = ffi.new("size_t*") - key_ptr = rocksdb.rocksdb_iter_key(it, klen) - val_ptr = rocksdb.rocksdb_iter_value(it, vlen) - - key_buf = ffi.buffer(key_ptr, klen[0]) - val_buf = ffi.buffer(val_ptr, vlen[0]) - print(f"Found KV-pair: {key_buf[:].hex()} -> {val_buf[:].hex()}") - - rocksdb.rocksdb_iter_next(it) From faedb9664dadb8721e5e3f25cbe60d25eb85a3d8 Mon Sep 17 00:00:00 2001 From: glyh Date: Tue, 21 Oct 2025 11:24:22 +0800 Subject: [PATCH 20/23] Fix: install RocksDB as shared library so it could be located by python CFFI --- scripts/rocksdb-compatibility/install-rocksdb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rocksdb-compatibility/install-rocksdb.sh b/scripts/rocksdb-compatibility/install-rocksdb.sh index 7893df8f5e6..deb9d94f2df 100755 --- a/scripts/rocksdb-compatibility/install-rocksdb.sh +++ b/scripts/rocksdb-compatibility/install-rocksdb.sh @@ -18,4 +18,4 @@ cd $ROCKSDB_SOURCE/rocksdb-${ROCKSDB_VERSION} # util/compression.cc:684:40: error: unused parameter ‘args’ [-Werror=unused-parameter] # 684 | Status ExtractUncompressedSize(Args& args) override { # | ~~~~~~^~~~ -sudo EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j"$(nproc)" install +sudo EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j"$(nproc)" install-shared From 9c1dd207799c13ff28b98b35ec897decbd17530a Mon Sep 17 00:00:00 2001 From: glyh Date: Tue, 21 Oct 2025 11:39:48 +0800 Subject: [PATCH 21/23] Make RocksDB compat test of size generic-multi --- buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall index eeae64fa489..026c078c5cb 100644 --- a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall +++ b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall @@ -43,7 +43,7 @@ in Pipeline.build ) , label = "RocksDB: Ledger Tar Compatibility Test" , key = key - , target = Size.Large + , target = Size.Multi } ] } From e8db7bac2899b2914a3b136aa27949dc38aa022f Mon Sep 17 00:00:00 2001 From: glyh Date: Tue, 21 Oct 2025 13:49:39 +0800 Subject: [PATCH 22/23] Invoke `sudo ldconfig` after installing rocksdb library --- scripts/rocksdb-compatibility/install-rocksdb.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/rocksdb-compatibility/install-rocksdb.sh b/scripts/rocksdb-compatibility/install-rocksdb.sh index deb9d94f2df..94f0bffb5b9 100755 --- a/scripts/rocksdb-compatibility/install-rocksdb.sh +++ b/scripts/rocksdb-compatibility/install-rocksdb.sh @@ -19,3 +19,6 @@ cd $ROCKSDB_SOURCE/rocksdb-${ROCKSDB_VERSION} # 684 | Status ExtractUncompressedSize(Args& args) override { # | ~~~~~~^~~~ sudo EXTRA_CXXFLAGS="-Wno-unused-parameter" make -j"$(nproc)" install-shared + +# Refresh LD cache so follow up programs can locate the dyn libaray +sudo ldconfig From e8b4c805cf3d857d8ad8e5ad60d2301dd0665ba6 Mon Sep 17 00:00:00 2001 From: glyh Date: Tue, 21 Oct 2025 13:26:16 +0800 Subject: [PATCH 23/23] Rewrite RocksDB Ledger Tar test job to run on Raw ubuntu:noble --- .../RocksDBLedgerTarCompatibilityTest.dhall | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall index 026c078c5cb..7b88d401d72 100644 --- a/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall +++ b/buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall @@ -10,9 +10,18 @@ let Command = ../../Command/Base.dhall let Size = ../../Command/Size.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall +let Cmd = ../../Lib/Cmds.dhall -let key = "test" +let Docker = ../../Command/Docker/Type.dhall + +let commands = + [ Cmd.run + "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --quiet --yes python3 python3-pip build-essential sudo curl" + , Cmd.run "./scripts/rocksdb-compatibility/install-rocksdb.sh" + , Cmd.run + "pip3 install --break-system-packages -r ./scripts/rocksdb-compatibility/requirements.txt" + , Cmd.run "python3 ./scripts/rocksdb-compatibility/test.py" + ] in Pipeline.build Pipeline.Config::{ @@ -26,7 +35,7 @@ in Pipeline.build , path = "Test" , name = "RocksDBLedgerTarCompatibilityTest" , tags = - [ PipelineTag.Type.Long + [ PipelineTag.Type.Fast , PipelineTag.Type.Test , PipelineTag.Type.Stable ] @@ -34,16 +43,11 @@ in Pipeline.build , steps = [ Command.build Command.Config::{ - , commands = - RunInToolchain.runInToolchain - ([] : List Text) - ( "./scripts/rocksdb-compatibility/install-rocksdb.sh" - ++ " && pip install -r ./scripts/rocksdb-compatibility/requirements.txt" - ++ " && python3 ./scripts/rocksdb-compatibility/rocksdb.py" - ) - , label = "RocksDB: Ledger Tar Compatibility Test" - , key = key + , commands = commands + , label = "Check RocksDB Ledger Tar Compatibility" + , key = "test" , target = Size.Multi + , docker = Some Docker::{ image = "ubuntu:noble" } } ] }