refactor: only extract metadata and don't try to calculate offset index

Author: kszucs

libs/libviewer/Cargo.lock

@@ -9,14 +9,14 @@ name = "libviewer"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "55", features = ["pyarrow"] }
+arrow = { version = "56", features = ["pyarrow"] }
 futures = "0.3"
 object_store = "0.12.0"
 object_store_opendal = "0.52.0"
 opendal = { version = "0.53.2", features = ["services-huggingface"] }
-parquet = { version = "55", features = ["async", "object_store"] }
-pyo3 = { version = "0.24", features = ["extension-module"] }
-pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"] }
+parquet = { version = "56", features = ["async", "object_store"] }
+pyo3 = { version = "^0.25", features = ["extension-module"] }
+pyo3-async-runtimes = { version = "^0.25", features = ["tokio-runtime"] }
 tempfile = "3.20.0"
 thiserror = "2.0.12"
 thrift = "0.17"

libs/libviewer/Cargo.toml

@@ -1,7 +1,10 @@
 import os
-from ._internal import PyDataset
+
 from huggingface_hub import hf_hub_download, list_repo_files
 
+from ._internal import PyDataset
+
+
 __all__ = ["Dataset"]
 
 
@@ -12,9 +15,9 @@ class Dataset(PyDataset):
     #     ...
     # ]
 
-    def from_hub(repo, metadata_store):
+    def from_hub(repo, metadata_store, revision=None):
         """Create a Dataset from Hugging Face Hub."""
-        repo_files = list_repo_files(repo, repo_type="dataset")
+        repo_files = list_repo_files(repo, repo_type="dataset", revision=revision)
 
         parquet_files = []
         for filename in repo_files:
@@ -31,12 +34,16 @@ def from_hub(repo, metadata_store):
             raise ValueError(f"No parquet files found in the dataset '{repo}'.")
 
         return Dataset(
-            repo, parquet_files, data_store="hf://", metadata_store=metadata_store
+            repo,
+            parquet_files,
+            revision=revision,
+            data_store=f"hf://datasets/{repo}",
+            metadata_store=metadata_store,
         )
 
-    def from_cache(repo, metadata_store, download=False):
+    def from_cache(repo, metadata_store, revision=None, download=False):
         """Create a Dataset from HF local cache."""
-        repo_files = list_repo_files(repo, repo_type="dataset")
+        repo_files = list_repo_files(repo, repo_type="dataset", revision=revision)
 
         parquet_files = []
         for filename in repo_files:
@@ -57,5 +64,9 @@ def from_cache(repo, metadata_store, download=False):
             raise ValueError(f"No parquet files found in the dataset '{repo}'.")
 
         return Dataset(
-            repo, parquet_files, data_store="file://", metadata_store=metadata_store
+            repo,
+            parquet_files,
+            revision=revision,
+            data_store="file://",
+            metadata_store=metadata_store,
         )