From b37c1b45d7ffde8f96382a83069d5e431c80650c Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 5 May 2025 14:45:59 +0300 Subject: [PATCH 1/7] Add example how one would implement expiring tags --- examples/expiring-tags.rs | 155 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 examples/expiring-tags.rs diff --git a/examples/expiring-tags.rs b/examples/expiring-tags.rs new file mode 100644 index 00000000..b736bcf6 --- /dev/null +++ b/examples/expiring-tags.rs @@ -0,0 +1,155 @@ +use std::time::{Duration, SystemTime}; + +use chrono::Utc; +use futures_lite::StreamExt; +use iroh::endpoint; +use iroh_blobs::store::GcConfig; +use iroh_blobs::{hashseq::HashSeq, BlobFormat, HashAndFormat}; +use iroh_blobs::Hash; + +use iroh_blobs::rpc::client::blobs::MemClient as BlobsClient; +use tokio::signal::ctrl_c; + +/// Using an iroh rpc client, create a tag that is marked to expire at `expiry` for all the given hashes. +/// +/// The tag name will be `prefix`- followed by the expiry date in iso8601 format (e.g. `expiry-2025-01-01T12:00:00Z`). +/// +async fn create_expiring_tag( + iroh: &BlobsClient, + hashes: &[Hash], + prefix: &str, + expiry: SystemTime, +) -> anyhow::Result<()> { + let expiry = chrono::DateTime::::from(expiry); + let expiry = expiry.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let tagname = format!("{}-{}", prefix, expiry); + let batch = iroh.batch().await?; + let tt = if hashes.is_empty() { + return Ok(()); + } else if hashes.len() == 1 { + let hash = hashes[0]; + batch.temp_tag(HashAndFormat::raw(hash)).await? + } else { + let hs = hashes.into_iter().copied().collect::(); + batch + .add_bytes_with_opts(hs.into_inner(), BlobFormat::HashSeq) + .await? + }; + batch.persist_to(tt, tagname.as_str().into()).await?; + println!("Created tag {}", tagname); + Ok(()) +} + +async fn delete_expired_tags(iroh: &BlobsClient, prefix: &str) -> anyhow::Result<()> { + let mut tags = iroh.tags().list().await?; + let prefix = format!("{}-", prefix); + let now = chrono::Utc::now(); + let mut to_delete = Vec::new(); + while let Some(tag) = tags.next().await { + let tag = tag?.name; + if let Some(rest) = tag.0.strip_prefix(prefix.as_bytes()) { + let Ok(expiry) = std::str::from_utf8(rest) else { + tracing::warn!("Tag {} does have non utf8 expiry", tag); + continue; + }; + let Ok(expiry) = chrono::DateTime::parse_from_rfc3339(expiry) else { + tracing::warn!("Tag {} does have invalid expiry date", tag); + continue; + }; + let expiry = expiry.with_timezone(&Utc); + if expiry < now { + to_delete.push(tag); + } + } + } + for tag in to_delete { + println!("Deleting expired tag {}", tag); + iroh.tags().delete(tag).await?; + } + Ok(()) +} + +async fn print_tags_task(blobs: BlobsClient) -> anyhow::Result<()> { + loop { + let now = chrono::Utc::now(); + let mut tags = blobs.tags().list().await?; + println!("Tags at {}:\n", now); + while let Some(tag) = tags.next().await { + let tag = tag?; + println!(" {:?}", tag); + } + println!(); + tokio::time::sleep(Duration::from_secs(5)).await; + } +} + +async fn print_blobs_task(blobs: BlobsClient) -> anyhow::Result<()> { + loop { + let now = chrono::Utc::now(); + let mut blobs = blobs.list().await?; + println!("Blobs at {}:\n", now); + while let Some(info) = blobs.next().await { + println!(" {:?}", info?); + } + println!(); + tokio::time::sleep(Duration::from_secs(5)).await; + } +} + +async fn delete_expired_tags_task(blobs: BlobsClient, prefix: &str, ) -> anyhow::Result<()> { + loop { + delete_expired_tags(&blobs, prefix).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt::init(); + let endpoint = endpoint::Endpoint::builder().bind().await?; + let store = iroh_blobs::store::fs::Store::load("blobs").await?; + let blobs = iroh_blobs::net_protocol::Blobs::builder(store) + .build(&endpoint); + // enable gc with a short period + blobs.start_gc(GcConfig { + period: Duration::from_secs(1), + done_callback: None, + })?; + // create a router and add blobs as a service + // + // You can skip this if you don't want to serve the data over the network. + let router = iroh::protocol::Router::builder(endpoint) + .accept(iroh_blobs::ALPN, blobs.clone()) + .spawn().await?; + + // setup: add some data and tag it + { + // add several blobs and tag them with an expiry date 10 seconds in the future + let batch = blobs.client().batch().await?; + let a = batch.add_bytes("blob 1".as_bytes()).await?; + let b = batch.add_bytes("blob 2".as_bytes()).await?; + let expires_at = SystemTime::now().checked_add(Duration::from_secs(10)).unwrap(); + create_expiring_tag(blobs.client(), &[*a.hash(), *b.hash()], "expiring", expires_at).await?; + + // add a single blob and tag it with an expiry date 60 seconds in the future + let c = batch.add_bytes("blob 3".as_bytes()).await?; + let expires_at = SystemTime::now().checked_add(Duration::from_secs(60)).unwrap(); + create_expiring_tag(blobs.client(), &[*c.hash()], "expiring", expires_at).await?; + // batch goes out of scope, so data is only protected by the tags we created + } + let client = blobs.client().clone(); + + // delete expired tags every 5 seconds + let check_task = tokio::spawn(delete_expired_tags_task(client.clone(), "expiring")); + // print tags every 5 seconds + let print_tags_task = tokio::spawn(print_tags_task(client.clone())); + // print blobs every 5 seconds + let print_blobs_task = tokio::spawn(print_blobs_task(client)); + + ctrl_c().await?; + router.shutdown().await?; + check_task.abort(); + print_tags_task.abort(); + print_blobs_task.abort(); + Ok(()) +} \ No newline at end of file From 0600dded42feb655157fc82e14d36afde483c95a Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 5 May 2025 15:03:47 +0300 Subject: [PATCH 2/7] prettify the info output --- examples/expiring-tags.rs | 137 ++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 56 deletions(-) diff --git a/examples/expiring-tags.rs b/examples/expiring-tags.rs index b736bcf6..9feb3e78 100644 --- a/examples/expiring-tags.rs +++ b/examples/expiring-tags.rs @@ -1,19 +1,24 @@ +//! This example shows how to create tags that expire after a certain time. +//! +//! We use a prefix so we can distinguish between expiring and normal tags, and +//! then encode the expiry date in the tag name after the prefix, in a format +//! that sorts in the same order as the expiry date. +//! +//! Then we can just use use std::time::{Duration, SystemTime}; use chrono::Utc; use futures_lite::StreamExt; use iroh::endpoint; -use iroh_blobs::store::GcConfig; -use iroh_blobs::{hashseq::HashSeq, BlobFormat, HashAndFormat}; -use iroh_blobs::Hash; - -use iroh_blobs::rpc::client::blobs::MemClient as BlobsClient; +use iroh_blobs::{ + hashseq::HashSeq, rpc::client::blobs::MemClient as BlobsClient, store::GcConfig, BlobFormat, + Hash, HashAndFormat, Tag, +}; use tokio::signal::ctrl_c; /// Using an iroh rpc client, create a tag that is marked to expire at `expiry` for all the given hashes. /// /// The tag name will be `prefix`- followed by the expiry date in iso8601 format (e.g. `expiry-2025-01-01T12:00:00Z`). -/// async fn create_expiring_tag( iroh: &BlobsClient, hashes: &[Hash], @@ -40,65 +45,78 @@ async fn create_expiring_tag( Ok(()) } -async fn delete_expired_tags(iroh: &BlobsClient, prefix: &str) -> anyhow::Result<()> { - let mut tags = iroh.tags().list().await?; +async fn delete_expired_tags(blobs: &BlobsClient, prefix: &str, bulk: bool) -> anyhow::Result<()> { + let mut tags = blobs.tags().list().await?; let prefix = format!("{}-", prefix); let now = chrono::Utc::now(); - let mut to_delete = Vec::new(); - while let Some(tag) = tags.next().await { - let tag = tag?.name; - if let Some(rest) = tag.0.strip_prefix(prefix.as_bytes()) { - let Ok(expiry) = std::str::from_utf8(rest) else { - tracing::warn!("Tag {} does have non utf8 expiry", tag); - continue; - }; - let Ok(expiry) = chrono::DateTime::parse_from_rfc3339(expiry) else { - tracing::warn!("Tag {} does have invalid expiry date", tag); - continue; - }; - let expiry = expiry.with_timezone(&Utc); - if expiry < now { - to_delete.push(tag); + let end = format!( + "{}-{}", + prefix, + now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + ); + if bulk { + // delete all tags with the prefix and an expiry date before now + // + // this should be very efficient, since it is just a single database operation + blobs + .tags() + .delete_range(Tag::from(prefix.clone())..Tag::from(end)) + .await?; + } else { + // find tags to delete one by one and then delete them + // + // this allows us to print the tags before deleting them + let mut to_delete = Vec::new(); + while let Some(tag) = tags.next().await { + let tag = tag?.name; + if let Some(rest) = tag.0.strip_prefix(prefix.as_bytes()) { + let Ok(expiry) = std::str::from_utf8(rest) else { + tracing::warn!("Tag {} does have non utf8 expiry", tag); + continue; + }; + let Ok(expiry) = chrono::DateTime::parse_from_rfc3339(expiry) else { + tracing::warn!("Tag {} does have invalid expiry date", tag); + continue; + }; + let expiry = expiry.with_timezone(&Utc); + if expiry < now { + to_delete.push(tag); + } } } - } - for tag in to_delete { - println!("Deleting expired tag {}", tag); - iroh.tags().delete(tag).await?; + for tag in to_delete { + println!("Deleting expired tag {}", tag); + blobs.tags().delete(tag).await?; + } } Ok(()) } -async fn print_tags_task(blobs: BlobsClient) -> anyhow::Result<()> { +async fn info_task(blobs: BlobsClient) -> anyhow::Result<()> { + tokio::time::sleep(Duration::from_secs(1)).await; loop { let now = chrono::Utc::now(); let mut tags = blobs.tags().list().await?; - println!("Tags at {}:\n", now); + println!("Current time: {}", now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)); + println!("Tags:"); while let Some(tag) = tags.next().await { let tag = tag?; println!(" {:?}", tag); } - println!(); - tokio::time::sleep(Duration::from_secs(5)).await; - } -} - -async fn print_blobs_task(blobs: BlobsClient) -> anyhow::Result<()> { - loop { - let now = chrono::Utc::now(); let mut blobs = blobs.list().await?; - println!("Blobs at {}:\n", now); + println!("Blobs:"); while let Some(info) = blobs.next().await { - println!(" {:?}", info?); + let info = info?; + println!(" {} {} bytes", info.hash, info.size); } println!(); tokio::time::sleep(Duration::from_secs(5)).await; } } -async fn delete_expired_tags_task(blobs: BlobsClient, prefix: &str, ) -> anyhow::Result<()> { +async fn delete_expired_tags_task(blobs: BlobsClient, prefix: &str) -> anyhow::Result<()> { loop { - delete_expired_tags(&blobs, prefix).await?; + delete_expired_tags(&blobs, prefix, false).await?; tokio::time::sleep(Duration::from_secs(5)).await; } } @@ -108,8 +126,7 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt::init(); let endpoint = endpoint::Endpoint::builder().bind().await?; let store = iroh_blobs::store::fs::Store::load("blobs").await?; - let blobs = iroh_blobs::net_protocol::Blobs::builder(store) - .build(&endpoint); + let blobs = iroh_blobs::net_protocol::Blobs::builder(store).build(&endpoint); // enable gc with a short period blobs.start_gc(GcConfig { period: Duration::from_secs(1), @@ -120,7 +137,8 @@ async fn main() -> anyhow::Result<()> { // You can skip this if you don't want to serve the data over the network. let router = iroh::protocol::Router::builder(endpoint) .accept(iroh_blobs::ALPN, blobs.clone()) - .spawn().await?; + .spawn() + .await?; // setup: add some data and tag it { @@ -128,28 +146,35 @@ async fn main() -> anyhow::Result<()> { let batch = blobs.client().batch().await?; let a = batch.add_bytes("blob 1".as_bytes()).await?; let b = batch.add_bytes("blob 2".as_bytes()).await?; - let expires_at = SystemTime::now().checked_add(Duration::from_secs(10)).unwrap(); - create_expiring_tag(blobs.client(), &[*a.hash(), *b.hash()], "expiring", expires_at).await?; + let expires_at = SystemTime::now() + .checked_add(Duration::from_secs(10)) + .unwrap(); + create_expiring_tag( + blobs.client(), + &[*a.hash(), *b.hash()], + "expiring", + expires_at, + ) + .await?; // add a single blob and tag it with an expiry date 60 seconds in the future let c = batch.add_bytes("blob 3".as_bytes()).await?; - let expires_at = SystemTime::now().checked_add(Duration::from_secs(60)).unwrap(); + let expires_at = SystemTime::now() + .checked_add(Duration::from_secs(60)) + .unwrap(); create_expiring_tag(blobs.client(), &[*c.hash()], "expiring", expires_at).await?; // batch goes out of scope, so data is only protected by the tags we created } let client = blobs.client().clone(); // delete expired tags every 5 seconds - let check_task = tokio::spawn(delete_expired_tags_task(client.clone(), "expiring")); - // print tags every 5 seconds - let print_tags_task = tokio::spawn(print_tags_task(client.clone())); - // print blobs every 5 seconds - let print_blobs_task = tokio::spawn(print_blobs_task(client)); + let delete_task = tokio::spawn(delete_expired_tags_task(client.clone(), "expiring")); + // print all tags and blobs every 5 seconds + let info_task = tokio::spawn(info_task(client.clone())); ctrl_c().await?; + delete_task.abort(); + info_task.abort(); router.shutdown().await?; - check_task.abort(); - print_tags_task.abort(); - print_blobs_task.abort(); Ok(()) -} \ No newline at end of file +} From ed89a3045f30faa0f69ebd487d7fa66b309b5c94 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 5 May 2025 15:11:43 +0300 Subject: [PATCH 3/7] fmt --- examples/expiring-tags.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/expiring-tags.rs b/examples/expiring-tags.rs index 9feb3e78..88d17b1f 100644 --- a/examples/expiring-tags.rs +++ b/examples/expiring-tags.rs @@ -97,7 +97,10 @@ async fn info_task(blobs: BlobsClient) -> anyhow::Result<()> { loop { let now = chrono::Utc::now(); let mut tags = blobs.tags().list().await?; - println!("Current time: {}", now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)); + println!( + "Current time: {}", + now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + ); println!("Tags:"); while let Some(tag) = tags.next().await { let tag = tag?; From b6380fbf912914fbf5a59bc1793bd46cf97d4202 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 5 May 2025 15:12:09 +0300 Subject: [PATCH 4/7] clippy --- examples/expiring-tags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/expiring-tags.rs b/examples/expiring-tags.rs index 88d17b1f..bda3b6bc 100644 --- a/examples/expiring-tags.rs +++ b/examples/expiring-tags.rs @@ -35,7 +35,7 @@ async fn create_expiring_tag( let hash = hashes[0]; batch.temp_tag(HashAndFormat::raw(hash)).await? } else { - let hs = hashes.into_iter().copied().collect::(); + let hs = hashes.iter().copied().collect::(); batch .add_bytes_with_opts(hs.into_inner(), BlobFormat::HashSeq) .await? From 3431efe005bc7b093fde0857662c66bce6a335a1 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Fri, 25 Jul 2025 12:35:55 +0300 Subject: [PATCH 5/7] clippy --- examples/expiring-tags.rs | 138 +++++++++++++++++++------------------- src/api/tags.rs | 11 ++- src/store/fs.rs | 10 ++- src/store/fs/meta.rs | 3 +- src/store/fs/options.rs | 3 +- 5 files changed, 91 insertions(+), 74 deletions(-) diff --git a/examples/expiring-tags.rs b/examples/expiring-tags.rs index bda3b6bc..3088ae1b 100644 --- a/examples/expiring-tags.rs +++ b/examples/expiring-tags.rs @@ -4,15 +4,21 @@ //! then encode the expiry date in the tag name after the prefix, in a format //! that sorts in the same order as the expiry date. //! -//! Then we can just use -use std::time::{Duration, SystemTime}; +//! The example creates a number of blobs and protects them directly or indirectly +//! with expiring tags. Watch as the expired tags are deleted and the blobs +//! are removed from the store. +use std::{ + ops::Deref, + time::{Duration, SystemTime}, +}; use chrono::Utc; use futures_lite::StreamExt; -use iroh::endpoint; use iroh_blobs::{ - hashseq::HashSeq, rpc::client::blobs::MemClient as BlobsClient, store::GcConfig, BlobFormat, - Hash, HashAndFormat, Tag, + api::{blobs::AddBytesOptions, Store, Tag}, + hashseq::HashSeq, + store::fs::options::{BatchOptions, GcConfig, InlineOptions, Options, PathOptions}, + BlobFormat, Hash, }; use tokio::signal::ctrl_c; @@ -20,34 +26,35 @@ use tokio::signal::ctrl_c; /// /// The tag name will be `prefix`- followed by the expiry date in iso8601 format (e.g. `expiry-2025-01-01T12:00:00Z`). async fn create_expiring_tag( - iroh: &BlobsClient, + store: &Store, hashes: &[Hash], prefix: &str, expiry: SystemTime, ) -> anyhow::Result<()> { let expiry = chrono::DateTime::::from(expiry); let expiry = expiry.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); - let tagname = format!("{}-{}", prefix, expiry); - let batch = iroh.batch().await?; - let tt = if hashes.is_empty() { + let tagname = format!("{prefix}-{expiry}"); + if hashes.is_empty() { return Ok(()); } else if hashes.len() == 1 { let hash = hashes[0]; - batch.temp_tag(HashAndFormat::raw(hash)).await? + store.tags().set(&tagname, hash).await?; } else { let hs = hashes.iter().copied().collect::(); - batch - .add_bytes_with_opts(hs.into_inner(), BlobFormat::HashSeq) - .await? + store + .add_bytes_with_opts(AddBytesOptions { + data: hs.into(), + format: BlobFormat::HashSeq, + }) + .with_named_tag(&tagname) + .await?; }; - batch.persist_to(tt, tagname.as_str().into()).await?; - println!("Created tag {}", tagname); + println!("Created tag {tagname}"); Ok(()) } -async fn delete_expired_tags(blobs: &BlobsClient, prefix: &str, bulk: bool) -> anyhow::Result<()> { - let mut tags = blobs.tags().list().await?; - let prefix = format!("{}-", prefix); +async fn delete_expired_tags(blobs: &Store, prefix: &str, bulk: bool) -> anyhow::Result<()> { + let prefix = format!("{prefix}-"); let now = chrono::Utc::now(); let end = format!( "{}-{}", @@ -66,6 +73,7 @@ async fn delete_expired_tags(blobs: &BlobsClient, prefix: &str, bulk: bool) -> a // find tags to delete one by one and then delete them // // this allows us to print the tags before deleting them + let mut tags = blobs.tags().list().await?; let mut to_delete = Vec::new(); while let Some(tag) = tags.next().await { let tag = tag?.name; @@ -85,41 +93,45 @@ async fn delete_expired_tags(blobs: &BlobsClient, prefix: &str, bulk: bool) -> a } } for tag in to_delete { - println!("Deleting expired tag {}", tag); + println!("Deleting expired tag {tag}\n"); blobs.tags().delete(tag).await?; } } Ok(()) } -async fn info_task(blobs: BlobsClient) -> anyhow::Result<()> { +async fn print_store_info(store: &Store) -> anyhow::Result<()> { + let now = chrono::Utc::now(); + let mut tags = store.tags().list().await?; + println!( + "Current time: {}", + now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + ); + println!("Tags:"); + while let Some(tag) = tags.next().await { + let tag = tag?; + println!(" {tag:?}"); + } + let mut blobs = store.list().stream().await?; + println!("Blobs:"); + while let Some(item) = blobs.next().await { + println!(" {}", item?); + } + println!(); + Ok(()) +} + +async fn info_task(store: Store) -> anyhow::Result<()> { tokio::time::sleep(Duration::from_secs(1)).await; loop { - let now = chrono::Utc::now(); - let mut tags = blobs.tags().list().await?; - println!( - "Current time: {}", - now.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) - ); - println!("Tags:"); - while let Some(tag) = tags.next().await { - let tag = tag?; - println!(" {:?}", tag); - } - let mut blobs = blobs.list().await?; - println!("Blobs:"); - while let Some(info) = blobs.next().await { - let info = info?; - println!(" {} {} bytes", info.hash, info.size); - } - println!(); + print_store_info(&store).await?; tokio::time::sleep(Duration::from_secs(5)).await; } } -async fn delete_expired_tags_task(blobs: BlobsClient, prefix: &str) -> anyhow::Result<()> { +async fn delete_expired_tags_task(store: Store, prefix: &str) -> anyhow::Result<()> { loop { - delete_expired_tags(&blobs, prefix, false).await?; + delete_expired_tags(&store, prefix, false).await?; tokio::time::sleep(Duration::from_secs(5)).await; } } @@ -127,57 +139,47 @@ async fn delete_expired_tags_task(blobs: BlobsClient, prefix: &str) -> anyhow::R #[tokio::main] async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt::init(); - let endpoint = endpoint::Endpoint::builder().bind().await?; - let store = iroh_blobs::store::fs::Store::load("blobs").await?; - let blobs = iroh_blobs::net_protocol::Blobs::builder(store).build(&endpoint); - // enable gc with a short period - blobs.start_gc(GcConfig { - period: Duration::from_secs(1), - done_callback: None, - })?; - // create a router and add blobs as a service - // - // You can skip this if you don't want to serve the data over the network. - let router = iroh::protocol::Router::builder(endpoint) - .accept(iroh_blobs::ALPN, blobs.clone()) - .spawn() - .await?; + let path = std::env::current_dir()?.join("blobs"); + let options = Options { + path: PathOptions::new(&path), + gc: Some(GcConfig { + interval: Duration::from_secs(10), + }), + inline: InlineOptions::default(), + batch: BatchOptions::default(), + }; + let store = + iroh_blobs::store::fs::FsStore::load_with_opts(path.join("blobs.db"), options).await?; // setup: add some data and tag it { // add several blobs and tag them with an expiry date 10 seconds in the future - let batch = blobs.client().batch().await?; + let batch = store.batch().await?; let a = batch.add_bytes("blob 1".as_bytes()).await?; let b = batch.add_bytes("blob 2".as_bytes()).await?; + let expires_at = SystemTime::now() .checked_add(Duration::from_secs(10)) .unwrap(); - create_expiring_tag( - blobs.client(), - &[*a.hash(), *b.hash()], - "expiring", - expires_at, - ) - .await?; + create_expiring_tag(&store, &[*a.hash(), *b.hash()], "expiring", expires_at).await?; // add a single blob and tag it with an expiry date 60 seconds in the future let c = batch.add_bytes("blob 3".as_bytes()).await?; let expires_at = SystemTime::now() .checked_add(Duration::from_secs(60)) .unwrap(); - create_expiring_tag(blobs.client(), &[*c.hash()], "expiring", expires_at).await?; + create_expiring_tag(&store, &[*c.hash()], "expiring", expires_at).await?; // batch goes out of scope, so data is only protected by the tags we created } - let client = blobs.client().clone(); // delete expired tags every 5 seconds - let delete_task = tokio::spawn(delete_expired_tags_task(client.clone(), "expiring")); + let delete_task = tokio::spawn(delete_expired_tags_task(store.deref().clone(), "expiring")); // print all tags and blobs every 5 seconds - let info_task = tokio::spawn(info_task(client.clone())); + let info_task = tokio::spawn(info_task(store.deref().clone())); ctrl_c().await?; delete_task.abort(); info_task.abort(); - router.shutdown().await?; + store.shutdown().await?; Ok(()) } diff --git a/src/api/tags.rs b/src/api/tags.rs index b235a8c6..d92a6207 100644 --- a/src/api/tags.rs +++ b/src/api/tags.rs @@ -58,17 +58,22 @@ impl Tags { Ok(stream.next().await.transpose()?) } - pub async fn set_with_opts(&self, options: SetOptions) -> super::RequestResult<()> { + pub async fn set_with_opts(&self, options: SetOptions) -> super::RequestResult { trace!("{:?}", options); + let info = TagInfo { + name: options.name.clone(), + hash: options.value.hash, + format: options.value.format, + }; self.client.rpc(options).await??; - Ok(()) + Ok(info) } pub async fn set( &self, name: impl AsRef<[u8]>, value: impl Into, - ) -> super::RequestResult<()> { + ) -> super::RequestResult { self.set_with_opts(SetOptions { name: Tag::from(name.as_ref()), value: value.into(), diff --git a/src/store/fs.rs b/src/store/fs.rs index b0c1eb60..d5278dbf 100644 --- a/src/store/fs.rs +++ b/src/store/fs.rs @@ -693,7 +693,14 @@ impl Actor { fs::create_dir_all(db_path.parent().unwrap())?; let (db_send, db_recv) = tokio::sync::mpsc::channel(100); let (protect, ds) = delete_set::pair(Arc::new(options.path.clone())); - let db_actor = meta::Actor::new(db_path, db_recv, ds, options.batch.clone())?; + let db_actor = meta::Actor::new(db_path, db_recv, ds, options.batch.clone()); + let db_actor = match db_actor { + Ok(actor) => actor, + Err(err) => { + println!("failed to create meta actor: {err}"); + return Err(err); + } + }; let slot_context = Arc::new(TaskContext { options: options.clone(), db: meta::Db::new(db_send), @@ -1209,6 +1216,7 @@ impl FsStore { let (commands_tx, commands_rx) = tokio::sync::mpsc::channel(100); let (fs_commands_tx, fs_commands_rx) = tokio::sync::mpsc::channel(100); let gc_config = options.gc.clone(); + println!("Creating actor"); let actor = handle .spawn(Actor::new( db_path, diff --git a/src/store/fs/meta.rs b/src/store/fs/meta.rs index 617db98c..31ba6c86 100644 --- a/src/store/fs/meta.rs +++ b/src/store/fs/meta.rs @@ -409,7 +409,8 @@ impl Actor { options: BatchOptions, ) -> anyhow::Result { debug!("creating or opening meta database at {}", db_path.display()); - let db = match redb::Database::create(db_path) { + let res = redb::Database::create(&db_path); + let db = match res { Ok(db) => db, Err(DatabaseError::UpgradeRequired(1)) => { return Err(anyhow::anyhow!("migration from v1 no longer supported")); diff --git a/src/store/fs/options.rs b/src/store/fs/options.rs index 6e123b75..8b2f89ad 100644 --- a/src/store/fs/options.rs +++ b/src/store/fs/options.rs @@ -4,7 +4,8 @@ use std::{ time::Duration, }; -use super::{gc::GcConfig, meta::raw_outboard_size, temp_name}; +pub use super::gc::GcConfig; +use super::{meta::raw_outboard_size, temp_name}; use crate::Hash; /// Options for directories used by the file store. From 166e384255acc05853c50422a1eb5a89a2196d04 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Fri, 25 Jul 2025 12:40:24 +0300 Subject: [PATCH 6/7] remove println --- src/store/fs.rs | 10 +--------- src/store/fs/meta.rs | 3 +-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/store/fs.rs b/src/store/fs.rs index d5278dbf..b0c1eb60 100644 --- a/src/store/fs.rs +++ b/src/store/fs.rs @@ -693,14 +693,7 @@ impl Actor { fs::create_dir_all(db_path.parent().unwrap())?; let (db_send, db_recv) = tokio::sync::mpsc::channel(100); let (protect, ds) = delete_set::pair(Arc::new(options.path.clone())); - let db_actor = meta::Actor::new(db_path, db_recv, ds, options.batch.clone()); - let db_actor = match db_actor { - Ok(actor) => actor, - Err(err) => { - println!("failed to create meta actor: {err}"); - return Err(err); - } - }; + let db_actor = meta::Actor::new(db_path, db_recv, ds, options.batch.clone())?; let slot_context = Arc::new(TaskContext { options: options.clone(), db: meta::Db::new(db_send), @@ -1216,7 +1209,6 @@ impl FsStore { let (commands_tx, commands_rx) = tokio::sync::mpsc::channel(100); let (fs_commands_tx, fs_commands_rx) = tokio::sync::mpsc::channel(100); let gc_config = options.gc.clone(); - println!("Creating actor"); let actor = handle .spawn(Actor::new( db_path, diff --git a/src/store/fs/meta.rs b/src/store/fs/meta.rs index 31ba6c86..e9643d95 100644 --- a/src/store/fs/meta.rs +++ b/src/store/fs/meta.rs @@ -409,8 +409,7 @@ impl Actor { options: BatchOptions, ) -> anyhow::Result { debug!("creating or opening meta database at {}", db_path.display()); - let res = redb::Database::create(&db_path); - let db = match res { + let db = match redb::Database::create(&db_path) { Ok(db) => db, Err(DatabaseError::UpgradeRequired(1)) => { return Err(anyhow::anyhow!("migration from v1 no longer supported")); From 8b31cdd97574e16e5fc9423422dc788b0517c339 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Fri, 25 Jul 2025 12:41:42 +0300 Subject: [PATCH 7/7] remove debugging changes --- src/api/tags.rs | 11 +++-------- src/store/fs/meta.rs | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/api/tags.rs b/src/api/tags.rs index d92a6207..b235a8c6 100644 --- a/src/api/tags.rs +++ b/src/api/tags.rs @@ -58,22 +58,17 @@ impl Tags { Ok(stream.next().await.transpose()?) } - pub async fn set_with_opts(&self, options: SetOptions) -> super::RequestResult { + pub async fn set_with_opts(&self, options: SetOptions) -> super::RequestResult<()> { trace!("{:?}", options); - let info = TagInfo { - name: options.name.clone(), - hash: options.value.hash, - format: options.value.format, - }; self.client.rpc(options).await??; - Ok(info) + Ok(()) } pub async fn set( &self, name: impl AsRef<[u8]>, value: impl Into, - ) -> super::RequestResult { + ) -> super::RequestResult<()> { self.set_with_opts(SetOptions { name: Tag::from(name.as_ref()), value: value.into(), diff --git a/src/store/fs/meta.rs b/src/store/fs/meta.rs index e9643d95..617db98c 100644 --- a/src/store/fs/meta.rs +++ b/src/store/fs/meta.rs @@ -409,7 +409,7 @@ impl Actor { options: BatchOptions, ) -> anyhow::Result { debug!("creating or opening meta database at {}", db_path.display()); - let db = match redb::Database::create(&db_path) { + let db = match redb::Database::create(db_path) { Ok(db) => db, Err(DatabaseError::UpgradeRequired(1)) => { return Err(anyhow::anyhow!("migration from v1 no longer supported"));