IndexedDB: add functionality for cleaning up MediaStore

crates/matrix-sdk-indexeddb/Cargo.toml

@@ -27,6 +27,7 @@ experimental-encrypted-state-events = [
 [dependencies]
 async-trait.workspace = true
 base64.workspace = true
+futures-util.workspace = true
 gloo-utils = { version = "0.2.0", features = ["serde"] }
 growable-bloom-filter = { workspace = true, optional = true }
 hkdf.workspace = true

crates/matrix-sdk-indexeddb/src/media_store/builder.rs

@@ -14,7 +14,7 @@
 
 use std::{rc::Rc, sync::Arc};
 
-use matrix_sdk_base::media::store::{MediaService, MemoryMediaStore};
+use matrix_sdk_base::media::store::MediaService;
 use matrix_sdk_store_encryption::StoreCipher;
 
 use crate::{
@@ -67,7 +67,6 @@ impl IndexeddbMediaStoreBuilder {
             inner: Rc::new(open_and_upgrade_db(&self.database_name).await?),
             serializer: IndexedTypeSerializer::new(SafeEncodeSerializer::new(self.store_cipher)),
             media_service: MediaService::new(),
-            memory_store: MemoryMediaStore::new(),
         })
     }
 }

crates/matrix-sdk-indexeddb/src/media_store/error.rs

@@ -28,6 +28,8 @@ pub enum IndexeddbMediaStoreError {
     Transaction(#[from] TransactionError),
     #[error("DomException {name} ({code}): {message}")]
     DomException { name: String, message: String, code: u16 },
+    #[error("cache size too big, cannot exceed 'usize::MAX' ({})", usize::MAX)]
+    CacheSizeTooBig,
 }
 
 impl From<IndexeddbMediaStoreError> for MediaStoreError {
@@ -38,6 +40,7 @@ impl From<IndexeddbMediaStoreError> for MediaStoreError {
             UnableToOpenDatabase(e) => GenericError::from(e).into(),
             DomException { .. } => Self::InvalidData { details: value.to_string() },
             Transaction(inner) => inner.into(),
+            CacheSizeTooBig => GenericError::from(value.to_string()).into(),
             MemoryStore(error) => error,
         }
     }

crates/matrix-sdk-indexeddb/src/media_store/migrations.rs

@@ -113,6 +113,7 @@ pub mod v1 {
         pub const LEASES: &str = "leases";
         pub const LEASES_KEY_PATH: &str = "id";
         pub const MEDIA_RETENTION_POLICY_KEY: &str = "media_retention_policy";
+        pub const MEDIA_CLEANUP_TIME_KEY: &str = "media_cleanup_time";
         pub const MEDIA: &str = "media";
         pub const MEDIA_KEY_PATH: &str = "id";
         pub const MEDIA_URI: &str = "media_uri";

crates/matrix-sdk-indexeddb/src/media_store/mod.rs

@@ -28,25 +28,26 @@ use std::{rc::Rc, time::Duration};
 
 pub use builder::IndexeddbMediaStoreBuilder;
 pub use error::IndexeddbMediaStoreError;
-use indexed_db_futures::{database::Database, Build};
+use indexed_db_futures::{
+    cursor::CursorDirection, database::Database, transaction::TransactionMode, Build,
+};
 use matrix_sdk_base::{
     media::{
         store::{
             IgnoreMediaRetentionPolicy, MediaRetentionPolicy, MediaService, MediaStore,
-            MediaStoreInner, MemoryMediaStore,
+            MediaStoreInner,
         },
         MediaRequestParameters,
     },
     timer,
 };
 use ruma::{time::SystemTime, MilliSecondsSinceUnixEpoch, MxcUri};
 use tracing::instrument;
-use web_sys::IdbTransactionMode;
 
 use crate::{
     media_store::{
         transaction::IndexeddbMediaStoreTransaction,
-        types::{Lease, Media, MediaMetadata},
+        types::{Lease, Media, MediaCleanupTime, MediaMetadata, UnixTime},
     },
     serializer::{Indexed, IndexedTypeSerializer},
     transaction::TransactionError,
@@ -66,12 +67,6 @@ pub struct IndexeddbMediaStore {
     // A service for conveniently delegating media-related queries to an `MediaStoreInner`
     // implementation
     media_service: MediaService,
-    // An in-memory store for providing temporary implementations for
-    // functions of `MediaStore`.
-    //
-    // NOTE: This will be removed once we have IndexedDB-backed implementations for all
-    // functions in `MediaStore`.
-    memory_store: MemoryMediaStore,
 }
 
 impl IndexeddbMediaStore {
@@ -87,7 +82,7 @@ impl IndexeddbMediaStore {
     pub fn transaction<'a>(
         &'a self,
         stores: &[&str],
-        mode: IdbTransactionMode,
+        mode: TransactionMode,
     ) -> Result<IndexeddbMediaStoreTransaction<'a>, IndexeddbMediaStoreError> {
         Ok(IndexeddbMediaStoreTransaction::new(
             self.inner
@@ -116,8 +111,7 @@ impl MediaStore for IndexeddbMediaStore {
 
         let now = Duration::from_millis(MilliSecondsSinceUnixEpoch::now().get().into());
 
-        let transaction =
-            self.transaction(&[Lease::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Lease::OBJECT_STORE], TransactionMode::Readwrite)?;
 
         if let Some(lease) = transaction.get_lease_by_id(key).await? {
             if lease.holder != holder && !lease.has_expired(now) {
@@ -155,8 +149,7 @@ impl MediaStore for IndexeddbMediaStore {
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
 
-        let transaction =
-            self.transaction(&[Media::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
         if let Some(mut media) = transaction.get_media_by_id(from).await? {
             // delete before adding, in case `from` and `to` generate the same key
             transaction.delete_media_by_id(from).await?;
@@ -183,8 +176,7 @@ impl MediaStore for IndexeddbMediaStore {
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
 
-        let transaction =
-            self.transaction(&[Media::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
         transaction.delete_media_by_id(request).await?;
         transaction.commit().await.map_err(Into::into)
     }
@@ -205,8 +197,7 @@ impl MediaStore for IndexeddbMediaStore {
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
 
-        let transaction =
-            self.transaction(&[Media::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
         transaction.delete_media_by_uri(uri).await?;
         transaction.commit().await.map_err(Into::into)
     }
@@ -253,7 +244,7 @@ impl MediaStoreInner for IndexeddbMediaStore {
         &self,
     ) -> Result<Option<MediaRetentionPolicy>, IndexeddbMediaStoreError> {
         let _timer = timer!("method");
-        self.transaction(&[MediaRetentionPolicy::OBJECT_STORE], IdbTransactionMode::Readonly)?
+        self.transaction(&[MediaRetentionPolicy::OBJECT_STORE], TransactionMode::Readonly)?
             .get_media_retention_policy()
             .await
             .map_err(Into::into)
@@ -265,10 +256,11 @@ impl MediaStoreInner for IndexeddbMediaStore {
         policy: MediaRetentionPolicy,
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
-        self.transaction(&[MediaRetentionPolicy::OBJECT_STORE], IdbTransactionMode::Readwrite)?
-            .put_item(&policy)
-            .await
-            .map_err(Into::into)
+
+        let transaction =
+            self.transaction(&[MediaRetentionPolicy::OBJECT_STORE], TransactionMode::Readwrite)?;
+        transaction.put_item(&policy).await?;
+        transaction.commit().await.map_err(Into::into)
     }
 
     #[instrument(skip_all)]
@@ -282,8 +274,7 @@ impl MediaStoreInner for IndexeddbMediaStore {
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
 
-        let transaction =
-            self.transaction(&[Media::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
 
         let media = Media {
             metadata: MediaMetadata {
@@ -305,10 +296,16 @@ impl MediaStoreInner for IndexeddbMediaStore {
         ignore_policy: IgnoreMediaRetentionPolicy,
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
-        self.memory_store
-            .set_ignore_media_retention_policy_inner(request, ignore_policy)
-            .await
-            .map_err(IndexeddbMediaStoreError::MemoryStore)
+
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
+        if let Some(mut media) = transaction.get_media_by_id(request).await? {
+            if media.metadata.ignore_policy != ignore_policy {
+                media.metadata.ignore_policy = ignore_policy;
+                transaction.put_media(&media).await?;
+                transaction.commit().await?;
+            }
+        }
+        Ok(())
     }
 
     #[instrument(skip_all)]
@@ -319,8 +316,7 @@ impl MediaStoreInner for IndexeddbMediaStore {
     ) -> Result<Option<Vec<u8>>, IndexeddbMediaStoreError> {
         let _timer = timer!("method");
 
-        let transaction =
-            self.transaction(&[Media::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
         let media = transaction.access_media_by_id(request, current_time).await?;
         transaction.commit().await?;
         Ok(media.map(|m| m.content))
@@ -334,8 +330,7 @@ impl MediaStoreInner for IndexeddbMediaStore {
     ) -> Result<Option<Vec<u8>>, IndexeddbMediaStoreError> {
         let _timer = timer!("method");
 
-        let transaction =
-            self.transaction(&[Media::OBJECT_STORE], IdbTransactionMode::Readwrite)?;
+        let transaction = self.transaction(&[Media::OBJECT_STORE], TransactionMode::Readwrite)?;
         let media = transaction.access_media_by_uri(uri, current_time).await?.pop();
         transaction.commit().await?;
         Ok(media.map(|m| m.content))
@@ -348,29 +343,83 @@ impl MediaStoreInner for IndexeddbMediaStore {
         current_time: SystemTime,
     ) -> Result<(), IndexeddbMediaStoreError> {
         let _timer = timer!("method");
-        self.memory_store
-            .clean_inner(policy, current_time)
-            .await
-            .map_err(IndexeddbMediaStoreError::MemoryStore)
+
+        if !policy.has_limitations() {
+            return Ok(());
+        }
+
+        let transaction = self.transaction(
+            &[Media::OBJECT_STORE, MediaCleanupTime::OBJECT_STORE],
+            TransactionMode::Readwrite,
+        )?;
+
+        let ignore_policy = IgnoreMediaRetentionPolicy::No;
+        let current_time = UnixTime::from(current_time);
+
+        if let Some(max_file_size) = policy.computed_max_file_size() {
+            transaction
+                .delete_media_by_content_size_greater_than(ignore_policy, max_file_size as usize)
+                .await?;
+        }
+
+        if let Some(expiry) = policy.last_access_expiry {
+            transaction
+                .delete_media_by_last_access_earlier_than(ignore_policy, current_time - expiry)
+                .await?;
+        }
+
+        if let Some(max_cache_size) = policy.max_cache_size {
+            let cache_size = transaction
+                .get_cache_size(ignore_policy)
+                .await?
+                .ok_or(Self::Error::CacheSizeTooBig)?;
+            if cache_size > (max_cache_size as usize) {
+                let (_, upper_key) = transaction
+                    .fold_media_keys_by_retention_metadata_while(
+                        CursorDirection::Prev,
+                        ignore_policy,
+                        0usize,
+                        |total, key| match total.checked_add(key.content_size()) {
+                            None => None,
+                            Some(total) if total > max_cache_size as usize => None,
+                            Some(total) => Some(total),
+                        },
+                    )
+                    .await?;
+                if let Some(upper_key) = upper_key {
+                    transaction
+                        .delete_media_by_retention_metadata_to(
+                            upper_key.ignore_policy(),
+                            upper_key.last_access(),
+                            upper_key.content_size(),
+                        )
+                        .await?;
+                }
+            }
+        }
+
+        transaction.put_media_cleanup_time(current_time).await?;
+        transaction.commit().await.map_err(Into::into)
     }
 
     #[instrument(skip_all)]
     async fn last_media_cleanup_time_inner(
         &self,
     ) -> Result<Option<SystemTime>, IndexeddbMediaStoreError> {
         let _timer = timer!("method");
-        self.memory_store
-            .last_media_cleanup_time_inner()
-            .await
-            .map_err(IndexeddbMediaStoreError::MemoryStore)
+        let time = self
+            .transaction(&[MediaCleanupTime::OBJECT_STORE], TransactionMode::Readonly)?
+            .get_media_cleanup_time()
+            .await?;
+        Ok(time.map(Into::into))
     }
 }
 
 #[cfg(all(test, target_family = "wasm"))]
 mod tests {
     use matrix_sdk_base::{
-        media::store::MediaStoreError, media_store_integration_tests,
-        media_store_integration_tests_time,
+        media::store::MediaStoreError, media_store_inner_integration_tests,
+        media_store_integration_tests, media_store_integration_tests_time,
     };
     use uuid::Uuid;
 
@@ -391,6 +440,9 @@ mod tests {
 
         #[cfg(target_family = "wasm")]
         media_store_integration_tests_time!();
+
+        #[cfg(target_family = "wasm")]
+        media_store_inner_integration_tests!(with_media_size_tests);
     }
 
     mod encrypted {
@@ -408,5 +460,8 @@ mod tests {
 
         #[cfg(target_family = "wasm")]
         media_store_integration_tests_time!();
+
+        #[cfg(target_family = "wasm")]
+        media_store_inner_integration_tests!();
     }
 }