fix(run): clear filesystem caches in between pixi run steps (#4523)

Co-authored-by: nichmor <[email protected]>

Author: baszalmstra

crates/pixi_cli/src/run.rs

@@ -266,6 +266,9 @@ pub async fn execute(args: Args) -> miette::Result<()> {
                 // Clear the current progress reports.
                 lock_file.command_dispatcher.clear_reporter().await;
 
+                // Clear caches based on the filesystem. The tasks might change files on disk.
+                lock_file.command_dispatcher.clear_filesystem_caches().await;
+
                 let command_env = get_task_env(
                     &executable_task.run_environment,
                     args.clean_env || executable_task.task().clean_env(),

crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs

@@ -236,6 +236,8 @@ pub(crate) enum ForegroundMessage {
     InstallPixiEnvironment(InstallPixiEnvironmentTask),
     InstantiateToolEnvironment(Task<InstantiateToolEnvironmentSpec>),
     ClearReporter(oneshot::Sender<()>),
+    #[from(ignore)]
+    ClearFilesystemCaches(oneshot::Sender<()>),
 }
 
 /// A message that is send to the background task to start solving a particular
@@ -361,6 +363,21 @@ impl CommandDispatcher {
         &self.data.discovery_cache
     }
 
+    /// Clears in-memory caches whose correctness depends on the filesystem.
+    ///
+    /// This invalidates memoized results that are derived from files on disk so
+    /// subsequent operations re-check the current state of the filesystem. It:
+    /// - clears glob hash memoization (`GlobHashCache`) used for input file hashing
+    /// - clears memoized SourceBuildCacheStatus results held by the processor,
+    ///   while preserving any in-flight queries
+    pub async fn clear_filesystem_caches(&self) {
+        if let Some(sender) = self.channel().sender() {
+            let (tx, rx) = oneshot::channel();
+            let _ = sender.send(ForegroundMessage::ClearFilesystemCaches(tx));
+            let _ = rx.await;
+        }
+    }
+
     /// Returns the download client used by the command dispatcher.
     pub fn download_client(&self) -> &ClientWithMiddleware {
         &self.data.download_client

crates/pixi_command_dispatcher/src/command_dispatcher_processor/mod.rs

@@ -391,6 +391,9 @@ impl CommandDispatcherProcessor {
                 self.on_source_build_cache_status(task)
             }
             ForegroundMessage::ClearReporter(sender) => self.clear_reporter(sender),
+            ForegroundMessage::ClearFilesystemCaches(sender) => {
+                self.clear_filesystem_caches(sender)
+            }
             ForegroundMessage::SourceMetadata(task) => self.on_source_metadata(task),
             ForegroundMessage::BackendSourceBuild(task) => self.on_backend_source_build(task),
         }
@@ -556,6 +559,17 @@ impl CommandDispatcherProcessor {
         let _ = sender.send(());
     }
 
+    /// Clears cached results based on the filesystem, preserving in-flight tasks.
+    fn clear_filesystem_caches(&mut self, sender: oneshot::Sender<()>) {
+        self.inner.glob_hash_cache.clear();
+
+        // Clear source build cache status, preserving in-flight tasks.
+        self.source_build_cache_status
+            .retain(|_, v| matches!(v, PendingDeduplicatingTask::Pending(_, _)));
+
+        let _ = sender.send(());
+    }
+
     /// Returns true if by following the parent chain of the `parent` context we
     /// stumble on `id`.
     pub fn contains_cycle<T: TryFrom<CommandDispatcherContext> + PartialEq>(

crates/pixi_command_dispatcher/tests/integration/main.rs

@@ -4,6 +4,7 @@ mod event_tree;
 use std::{
     collections::HashMap,
     path::{Path, PathBuf},
+    // ptr,
     str::FromStr,
 };
 
@@ -14,14 +15,17 @@ use pixi_build_frontend::{
 };
 use pixi_command_dispatcher::{
     BuildEnvironment, CacheDirs, CommandDispatcher, Executor, InstallPixiEnvironmentSpec,
-    InstantiateToolEnvironmentSpec, PixiEnvironmentSpec,
+    InstantiateToolEnvironmentSpec, PackageIdentifier, PixiEnvironmentSpec,
+    SourceBuildCacheStatusSpec,
 };
 use pixi_config::default_channel_config;
+use pixi_record::PinnedPathSpec;
 use pixi_spec::{GitReference, GitSpec, PathSpec, PixiSpec};
 use pixi_spec_containers::DependencyMap;
 use pixi_test_utils::format_diagnostic;
 use rattler_conda_types::{
-    GenericVirtualPackage, PackageName, Platform, VersionSpec, prefix::Prefix,
+    ChannelUrl, GenericVirtualPackage, PackageName, Platform, VersionSpec, VersionWithSource,
+    prefix::Prefix,
 };
 use rattler_virtual_packages::{VirtualPackageOverrides, VirtualPackages};
 use url::Url;
@@ -533,3 +537,76 @@ pub async fn instantiate_backend_with_from_source() {
 
     insta::assert_debug_snapshot!(err);
 }
+
+#[tokio::test]
+async fn source_build_cache_status_clear_works() {
+    let tmp_dir = tempfile::tempdir().unwrap();
+
+    let dispatcher = CommandDispatcher::builder()
+        .with_cache_dirs(CacheDirs::new(tmp_dir.path().to_path_buf()))
+        .finish();
+
+    let host = Platform::current();
+    let build_env = BuildEnvironment {
+        host_platform: host,
+        build_platform: host,
+        build_virtual_packages: vec![],
+        host_virtual_packages: vec![],
+    };
+
+    let pkg = PackageIdentifier {
+        name: PackageName::try_from("dummy-pkg").unwrap(),
+        version: VersionWithSource::from_str("0.0.0").unwrap(),
+        build: "0".to_string(),
+        subdir: host.to_string(),
+    };
+
+    let spec = SourceBuildCacheStatusSpec {
+        package: pkg,
+        source: PinnedPathSpec {
+            path: tmp_dir.path().to_string_lossy().into_owned().into(),
+        }
+        .into(),
+        channels: Vec::<ChannelUrl>::new(),
+        build_environment: build_env,
+        channel_config: default_channel_config(),
+        enabled_protocols: Default::default(),
+    };
+
+    let first = dispatcher
+        .source_build_cache_status(spec.clone())
+        .await
+        .expect("query succeeds");
+
+    // Create a weak reference to track that the original Arc is dropped
+    // after clearing the cache
+    let weak_first = std::sync::Arc::downgrade(&first);
+
+    let second = dispatcher
+        .source_build_cache_status(spec.clone())
+        .await
+        .expect("query succeeds");
+
+    // Cached result should return the same Arc
+    assert!(std::sync::Arc::ptr_eq(&first, &second));
+
+    // now drop the cached entries to release the Arc
+    // which will unlock the fd locks that we hold on the cache files
+    drop(first);
+    drop(second);
+
+    // Clear and expect a fresh Arc on next query
+    dispatcher.clear_filesystem_caches().await;
+
+    let _third = dispatcher
+        .source_build_cache_status(spec)
+        .await
+        .expect("query succeeds");
+
+    // Check if the original Arc is truly gone
+    // and we have a fresh one
+    assert!(
+        weak_first.upgrade().is_none(),
+        "Original Arc should be deallocated after cache clear"
+    );
+}

crates/pixi_glob/src/glob_hash_cache.rs

@@ -129,4 +129,9 @@ impl GlobHashCache {
             }
         }
     }
+
+    /// Clears all memoized glob hashes. In-flight computations are unaffected.
+    pub fn clear(&self) {
+        self.cache.clear();
+    }
 }