Merge pull request #13223 from NixOS/remove-global-fetcher-cache

Mic92 · web-flow · commit f2671886f5ca · 2025-05-18T15:58:53.000+02:00
Remove global fetcher cache
diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
@@ -34,7 +34,12 @@ EvalSettings evalSettings {
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
                 auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
-                auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+                auto storePath = nix::fetchToStore(
+                    state.fetchSettings,
+                    *state.store,
+                    SourcePath(accessor),
+                    FetchMode::Copy,
+                    lockedRef.input.getName());
                 state.allowPath(storePath);
                 return state.storePath(storePath);
             },
@@ -177,15 +182,24 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
             state.store,
             state.fetchSettings,
             EvalSettings::resolvePseudoUrl(s));
-        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
+        auto storePath = fetchToStore(
+            state.fetchSettings,
+            *state.store,
+            SourcePath(accessor),
+            FetchMode::Copy);
         return state.storePath(storePath);
     }
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
         auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
-        auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
+        auto storePath = nix::fetchToStore(
+            state.fetchSettings,
+            *state.store,
+            SourcePath(accessor),
+            FetchMode::Copy,
+            lockedRef.input.getName());
         state.allowPath(storePath);
         return state.storePath(storePath);
     }
diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc
@@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
+        auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
@@ -2423,6 +2423,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
         ? *dstPathCached
         : [&]() {
             auto dstPath = fetchToStore(
+                fetchSettings,
                 *store,
                 path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
@@ -3125,7 +3126,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
                 store,
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
-            auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
+            auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
             return finish(this->storePath(storePath));
         } catch (Error & e) {
             logWarning({
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
@@ -2545,6 +2545,7 @@ static void addPath(
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
             auto dstPath = fetchToStore(
+                state.fetchSettings,
                 *state.store,
                 path.resolveSymlinks(),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
@@ -537,11 +537,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     auto storePath =
         unpack
         ? fetchToStore(
+            state.fetchSettings,
             *state.store,
             fetchers::downloadTarball(state.store, state.fetchSettings, *url),
             FetchMode::Copy,
             name)
-        : fetchers::downloadFile(state.store, *url, name).storePath;
+        : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack
diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
@@ -1,4 +1,5 @@
 #include "nix/fetchers/cache.hh"
+#include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/users.hh"
 #include "nix/store/sqlite.hh"
 #include "nix/util/sync.hh"
@@ -162,10 +163,12 @@ struct CacheImpl : Cache
     }
 };
 
-ref<Cache> getCache()
+ref<Cache> Settings::getCache() const
 {
-    static auto cache = std::make_shared<CacheImpl>();
-    return ref<Cache>(cache);
+    auto cache(_cache.lock());
+    if (!*cache)
+        *cache = std::make_shared<CacheImpl>();
+    return ref<Cache>(*cache);
 }
 
 }
diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc
@@ -1,13 +1,14 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/fetchers.hh"
+#include "nix/fetchers/fetch-settings.hh"
 
 namespace nix {
 
 fetchers::Cache::Key makeFetchToStoreCacheKey(
-    const std::string &name,
-    const std::string &fingerprint,
+    const std::string & name,
+    const std::string & fingerprint,
     ContentAddressMethod method,
-    const std::string &path)
+    const std::string & path)
 {
     return fetchers::Cache::Key{"fetchToStore", {
         {"name", name},
@@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey(
 }
 
 StorePath fetchToStore(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,
@@ -34,7 +36,7 @@ StorePath fetchToStore(
 
     if (!filter && path.accessor->fingerprint) {
         cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
-        if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
+        if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
             debug("store path cache hit for '%s'", path);
             return res->storePath;
         }
@@ -56,7 +58,7 @@ StorePath fetchToStore(
     debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
 
     if (cacheKey && mode == FetchMode::Copy)
-        fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
+        settings.getCache()->upsert(*cacheKey, store, {}, storePath);
 
     return storePath;
 }
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -198,7 +198,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
         try {
             auto [accessor, result] = getAccessorUnchecked(store);
 
-            auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
+            auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
 
             auto narHash = store->queryPathInfo(storePath)->narHash;
             result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
@@ -1,6 +1,7 @@
 #include "nix/fetchers/git-utils.hh"
 #include "nix/fetchers/git-lfs-fetch.hh"
 #include "nix/fetchers/cache.hh"
+#include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/finally.hh"
 #include "nix/util/processes.hh"
 #include "nix/util/signals.hh"
@@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
     }
 
-    Hash treeHashToNarHash(const Hash & treeHash) override
+    Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
         auto accessor = getAccessor(treeHash, false, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
-        if (auto res = fetchers::getCache()->lookup(cacheKey))
+        if (auto res = settings.getCache()->lookup(cacheKey))
             return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
 
         auto narHash = accessor->hashPath(CanonPath::root);
 
-        fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
+        settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
 
         return narHash;
     }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
@@ -480,11 +480,11 @@ struct GitInputScheme : InputScheme
         return repoInfo;
     }
 
-    uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
+    uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
     {
         Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
 
-        auto cache = getCache();
+        auto cache = settings.getCache();
 
         if (auto res = cache->lookup(key))
             return getIntAttr(*res, "lastModified");
@@ -496,11 +496,11 @@ struct GitInputScheme : InputScheme
         return lastModified;
     }
 
-    uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
+    uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
     {
         Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
 
-        auto cache = getCache();
+        auto cache = settings.getCache();
 
         if (auto revCountAttrs = cache->lookup(key))
             return getIntAttr(*revCountAttrs, "revCount");
@@ -678,12 +678,12 @@ struct GitInputScheme : InputScheme
 
         Attrs infoAttrs({
             {"rev", rev.gitRev()},
-            {"lastModified", getLastModified(repoInfo, repoDir, rev)},
+            {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
         });
 
         if (!getShallowAttr(input))
             infoAttrs.insert_or_assign("revCount",
-                getRevCount(repoInfo, repoDir, rev));
+                getRevCount(*input.settings, repoInfo, repoDir, rev));
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
 
@@ -799,7 +799,7 @@ struct GitInputScheme : InputScheme
 
             input.attrs.insert_or_assign("rev", rev.gitRev());
             input.attrs.insert_or_assign("revCount",
-                rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev));
+                rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
 
             verifyCommit(input, repo);
         } else {
@@ -818,7 +818,7 @@ struct GitInputScheme : InputScheme
         input.attrs.insert_or_assign(
             "lastModified",
             repoInfo.workdirInfo.headRev
-            ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
+            ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
             : 0);
 
         return {accessor, std::move(input)};
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
@@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.erase("ref");
         input.attrs.insert_or_assign("rev", rev->gitRev());
 
-        auto cache = getCache();
+        auto cache = input.settings->getCache();
 
         Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
         Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@@ -407,7 +407,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", headers).storePath)));
+                    downloadFile(store, *input.settings, url, "source", headers).storePath)));
 
         return RefInfo {
             .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
@@ -481,7 +481,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", headers).storePath)));
+                    downloadFile(store, *input.settings, url, "source", headers).storePath)));
 
         if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
           return RefInfo {
@@ -551,7 +551,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::string refUri;
         if (ref == "HEAD") {
             auto file = store->toRealPath(
-                downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath);
+                downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
             std::ifstream is(file);
             std::string line;
             getline(is, line);
@@ -567,7 +567,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::regex refRegex(refUri);
 
         auto file = store->toRealPath(
-            downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath);
+            downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
         std::ifstream is(file);
 
         std::string line;
diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh
@@ -91,6 +91,4 @@ struct Cache
         Store & store) = 0;
 };
 
-ref<Cache> getCache();
-
 }
diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh
@@ -3,6 +3,8 @@
 
 #include "nix/util/types.hh"
 #include "nix/util/configuration.hh"
+#include "nix/util/ref.hh"
+#include "nix/util/sync.hh"
 
 #include <map>
 #include <limits>
@@ -11,6 +13,8 @@
 
 namespace nix::fetchers {
 
+struct Cache;
+
 struct Settings : public Config
 {
     Settings();
@@ -110,6 +114,11 @@ struct Settings : public Config
           When empty, disables the global flake registry.
         )",
         {}, true, Xp::Flakes};
+
+    ref<Cache> getCache() const;
+
+private:
+    mutable Sync<std::shared_ptr<Cache>> _cache;
 };
 
 }
diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh
@@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy };
  * Copy the `path` to the Nix store.
  */
 StorePath fetchToStore(
+    const fetchers::Settings & settings,
     Store & store,
     const SourcePath & path,
     FetchMode mode,
diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-namespace fetchers { struct PublicKey; }
+namespace fetchers { struct PublicKey; struct Settings; }
 
 /**
  * A sink that writes into a Git repository. Note that nothing may be written
@@ -115,7 +115,7 @@ struct GitRepo
      * Given a Git tree hash, compute the hash of its NAR
      * serialisation. This is memoised on-disk.
      */
-    virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
+    virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0;
 
     /**
      * If the specified Git object is a directory with a single entry
diff --git a/src/libfetchers/include/nix/fetchers/tarball.hh b/src/libfetchers/include/nix/fetchers/tarball.hh
@@ -26,6 +26,7 @@ struct DownloadFileResult
 
 DownloadFileResult downloadFile(
     ref<Store> store,
+    const Settings & settings,
     const std::string & url,
     const std::string & name,
     const Headers & headers = {});
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc
diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
diff --git a/src/nix/flake.cc b/src/nix/flake.cc

Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)`
`45`	`45`	`std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)`
`46`	`46`	`{`
`47`	`47`	`if (v.type() == nPath) {`
`48`		`- auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);`
	`48`	`+ auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);`
`49`	`49`	`return {{`
`50`	`50`	`.path = DerivedPath::Opaque {`
`51`	`51`	`.path = std::move(storePath),`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`#include "nix/fetchers/cache.hh"`
	`2`	`+#include "nix/fetchers/fetch-settings.hh"`
`2`	`3`	`#include "nix/util/users.hh"`
`3`	`4`	`#include "nix/store/sqlite.hh"`
`4`	`5`	`#include "nix/util/sync.hh"`
`@@ -162,10 +163,12 @@ struct CacheImpl : Cache`
`162`	`163`	`}`
`163`	`164`	`};`
`164`	`165`
`165`		`-ref<Cache> getCache()`
	`166`	`+ref<Cache> Settings::getCache() const`
`166`	`167`	`{`
`167`		`- static auto cache = std::make_shared<CacheImpl>();`
`168`		`- return ref<Cache>(cache);`
	`168`	`+ auto cache(_cache.lock());`
	`169`	`+ if (!*cache)`
	`170`	`+ *cache = std::make_shared<CacheImpl>();`
	`171`	`+ return ref<Cache>(*cache);`
`169`	`172`	`}`
`170`	`173`
`171`	`174`	`}`