Add nix derivation source-origins command

doc/manual/meson.build

@@ -271,6 +271,7 @@ nix3_manpages = [
   'nix3-derivation',
   'nix3-derivation-add',
   'nix3-derivation-show',
+  'nix3-derivation-source-origins',
   'nix3-develop',
   'nix3-edit',
   'nix3-env',

doc/manual/rl-next/derivation-source-origins.md

@@ -0,0 +1,9 @@
+---
+synopsis: "New command `nix derivation source-origins`"
+---
+
+The new command [`nix derivation source-origins`](@docroot@/command-ref/new-cli/nix3-derivation-source-origins.md) evaluates the given installables and prints, for every derivation in their build closure, a JSON mapping from each `inputSrcs` store path back to the filesystem path it was copied from during evaluation.
+This covers plain path references (`src = ./.`), paths inside local flakes (`path:` and `git+file:` inputs, resolved to the original directory rather than `/nix/store/...-source`), and filtered sources created with `builtins.path`, `builtins.filterSource` or `lib.cleanSourceWith`.
+For directory sources, the individual files in the store path are listed as `sourceFiles`, which gives file-level precision even for filtered sources with a broad root.
+
+This is useful for tooling that needs to know which parts of a working tree (e.g. of a monorepo) contribute to a given build.

src/libexpr/eval.cc

@@ -317,6 +317,8 @@ EvalState::EvalState(
     , debugRepl(nullptr)
     , debugStop(false)
     , trylevel(0)
+    , storeToSrc(make_ref<decltype(storeToSrc)::element_type>())
+    , sourceStoreToOriginalPath(make_ref<decltype(sourceStoreToOriginalPath)::element_type>())
     , importResolutionCache(make_ref<decltype(importResolutionCache)::element_type>())
     , fileEvalCache(make_ref<decltype(fileEvalCache)::element_type>())
     , positionToDocComment(make_ref<decltype(positionToDocComment)::element_type>())
@@ -2647,11 +2649,95 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
         nullptr,
         repair);
     allowPath(dstPath);
+    recordPathOrigin(dstPath, path);
 
     context.insert(NixStringContextElem::Opaque{.path = dstPath});
     return dstPath;
 }
 
+std::optional<SourcePath> EvalState::getSourceOrigin(const StorePath & storePath) const
+{
+    return getConcurrent(*storeToSrc, storePath);
+}
+
+std::map<StorePath, SourcePath> EvalState::getSourceOrigins() const
+{
+    std::map<StorePath, SourcePath> result;
+    storeToSrc->cvisit_all([&](const auto & entry) { result.emplace(entry.first, entry.second); });
+    return result;
+}
+
+std::optional<std::filesystem::path> EvalState::getOriginalPath(const StorePath & storePath) const
+{
+    return getConcurrent(*sourceStoreToOriginalPath, storePath);
+}
+
+void EvalState::recordPathOrigin(const StorePath & storePath, const SourcePath & srcPath)
+{
+    /* This runs on every copyPathToStore(), so bail out cheaply if we
+       have already recorded this store path. */
+    if (!storeToSrc->try_emplace(storePath, srcPath))
+        return;
+
+    /* Try to resolve the original filesystem path right away, so that
+       `getOriginalPath()` can answer for this store path without
+       having to trace through accessor chains later. */
+
+    auto appendRel = [](const std::filesystem::path & root, const CanonPath & rel) {
+        return rel.isRoot() ? root : root / std::string(rel.rel());
+    };
+
+    /* Strategy 1: the source path refers to a store path within
+       `rootFS`/`storeFS` (e.g. `/nix/store/xxx-source/sub`, as is the
+       case for anything in a flake). If we know where that store path
+       came from, the original path is just the corresponding
+       subpath. */
+    if (srcPath.accessor == rootFS && store->isInStore(srcPath.path.abs())) {
+        auto [srcStorePath, rel] = store->toStorePath(srcPath.path.abs());
+        if (auto origRoot = getConcurrent(*sourceStoreToOriginalPath, srcStorePath)) {
+            sourceStoreToOriginalPath->try_emplace(storePath, appendRel(*origRoot, rel));
+            return;
+        }
+        /* A store path we know nothing about; don't fall through to
+           the `rootFS` physical path, which would just be the store
+           path itself. */
+        return;
+    }
+
+    /* Strategy 2: the accessor itself knows its original root
+       (per-input accessors created by the `git` and `path` input
+       schemes). */
+    if (srcPath.accessor->originalRootPath) {
+        sourceStoreToOriginalPath->try_emplace(storePath, appendRel(*srcPath.accessor->originalRootPath, srcPath.path));
+        return;
+    }
+
+    /* Strategy 3: the accessor is one that has been mounted in
+       `storeFS` by `mountInput()` and registered in
+       `sourceStoreToOriginalPath`. Find it by identity. Snapshot the
+       map first since we must not insert while iterating over a
+       concurrent map. */
+    if (srcPath.accessor != rootFS) {
+        std::vector<std::pair<StorePath, std::filesystem::path>> snapshot;
+        sourceStoreToOriginalPath->cvisit_all(
+            [&](const auto & entry) { snapshot.emplace_back(entry.first, entry.second); });
+
+        for (auto & [srcStorePath, origRoot] : snapshot) {
+            auto mount = storeFS->getMount(CanonPath(store->printStorePath(srcStorePath)));
+            if (mount && mount.get() == &*srcPath.accessor) {
+                sourceStoreToOriginalPath->try_emplace(storePath, appendRel(origRoot, srcPath.path));
+                return;
+            }
+        }
+        return;
+    }
+
+    /* Strategy 4: a plain path in the ambient filesystem (e.g. `nix
+       derivation source-origins -f ./foo.nix`). */
+    if (auto physical = srcPath.getPhysicalPath())
+        sourceStoreToOriginalPath->try_emplace(storePath, *physical);
+}
+
 SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
 {
     return peelToStringOutPath(

src/libexpr/include/nix/expr/eval.hh

@@ -498,6 +498,24 @@ public:
 
 private:
 
+    /**
+     * Reverse mapping from store paths back to the source paths they
+     * were copied from during this evaluation. Populated by
+     * `copyPathToStore()` and `recordPathOrigin()` so that provenance
+     * can be recovered afterwards (see `nix derivation
+     * source-origins`).
+     */
+    const ref<boost::concurrent_flat_map<StorePath, SourcePath>> storeToSrc;
+
+    /**
+     * Mapping from source store paths (e.g. `/nix/store/xxx-source`)
+     * to the original filesystem paths they were obtained from (e.g.
+     * `/home/user/project`). Populated by `mountInput()` for inputs
+     * whose accessor has an `originalRootPath` (such as `path:` and
+     * `git+file:` inputs) and by `recordPathOrigin()`.
+     */
+    const ref<boost::concurrent_flat_map<StorePath, std::filesystem::path>> sourceStoreToOriginalPath;
+
     /**
      * A cache that maps paths to "resolved" paths for importing Nix
      * expressions, i.e. `/foo` to `/foo/default.nix`.
@@ -614,6 +632,36 @@ public:
      */
     void allowAndSetStorePathString(const StorePath & storePath, Value & v);
 
+    /**
+     * Look up the source path from which a store path was copied
+     * during this evaluation. Returns `std::nullopt` when the store
+     * path wasn't produced by this evaluator (e.g. it came from a
+     * substituter or a previous evaluation).
+     */
+    std::optional<SourcePath> getSourceOrigin(const StorePath & storePath) const;
+
+    /**
+     * Return the full store path → source path mapping built during
+     * this evaluation.
+     */
+    std::map<StorePath, SourcePath> getSourceOrigins() const;
+
+    /**
+     * Look up the original filesystem path for a store path that was
+     * obtained from a local filesystem location (e.g. a `path:` flake
+     * input, or a filtered source thereof). Returns `std::nullopt` if
+     * no such mapping is known.
+     */
+    std::optional<std::filesystem::path> getOriginalPath(const StorePath & storePath) const;
+
+    /**
+     * Record that `storePath` was produced from `srcPath`, and try to
+     * resolve `srcPath` to an original filesystem path. Used by
+     * `addPath()` (`builtins.path` / `builtins.filterSource`) so that
+     * filtered sources also appear in the provenance maps.
+     */
+    void recordPathOrigin(const StorePath & storePath, const SourcePath & srcPath);
+
     void checkURI(const std::string & uri);
 
     /**

src/libexpr/paths.cc

@@ -3,6 +3,8 @@
 #include "nix/util/mounted-source-accessor.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 
+#include <boost/unordered/concurrent_flat_map.hpp>
+
 namespace nix {
 
 SourcePath EvalState::rootPath(CanonPath path)
@@ -76,6 +78,21 @@ EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalI
 
     storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
 
+    /* Record where this source tree came from in the local filesystem
+       (if anywhere) so that `nix derivation source-origins` can map
+       store paths derived from it back to their original location.
+       Prefer the accessor's `originalRootPath` (set by the `git` and
+       `path` input schemes to the root of the source tree) over
+       `input.getSourcePath()`, which for a git-tracked flake in a
+       subdirectory would be the flake directory rather than the repo
+       root. */
+    if (accessor->originalRootPath)
+        sourceStoreToOriginalPath->try_emplace(storePath, *accessor->originalRootPath);
+    else if (auto origPath = input.getSourcePath(); origPath && !store->isInStore(origPath->string())) {
+        accessor->originalRootPath = *origPath;
+        sourceStoreToOriginalPath->try_emplace(storePath, *origPath);
+    }
+
     input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
     if (originalInput.getNarHash() && narHash != *originalInput.getNarHash())

src/libexpr/primops.cc

@@ -2959,9 +2959,16 @@ static void addPath(
                 state.error<EvalError>("store path mismatch in (possibly filtered) path added from '%s'", path)
                     .atPos(noPos)
                     .debugThrow();
+            /* Record provenance so that `nix derivation source-origins`
+               can trace filtered paths (`builtins.path`,
+               `builtins.filterSource`, `cleanSourceWith`) back to their
+               original source location. */
+            state.recordPathOrigin(dstPath, path);
             state.allowAndSetStorePathString(dstPath, v);
-        } else
+        } else {
+            state.recordPathOrigin(*expectedStorePath, path);
             state.allowAndSetStorePathString(*expectedStorePath, v);
+        }
     } catch (Error & e) {
         e.addTrace(nullptr, "while adding path '%s'", path);
         throw;

src/libfetchers/git.cc

@@ -1001,6 +1001,11 @@ struct GitInputScheme : InputScheme
         ref<SourceAccessor> accessor =
             repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath));
 
+        /* Remember the repo root so that store paths derived from this
+           accessor can be mapped back to the working tree (see `nix
+           derivation source-origins`). */
+        accessor->originalRootPath = repoPath;
+
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and, per
            submodule, either its workdir accessor or an empty directory
@@ -1038,6 +1043,8 @@ struct GitInputScheme : InputScheme
 
             mounts.insert_or_assign(CanonPath::root, accessor);
             accessor = makeMountedSourceAccessor(std::move(mounts));
+            /* The submodule workdirs live under the same root. */
+            accessor->originalRootPath = repoPath;
         }
 
         if (!repoInfo.workdirInfo.isDirty) {

src/libfetchers/path.cc

@@ -163,6 +163,14 @@ struct PathInputScheme : InputScheme
 
         auto accessor = store.requireStoreObjectAccessor(*storePath);
 
+        // Remember where the contents came from so that store paths
+        // derived from this accessor can be mapped back to the
+        // original filesystem location (see `nix derivation
+        // source-origins`). Don't do this if the input *is* a store
+        // path already, since that tells us nothing about provenance.
+        if (!store.isInStore(absPath.string()))
+            accessor->originalRootPath = absPath;
+
         // To prevent `fetchToStore()` copying the path again to Nix
         // store, pre-create an entry in the fetcher cache.
         auto narHash = store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true);

src/libutil/include/nix/util/source-accessor.hh

@@ -207,6 +207,15 @@ public:
      */
     CanonPath resolveSymlinks(const CanonPath & path, SymlinkResolution mode = SymlinkResolution::Full);
 
+    /**
+     * For accessors whose contents were obtained from a local
+     * filesystem path (e.g. `path:` or `git+file:` flake inputs), this
+     * records the original filesystem root so that store paths derived
+     * from this accessor can be mapped back to their original
+     * locations (see `nix derivation source-origins`).
+     */
+    std::optional<std::filesystem::path> originalRootPath;
+
     /**
      * A string that uniquely represents the contents of this
      * accessor. This is used for caching lookups (see `fetchToStore()`).