diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 821c9e0565cd..ca2fcf842e20 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -94,7 +94,7 @@ ref StoreCommand::getStore() ref StoreCommand::createStore() { - auto store = getStoreConfig()->openStore(); + auto store = getStoreConfig()->openStore(SecretContext{}); store->init(); return store; } @@ -167,7 +167,12 @@ ref EvalCommand::getEvalState() { if (!evalState) { evalState = std::allocate_shared( - traceable_allocator(), lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()); + traceable_allocator(), + lookupPath, + getEvalStore(), + fetchers::FetchContext{fetchSettings, {}}, + evalSettings, + getStore()); evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index bf24d53accea..96b4eaa63977 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -33,7 +33,7 @@ EvalSettings evalSettings{ auto flakeRef = parseFlakeRef(rest, {}, true, false); debug("fetching flake search path element '%s''", rest); auto [accessor, lockedRef] = - flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store); + flakeRef.resolve(state.fetchContext, *state.store).lazyFetch(state.fetchContext, *state.store); auto storePath = nix::fetchToStore( fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); state.allowPath(storePath); @@ -177,7 +177,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::files const auto & fetchSettings = state.fetchSettings; if (EvalSettings::isPseudoUrl(s)) { - auto accessor = fetchers::downloadTarball(*state.store, fetchSettings, EvalSettings::resolvePseudoUrl(s)); + auto accessor = fetchers::downloadTarball(*state.store, state.fetchContext, EvalSettings::resolvePseudoUrl(s)); auto storePath = fetchToStore(fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy); return state.storePath(storePath); } @@ -186,7 +186,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::files experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = - flakeRef.resolve(state.fetchSettings, *state.store).lazyFetch(state.fetchSettings, *state.store); + flakeRef.resolve(state.fetchContext, *state.store).lazyFetch(state.fetchContext, *state.store); auto storePath = nix::fetchToStore( fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); state.allowPath(storePath); diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ac8e3b483252..31c27cff96be 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -146,7 +146,7 @@ std::pair InstallableFlake::toValue(EvalState & state, AutoCall return {&getCursor(state, autoCall)->forceValue(), noPos}; } -std::vector> InstallableFlake::getCursors(EvalState & state, AutoCall) +std::vector> InstallableFlake::getCursors(EvalState & state, AutoCall autoCall) { auto evalCache = openEvalCache(state, getLockedFlake()); @@ -163,7 +163,7 @@ std::vector> InstallableFlake::getCursors(EvalState try { auto attr = root->findAlongAttrPath(AttrPath::parse(state, attrPath)); if (attr) { - res.push_back(ref(*attr)); + res.push_back(autoCall == AutoCall::Yes ? (*attr)->autoCall() : ref(*attr)); } else { suggestions += attr.getSuggestions(); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f6e0b1b3811b..2702126c6168 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -411,7 +411,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(fetchSettings, *store)) { + for (auto & registry : fetchers::getRegistries(fetchers::FetchContext{fetchSettings, {}}, *store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 9c6bcefc31e2..b424b25ea9be 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -171,8 +171,8 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder try { auto fetchSettings = std::make_unique(std::move(builder->fetchSettings)); auto settings = std::make_unique(std::move(builder->settings)); - auto ownedState = - std::make_shared(builder->lookupPath, builder->store, *fetchSettings, *settings); + auto ownedState = std::make_shared( + builder->lookupPath, builder->store, nix::fetchers::FetchContext{*fetchSettings, {}}, *settings); auto & stateRef = *ownedState; void * p = ::operator new(sizeof(EvalState), static_cast(alignof(EvalState))); return new (p) EvalState{stateRef, std::move(fetchSettings), std::move(settings), std::move(ownedState)}; diff --git a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh index 658a6ffe0a35..19533a463f37 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh @@ -29,7 +29,7 @@ protected: LibExprTest(ref store, auto && makeEvalSettings) : LibStoreTest() , evalSettings(makeEvalSettings(readOnlyMode)) - , state({}, store, fetchSettings, evalSettings, nullptr) + , state({}, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings, nullptr) { } diff --git a/src/libexpr-tests/dynamic-attrs-bench.cc b/src/libexpr-tests/dynamic-attrs-bench.cc index ea0736564f59..c404a325de7f 100644 --- a/src/libexpr-tests/dynamic-attrs-bench.cc +++ b/src/libexpr-tests/dynamic-attrs-bench.cc @@ -37,7 +37,8 @@ static void BM_EvalDynamicAttrs(benchmark::State & state) EvalSettings evalSettings{readOnlyMode}; evalSettings.nixPath = {}; - auto stPtr = std::make_shared(LookupPath{}, store, fetchSettings, evalSettings, nullptr); + auto stPtr = std::make_shared( + LookupPath{}, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings, nullptr); auto & st = *stPtr; Expr * expr = st.parseExprFromString(exprStr, st.rootPath(CanonPath::root)); diff --git a/src/libexpr-tests/get-drvs-bench.cc b/src/libexpr-tests/get-drvs-bench.cc index e47d85193930..29cc621c163e 100644 --- a/src/libexpr-tests/get-drvs-bench.cc +++ b/src/libexpr-tests/get-drvs-bench.cc @@ -27,7 +27,9 @@ struct GetDerivationsEnv settings.nixPath = {}; return settings; }()) - , statePtr(std::make_shared(LookupPath{}, store, fetchSettings, evalSettings, nullptr)) + , statePtr( + std::make_shared( + LookupPath{}, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings, nullptr)) , state(*statePtr) { autoArgs = state.buildBindings(0).finish(); diff --git a/src/libexpr-tests/regex-cache-bench.cc b/src/libexpr-tests/regex-cache-bench.cc index c0c61e58e993..e8479b7bc2af 100644 --- a/src/libexpr-tests/regex-cache-bench.cc +++ b/src/libexpr-tests/regex-cache-bench.cc @@ -27,7 +27,8 @@ static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) EvalSettings evalSettings{readOnlyMode}; evalSettings.nixPath = {}; - auto stPtr = std::make_shared(LookupPath{}, store, fetchSettings, evalSettings, nullptr); + auto stPtr = std::make_shared( + LookupPath{}, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings, nullptr); auto & st = *stPtr; Expr * expr = st.parseExprFromString(std::string(exprStr), st.rootPath(CanonPath::root)); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7422cd095bc6..c7f9fcd2fc6d 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -38,10 +38,11 @@ static const char * schema = R"sql( create table if not exists Attributes ( parent integer not null, name text, + kind integer not null, type integer not null, value text, context text, - primary key (parent, name) + primary key (parent, name, kind) ); )sql"; @@ -72,7 +73,7 @@ struct AttrDb { auto state(_state->lock()); - auto cacheDir = getCacheDir() / "eval-cache-v6"; + auto cacheDir = getCacheDir() / "eval-cache-v7"; createDirs(cacheDir); auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"); @@ -82,15 +83,16 @@ struct AttrDb state->db.exec(schema); state->insertAttribute.create( - state->db, "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + state->db, "insert or replace into Attributes(parent, name, kind, type, value) values (?, ?, ?, ?, ?)"); state->insertAttributeWithContext.create( - state->db, "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); + state->db, + "insert or replace into Attributes(parent, name, kind, type, value, context) values (?, ?, ?, ?, ?, ?)"); state->queryAttribute.create( - state->db, "select rowid, type, value, context from Attributes where parent = ? and name = ?"); + state->db, "select rowid, type, value, context from Attributes where parent = ? and name = ? and kind = ?"); - state->queryAttributes.create(state->db, "select name from Attributes where parent = ?"); + state->queryAttributes.create(state->db, "select name from Attributes where parent = ? and kind = 0"); state->txn = std::make_unique(state->db); } @@ -127,8 +129,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::FullAttrs) .apply(0, false) .exec(); @@ -140,6 +143,7 @@ struct AttrDb state->insertAttribute.use() .apply(rowId) .apply(symbols[attr]) + .apply(static_cast(AttrKeyKind::Attribute)) .apply(AttrType::Placeholder) .apply(0, false) .exec(); @@ -163,16 +167,18 @@ struct AttrDb first = false; } state->insertAttributeWithContext.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::String) .apply(s) .apply(ctx) .exec(); } else { state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::String) .apply(s) .exec(); @@ -188,8 +194,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::Bool) .apply(b ? 1 : 0) .exec(); @@ -204,8 +211,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::Int) .apply(n) .exec(); @@ -220,8 +228,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::ListOfStrings) .apply(dropEmptyInitThenConcatStringsSep("\t", l)) .exec(); @@ -236,8 +245,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::Placeholder) .apply(0, false) .exec(); @@ -252,8 +262,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::Missing) .apply(0, false) .exec(); @@ -268,8 +279,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::Misc) .apply(0, false) .exec(); @@ -284,8 +296,9 @@ struct AttrDb auto state(_state->lock()); state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) + .apply(key.parent) + .apply(symbols[key.name]) + .apply(static_cast(key.kind)) .apply(AttrType::Failed) .apply(0, false) .exec(); @@ -298,7 +311,8 @@ struct AttrDb { auto state(_state->lock()); - auto queryAttribute(state->queryAttribute.use().apply(key.first).apply(symbols[key.second])); + auto queryAttribute( + state->queryAttribute.use().apply(key.parent).apply(symbols[key.name]).apply(static_cast(key.kind))); if (!queryAttribute.next()) return {}; @@ -374,10 +388,15 @@ ref EvalCache::getRoot() } AttrCursor::AttrCursor( - ref root, Parent parent, Value * value, std::optional> && cachedValue) + ref root, + Parent parent, + Value * value, + std::optional> && cachedValue, + bool autoCalled) : root(root) , parent(parent) , cachedValue(std::move(cachedValue)) + , autoCalled(autoCalled) { if (value) _value = allocRootValue(value); @@ -391,13 +410,22 @@ AttrKey AttrCursor::getKey() parent->first->cachedValue = root->db->getAttr(parent->first->getKey()); assert(parent->first->cachedValue); } - return {parent->first->cachedValue->first, parent->second}; + return { + parent->first->cachedValue->first, + parent->second, + autoCalled ? AttrKeyKind::AutoCall : AttrKeyKind::Attribute, + }; } Value & AttrCursor::getValue() { if (!_value) { - if (parent) { + if (autoCalled) { + assert(parent); + auto * result = root->state.allocValue(); + root->state.autoCallFunction(Bindings::emptyBindings, parent->first->forceValue(), *result); + _value = allocRootValue(result); + } else if (parent) { auto & vParent = parent->first->getValue(); root->state.forceAttrs(vParent, noPos, "while searching for an attribute"); auto attr = vParent.attrs()->get(parent->second); @@ -422,7 +450,10 @@ AttrPath AttrCursor::getAttrPath() const { if (parent) { auto attrPath = parent->first->getAttrPath(); - attrPath.push_back(parent->second); + /* The auto-call slot is an implementation detail, so it doesn't + show up in the user-visible attribute path. */ + if (!autoCalled) + attrPath.push_back(parent->second); return attrPath; } else return {}; @@ -455,7 +486,10 @@ Value & AttrCursor::forceValue() root->state.forceValue(v, noPos); } catch (EvalError &) { debug("setting '%s' to failed", getAttrPathStr()); - if (root->db) + /* An auto-call is not an attribute of its parent, so + `CachedEvalError::force()` could not reproduce the original + error from a cached failure. Just don't cache it. */ + if (root->db && !autoCalled) cachedValue = {root->db->setFailed(getKey()), failed_t()}; throw; } @@ -757,6 +791,29 @@ bool AttrCursor::isDerivation() return aType && aType->getString() == "derivation"; } +ref AttrCursor::autoCall() +{ + /* The auto-called value generally differs from the one at this + attribute path, so it gets its own slot in the evaluation cache, + in a namespace separate from real Nix attributes. */ + auto name = root->state.s.epsilon; + + std::optional> cachedValue2; + + if (root->db) { + fetchCachedValue(); + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + AttrKey key{cachedValue->first, name, AttrKeyKind::AutoCall}; + cachedValue2 = root->db->getAttr(key); + if (!cachedValue2) + cachedValue2 = {root->db->setPlaceholder(key), placeholder_t()}; + } + + return make_ref( + root, std::make_pair(ref(shared_from_this()), name), nullptr, std::move(cachedValue2), true); +} + StorePath AttrCursor::forceDerivation() { auto aDrvPath = getAttr(root->state.s.drvPath); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dcf41d713f9d..ced7ae587178 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -244,10 +244,11 @@ EvalMemory::EvalMemory() EvalState::EvalState( const LookupPath & lookupPathFromArguments, ref store, - const fetchers::Settings & fetchSettings, + const fetchers::FetchContext & fetchContext, const EvalSettings & settings, std::shared_ptr buildStore) - : fetchSettings{fetchSettings} + : fetchContext{fetchContext} + , fetchSettings{this->fetchContext.settings} , settings{settings} , symbols(StaticEvalSymbols::staticSymbolTable()) , repair(NoRepair) @@ -3369,7 +3370,7 @@ EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAcces if (EvalSettings::isPseudoUrl(value)) { try { - auto accessor = fetchers::downloadTarball(*store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); + auto accessor = fetchers::downloadTarball(*store, fetchContext, EvalSettings::resolvePseudoUrl(value)); auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); return finish(this->storePath(storePath)); } catch (Error & e) { diff --git a/src/libexpr/include/nix/expr/eval-cache.hh b/src/libexpr/include/nix/expr/eval-cache.hh index 0feb7d5649ff..bd4721cab4f6 100644 --- a/src/libexpr/include/nix/expr/eval-cache.hh +++ b/src/libexpr/include/nix/expr/eval-cache.hh @@ -82,7 +82,19 @@ struct int_t }; typedef uint64_t AttrId; -typedef std::pair AttrKey; + +enum class AttrKeyKind { + Attribute = 0, + AutoCall = 1, +}; + +struct AttrKey +{ + AttrId parent; + Symbol name; + AttrKeyKind kind = AttrKeyKind::Attribute; +}; + typedef std::pair string_t; typedef std::variant< @@ -108,6 +120,12 @@ class AttrCursor : public std::enable_shared_from_this RootValue _value; std::optional> cachedValue; + /** + * Whether this cursor holds the result of auto-calling its parent + * (see `autoCall()`) rather than one of its parent's attributes. + */ + bool autoCalled = false; + AttrKey getKey(); Value & getValue(); @@ -126,7 +144,8 @@ public: ref root, Parent parent, Value * value = nullptr, - std::optional> && cachedValue = {}); + std::optional> && cachedValue = {}, + bool autoCalled = false); AttrPath getAttrPath() const; @@ -168,6 +187,18 @@ public: Value & forceValue(); + /** + * Return a cursor for this value, auto-called with no automatic + * arguments if it is a function (see `EvalState::autoCallFunction()`). + * The returned cursor keeps this cursor's attribute path, but gets its + * own slot in the evaluation cache, since auto-calling generally + * yields a value other than the one at that path. + * + * The call happens lazily, when the cursor's value is first needed, so + * cached attributes are still served without evaluating anything. + */ + ref autoCall(); + /** * Force creation of the .drv file in the Nix store. */ diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index a421a680b90a..122a1c874559 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -17,6 +17,7 @@ #include "nix/expr/repl-exit-status.hh" #include "nix/util/ref.hh" #include "nix/expr/counter.hh" +#include "nix/fetchers/fetch-settings.hh" // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS` #include "nix/expr/config.hh" @@ -41,7 +42,6 @@ constexpr size_t maxPrimOpArity = 8; class Store; namespace fetchers { -struct Settings; struct InputCache; struct Input; } // namespace fetchers @@ -399,6 +399,7 @@ class EvalState : public std::enable_shared_from_this public: static constexpr StaticEvalSymbols s = StaticEvalSymbols::create(); + const fetchers::FetchContext fetchContext; const fetchers::Settings & fetchSettings; const EvalSettings & settings; @@ -544,14 +545,14 @@ public: /** * @param lookupPath Only used during construction. * @param store The store to use for instantiation - * @param fetchSettings Must outlive the lifetime of this EvalState! + * @param fetchContext Its Settings reference must outlive this EvalState. * @param settings Must outlive the lifetime of this EvalState! * @param buildStore The store to use for builds ("import from derivation", C API `nix_string_realise`) */ EvalState( const LookupPath & lookupPath, ref store, - const fetchers::Settings & fetchSettings, + const fetchers::FetchContext & fetchContext, const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index e3f8a3abd5bb..f997b2a16fe4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -196,7 +196,9 @@ static void prim_fetchClosure(EvalState & state, CallSite callSite, Value * cons if (!storeRef.params.empty()) throw Error({.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl)}); - auto fromStore = openStore(std::move(storeRef)); + /* Only http(s) stores get here, so propagate the evaluator's credential + authority to the store that performs the transfer. */ + auto fromStore = openStore(SecretContext{.secretResolver = state.fetchContext.secretResolver}, std::move(storeRef)); if (toPath) runFetchClosureWithRewrite(state, *fromStore, *fromPath, *toPath, v); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index f785acba52a6..92635a544523 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, CallSite callSite, Value * co attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(std::move(attrs)); - auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store); + auto [storePath, input2] = input.fetchToStore(state.fetchContext, *state.store); auto attrs2 = state.buildBindings(8); state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath)); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f3c301f20898..b84fd811c213 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -301,7 +301,7 @@ static void fetchTree( } if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) - input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first; + input = lookupInRegistries(state.fetchContext, *state.store, input, fetchers::UseRegistries::Limited).first; if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) { if (input.getNarHash()) @@ -327,7 +327,7 @@ static void fetchTree( } auto cachedInput = - state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No); + state.inputCache->getAccessor(state.fetchContext, *state.store, input, fetchers::UseRegistries::No); auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); @@ -587,11 +587,11 @@ fetch(EvalState & state, Value * const * args, Value & v, const std::string & wh attrs.emplace("narHash", expectedHash->to_string(HashFormat::SRI, true)); auto input = fetchers::Input::fromAttrs(std::move(attrs)); auto cachedInput = - state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No); + state.inputCache->getAccessor(state.fetchContext, *state.store, input, fetchers::UseRegistries::No); auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); state.mkStorePathString(storePath, v); } else { - auto storePath = fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; + auto storePath = fetchers::downloadFile(*state.store, state.fetchContext, *url, name).storePath; if (expectedHash) { auto hash = hashPath( {state.store->requireStoreObjectAccessor(storePath)}, diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 0614873fd281..7300cba2a69f 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -27,7 +27,7 @@ TEST_F(AccessKeysTest, singleOrgGitHub) fetchSettings.accessTokens.get().insert({"github.com/a", "token"}); auto i = Input::fromURL("github:a/b"); - auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); + auto token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "github.com", "github.com/a/b"); ASSERT_EQ(token, "token"); } @@ -37,7 +37,7 @@ TEST_F(AccessKeysTest, nonMatches) fetchSettings.accessTokens.get().insert({"github.com", "token"}); auto i = Input::fromURL("gitlab:github.com/evil"); - auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/github.com/evil"); + auto token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "gitlab.com", "gitlab.com/github.com/evil"); ASSERT_EQ(token, std::nullopt); } @@ -47,7 +47,7 @@ TEST_F(AccessKeysTest, noPartialMatches) fetchSettings.accessTokens.get().insert({"github.com/partial", "token"}); auto i = Input::fromURL("github:partial-match/repo"); - auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/partial-match"); + auto token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "github.com", "github.com/partial-match"); ASSERT_EQ(token, std::nullopt); } @@ -59,13 +59,13 @@ TEST_F(AccessKeysTest, repoGitHub) fetchSettings.accessTokens.get().insert({"github.com/a/c", "yet_another_token"}); auto i = Input::fromURL("github:a/a"); - auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/a"); + auto token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "github.com", "github.com/a/a"); ASSERT_EQ(token, "token"); - token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); + token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "github.com", "github.com/a/b"); ASSERT_EQ(token, "another_token"); - token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/c"); + token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "github.com", "github.com/a/c"); ASSERT_EQ(token, "yet_another_token"); } @@ -76,10 +76,10 @@ TEST_F(AccessKeysTest, multipleGitLab) fetchSettings.accessTokens.get().insert({"gitlab.com/a/b", "another_token"}); auto i = Input::fromURL("gitlab:a/b"); - auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/b"); + auto token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "gitlab.com", "gitlab.com/a/b"); ASSERT_EQ(token, "another_token"); - token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/c"); + token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "gitlab.com", "gitlab.com/a/c"); ASSERT_EQ(token, "token"); } @@ -90,10 +90,10 @@ TEST_F(AccessKeysTest, multipleSourceHut) fetchSettings.accessTokens.get().insert({"git.sr.ht/~a/b", "another_token"}); auto i = Input::fromURL("sourcehut:a/b"); - auto token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/b"); + auto token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "git.sr.ht", "git.sr.ht/~a/b"); ASSERT_EQ(token, "another_token"); - token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/c"); + token = i.scheme->getAccessToken(FetchContext{fetchSettings, {}}, "git.sr.ht", "git.sr.ht/~a/c"); ASSERT_EQ(token, "token"); } diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index 5550b147f579..73959fed9cd8 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -183,7 +183,7 @@ TEST_F(GitTest, submodulePeriodSupport) auto store = [] { auto cfg = make_ref(StoreReference::Params{}); cfg->readOnly = false; - return cfg->openStore(); + return cfg->openStore(SecretContext{}); }(); auto settings = fetchers::Settings{}; @@ -194,7 +194,7 @@ TEST_F(GitTest, submodulePeriodSupport) {"ref", "main"}, }); - auto [accessor, i] = input.getAccessor(settings, *store); + auto [accessor, i] = input.getAccessor(FetchContext{settings, {}}, *store); ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index eef415d7cf16..f2a9dcffe8e8 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -197,14 +197,15 @@ bool Input::contains(const Input & other) const } // FIXME: remove -std::pair Input::fetchToStore(const Settings & settings, Store & store) const +std::pair Input::fetchToStore(const FetchContext & context, Store & store) const { + auto & settings = context.settings; if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); auto [storePath, input] = [&]() -> std::pair { try { - auto [accessor, result] = getAccessorUnchecked(settings, store); + auto [accessor, result] = getAccessorUnchecked(context, store); auto storePath = nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName()); @@ -285,10 +286,10 @@ void Input::checkLocks(Input specified, Input & result) } } -std::pair, Input> Input::getAccessor(const Settings & settings, Store & store) const +std::pair, Input> Input::getAccessor(const FetchContext & context, Store & store) const { try { - auto [accessor, result] = getAccessorUnchecked(settings, store); + auto [accessor, result] = getAccessorUnchecked(context, store); result.attrs.insert_or_assign("__final", Explicit(true)); @@ -301,8 +302,9 @@ std::pair, Input> Input::getAccessor(const Settings & settin } } -std::pair, Input> Input::getAccessorUnchecked(const Settings & settings, Store & store) const +std::pair, Input> Input::getAccessorUnchecked(const FetchContext & context, Store & store) const { + auto & settings = context.settings; // FIXME: cache the accessor if (!scheme) @@ -366,7 +368,7 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (inTest) std::this_thread::sleep_for(std::chrono::seconds(1)); - auto [accessor, result] = scheme->getAccessor(settings, store, *this); + auto [accessor, result] = scheme->getAccessor(context, store, *this); if (!accessor->fingerprint) accessor->fingerprint = result.getFingerprint(store); @@ -383,11 +385,11 @@ Input Input::applyOverrides(std::optional ref, std::optional return scheme->applyOverrides(*this, ref, rev); } -void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const +void Input::clone(const FetchContext & context, Store & store, const std::filesystem::path & destDir) const { if (!scheme) throw Error("cannot clone unsupported input '%s'", attrsToJSON(attrs)); - scheme->clone(settings, store, *this, destDir); + scheme->clone(context, store, *this, destDir); } std::optional Input::getSourcePath() const @@ -501,12 +503,12 @@ void InputScheme::putFile( } void InputScheme::clone( - const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const + const FetchContext & context, Store & store, const Input & input, const std::filesystem::path & destDir) const { if (std::filesystem::exists(destDir)) throw Error("cannot clone into existing path %s", PathFmt(destDir)); - auto [accessor, input2] = getAccessor(settings, store, input); + auto [accessor, input2] = getAccessor(context, store, input); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), PathFmt(destDir))); diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index c80732a0c21f..206312029f98 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -25,6 +25,7 @@ namespace nix::lfs { static void downloadToSink( const std::string & url, const std::optional & authHeader, + const FileTransferContext & context, Sink & sink, std::string sha256Expected, size_t sizeExpected) @@ -38,7 +39,7 @@ static void downloadToSink( HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(hashSink, sink); - getFileTransfer()->download(std::move(request), teeSink); + getFileTransfer()->download(context, std::move(request), teeSink); auto hashResult = hashSink.finish(); @@ -210,10 +211,11 @@ static std::optional parseLfsPointer(std::string_view content, std::str return std::make_optional(Pointer{oid, std::stoul(size)}); } -Fetch::Fetch(git_repository * repo, git_oid rev) +Fetch::Fetch(git_repository * repo, git_oid rev, std::shared_ptr secretResolver) { this->repo = repo; this->rev = rev; + this->secretResolver = std::move(secretResolver); const auto remoteUrl = lfs::getLfsEndpointUrl(repo); @@ -260,7 +262,8 @@ std::vector Fetch::fetchUrls(const std::vector & pointe StringSource source{payload}; request.data = {source}; - FileTransferResult result = getFileTransfer()->upload(request); + FileTransferResult result = + getFileTransfer()->upload(FileTransferContext{.secretResolver = secretResolver}, request); auto responseString = result.data; std::vector objects; @@ -360,7 +363,7 @@ void Fetch::fetch( auto [tempFile, tempPath] = createTempFile(cachePath.parent_path(), {}); AutoDelete tempDeleter(tempPath); FdSink tempSink(tempFile.get()); - downloadToSink(ourl, authHeader, tempSink, sha256, size); + downloadToSink(ourl, authHeader, FileTransferContext{.secretResolver = secretResolver}, tempSink, sha256, size); tempSink.flush(); std::filesystem::rename(tempPath, cachePath); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 46b73ece3476..d9e5e1cc0212 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -858,8 +858,14 @@ struct GitSourceAccessor final : SourceAccessor : state_{State{ .repo = repo_, .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()), - .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt, - .options = options, + .lfsFetch = options.smudgeLfs + ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev), options.secretResolver)) + : std::nullopt, + .options = + { + .exportIgnore = options.exportIgnore, + .smudgeLfs = options.smudgeLfs, + }, }} { } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f9a10ad8a3bf..4222d8670d3f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -442,7 +442,7 @@ struct GitInputScheme : InputScheme return res; } - void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) + void clone(const FetchContext & context, Store & store, const Input & input, const std::filesystem::path & destDir) const override { auto repoInfo = getRepoInfo(input); @@ -808,8 +808,9 @@ struct GitInputScheme : InputScheme } std::pair, Input> - getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const + getAccessorFromCommit(const FetchContext & context, Store & store, RepoInfo & repoInfo, Input && input) const { + auto & settings = context.settings; assert(!repoInfo.workdirInfo.isDirty); auto origRev = input.getRev(); @@ -930,7 +931,13 @@ struct GitInputScheme : InputScheme bool exportIgnore = getExportIgnoreAttr(input); bool smudgeLfs = getLfsAttr(input); auto accessor = repo->getAccessor( - rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»"); + rev, + { + .exportIgnore = exportIgnore, + .smudgeLfs = smudgeLfs, + .secretResolver = context.secretResolver, + }, + "«" + input.to_string() + "»"); /* If the repo has submodules, fetch them and return a mounted input accessor consisting of the accessor for the top-level @@ -966,7 +973,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("lfs", Explicit{smudgeLfs}); attrs.insert_or_assign("allRefs", Explicit{true}); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); - auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); + auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(context, store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); mounts.insert_or_assign(submodule.path, submoduleAccessor); } @@ -983,8 +990,9 @@ struct GitInputScheme : InputScheme } std::pair, Input> - getAccessorFromWorkdir(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const + getAccessorFromWorkdir(const FetchContext & context, Store & store, RepoInfo & repoInfo, Input && input) const { + auto & settings = context.settings; auto repoPath = repoInfo.getPath().value(); if (getSubmodulesAttr(input)) @@ -1016,7 +1024,7 @@ struct GitInputScheme : InputScheme // attrs.insert_or_assign("allRefs", Explicit{ true }); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); - auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); + auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(context, store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); /* If the submodule is dirty, mark this repo dirty as @@ -1070,7 +1078,7 @@ struct GitInputScheme : InputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + getAccessor(const FetchContext & context, Store & store, const Input & _input) const override { Input input(_input); @@ -1086,8 +1094,8 @@ struct GitInputScheme : InputScheme } auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath() - ? getAccessorFromCommit(settings, store, repoInfo, std::move(input)) - : getAccessorFromWorkdir(settings, store, repoInfo, std::move(input)); + ? getAccessorFromCommit(context, store, repoInfo, std::move(input)) + : getAccessorFromWorkdir(context, store, repoInfo, std::move(input)); return {accessor, std::move(final)}; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ac608da37a31..cf5f633abc11 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -194,9 +194,10 @@ struct GitArchiveInputScheme : InputScheme } // Search for the longest possible match starting from the beginning and ending at either the end or a path segment. - std::optional getAccessToken( - const fetchers::Settings & settings, const std::string & host, const std::string & url) const override + std::optional + getAccessToken(const FetchContext & context, const std::string & host, const std::string & url) const override { + auto & settings = context.settings; auto tokens = settings.accessTokens.get(); std::string answer; size_t answer_match_len = 0; @@ -218,20 +219,19 @@ struct GitArchiveInputScheme : InputScheme return {}; } - Headers - makeHeadersWithAuthTokens(const fetchers::Settings & settings, const std::string & host, const Input & input) const + Headers makeHeadersWithAuthTokens(const FetchContext & context, const std::string & host, const Input & input) const { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); auto hostAndPath = fmt("%s/%s/%s", host, owner, repo); - return makeHeadersWithAuthTokens(settings, host, hostAndPath); + return makeHeadersWithAuthTokens(context, host, hostAndPath); } Headers makeHeadersWithAuthTokens( - const fetchers::Settings & settings, const std::string & host, const std::string & hostAndPath) const + const FetchContext & context, const std::string & host, const std::string & hostAndPath) const { Headers headers; - auto accessToken = getAccessToken(settings, host, hostAndPath); + auto accessToken = getAccessToken(context, host, hostAndPath); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -248,9 +248,9 @@ struct GitArchiveInputScheme : InputScheme std::optional treeHash; }; - virtual RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const = 0; + virtual RefInfo getRevFromRef(const FetchContext & context, nix::Store & store, const Input & input) const = 0; - virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0; + virtual DownloadUrl getDownloadUrl(const FetchContext & context, const Input & input) const = 0; struct TarballInfo { @@ -258,8 +258,9 @@ struct GitArchiveInputScheme : InputScheme time_t lastModified; }; - std::pair downloadArchive(const Settings & settings, Store & store, Input input) const + std::pair downloadArchive(const FetchContext & context, Store & store, Input input) const { + auto & settings = context.settings; if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); @@ -267,7 +268,7 @@ struct GitArchiveInputScheme : InputScheme auto rev = input.getRev(); if (!rev) { - auto refInfo = getRevFromRef(settings, store, input); + auto refInfo = getRevFromRef(context, store, input); rev = refInfo.rev; upstreamTreeHash = refInfo.treeHash; debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev()); @@ -293,12 +294,12 @@ struct GitArchiveInputScheme : InputScheme } /* Stream the tarball into the tarball cache. */ - auto url = getDownloadUrl(settings, input); + auto url = getDownloadUrl(context, input); auto source = sinkToSource([&](Sink & sink) { FileTransferRequest req(url.url); req.headers = url.headers; - getFileTransfer()->download(std::move(req), sink); + getFileTransfer()->download(FileTransferContext{context.secretResolver}, std::move(req), sink); }); auto act = std::make_unique( @@ -331,9 +332,10 @@ struct GitArchiveInputScheme : InputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + getAccessor(const FetchContext & context, Store & store, const Input & _input) const override { - auto [input, tarballInfo] = downloadArchive(settings, store, _input); + auto & settings = context.settings; + auto [input, tarballInfo] = downloadArchive(context, store, _input); #if 0 input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); @@ -408,7 +410,7 @@ struct GitHubInputScheme : GitArchiveInputScheme return getStrAttr(input.attrs, "repo"); } - RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override + RefInfo getRevFromRef(const FetchContext & context, nix::Store & store, const Input & input) const override { auto host = getHost(input); auto url = fmt( @@ -418,9 +420,9 @@ struct GitHubInputScheme : GitArchiveInputScheme getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(settings, host, input); + Headers headers = makeHeadersWithAuthTokens(context, host, input); - auto downloadResult = downloadFile(store, settings, url, "source", headers); + auto downloadResult = downloadFile(store, context, url, "source", headers); auto json = nlohmann::json::parse( store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); @@ -429,11 +431,11 @@ struct GitHubInputScheme : GitArchiveInputScheme .treeHash = Hash::parseAny(std::string{json["commit"]["tree"]["sha"]}, HashAlgorithm::SHA1)}; } - DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override + DownloadUrl getDownloadUrl(const FetchContext & context, const Input & input) const override { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(settings, host, input); + Headers headers = makeHeadersWithAuthTokens(context, host, input); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -447,13 +449,13 @@ struct GitHubInputScheme : GitArchiveInputScheme return DownloadUrl{parseURL(url), headers}; } - void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) + void clone(const FetchContext & context, Store & store, const Input & input, const std::filesystem::path & destDir) const override { auto host = getHost(input); Input::fromURL(fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) - .clone(settings, store, destDir); + .clone(context, store, destDir); } }; @@ -489,7 +491,7 @@ struct GitLabInputScheme : GitArchiveInputScheme return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1)); } - RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override + RefInfo getRevFromRef(const FetchContext & context, nix::Store & store, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // See rate limiting note below @@ -500,9 +502,9 @@ struct GitLabInputScheme : GitArchiveInputScheme getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(settings, host, input); + Headers headers = makeHeadersWithAuthTokens(context, host, input); - auto downloadResult = downloadFile(store, settings, url, "source", headers); + auto downloadResult = downloadFile(store, context, url, "source", headers); auto json = nlohmann::json::parse( store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); @@ -516,7 +518,7 @@ struct GitLabInputScheme : GitArchiveInputScheme } } - DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override + DownloadUrl getDownloadUrl(const FetchContext & context, const Input & input) const override { // This endpoint has a rate limit threshold that may be // server-specific and vary based whether the user is @@ -531,11 +533,11 @@ struct GitLabInputScheme : GitArchiveInputScheme getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(settings, host, input); + Headers headers = makeHeadersWithAuthTokens(context, host, input); return DownloadUrl{parseURL(url), headers}; } - void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) + void clone(const FetchContext & context, Store & store, const Input & input, const std::filesystem::path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); @@ -543,7 +545,7 @@ struct GitLabInputScheme : GitArchiveInputScheme Input::fromURL( fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) - .clone(settings, store, destDir); + .clone(context, store, destDir); } }; @@ -570,7 +572,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme // Once it is implemented, however, should work as expected. } - RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override + RefInfo getRevFromRef(const FetchContext & context, nix::Store & store, const Input & input) const override { // TODO: In the future, when the sourcehut graphql API is implemented for mercurial // and with anonymous access, this method should use it instead. @@ -581,11 +583,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(settings, host, input); + Headers headers = makeHeadersWithAuthTokens(context, host, input); std::string refUri; if (ref == "HEAD") { - auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers); + auto downloadFileResult = downloadFile(store, context, fmt("%s/HEAD", base_url), "source", headers); auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); auto remoteLine = git::parseLsRemoteLine(getLine(contents).first); @@ -598,7 +600,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme } std::regex refRegex(refUri); - auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers); + auto downloadFileResult = downloadFile(store, context, fmt("%s/info/refs", base_url), "source", headers); auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); std::istringstream is(contents); @@ -616,7 +618,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme return RefInfo{.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)}; } - DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override + DownloadUrl getDownloadUrl(const FetchContext & context, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = @@ -626,18 +628,18 @@ struct SourceHutInputScheme : GitArchiveInputScheme getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(settings, host, input); + Headers headers = makeHeadersWithAuthTokens(context, host, input); return DownloadUrl{parseURL(url), headers}; } - void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) + void clone(const FetchContext & context, Store & store, const Input & input, const std::filesystem::path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); Input::fromURL( fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) - .clone(settings, store, destDir); + .clone(context, store, destDir); } }; diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index bb9a67d068d4..76bfabe77ba4 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -5,9 +5,11 @@ #include "nix/util/configuration.hh" #include "nix/util/ref.hh" #include "nix/util/sync.hh" +#include "nix/store/secret-resolver.hh" #include #include +#include #include @@ -168,4 +170,17 @@ private: mutable Sync> _cache; }; +/** + * Dependencies shared by one fetch operation tree. + * + * The resolver is deliberately owned here rather than by Settings or a + * process-global singleton. This lets callers isolate resolver state and + * select the lifetime appropriate for a command or evaluation. + */ +struct FetchContext +{ + const Settings & settings; + std::shared_ptr secretResolver; +}; + } // namespace nix::fetchers diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 64644302149f..b75ff501fe9e 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -24,6 +24,7 @@ namespace nix::fetchers { struct InputScheme; struct Settings; +struct FetchContext; /** * The `Input` object is generated by a specific fetcher, based on @@ -113,7 +114,7 @@ public: * Fetch the entire input into the Nix store, returning the * location in the Nix store and the locked input. */ - std::pair fetchToStore(const Settings & settings, Store & store) const; + std::pair fetchToStore(const FetchContext & context, Store & store) const; /** * Check the locking attributes in `result` against @@ -133,17 +134,17 @@ public: * input without copying it to the store. Also return a possibly * unlocked input. */ - std::pair, Input> getAccessor(const Settings & settings, Store & store) const; + std::pair, Input> getAccessor(const FetchContext & context, Store & store) const; private: - std::pair, Input> getAccessorUnchecked(const Settings & settings, Store & store) const; + std::pair, Input> getAccessorUnchecked(const FetchContext & context, Store & store) const; public: Input applyOverrides(std::optional ref, std::optional rev) const; - void clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const; + void clone(const FetchContext & context, Store & store, const std::filesystem::path & destDir) const; std::optional getSourcePath() const; @@ -228,8 +229,8 @@ struct InputScheme virtual Input applyOverrides(const Input & input, std::optional ref, std::optional rev) const; - virtual void - clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const; + virtual void clone( + const FetchContext & context, Store & store, const Input & input, const std::filesystem::path & destDir) const; virtual std::optional getSourcePath(const Input & input) const; @@ -240,7 +241,7 @@ struct InputScheme std::optional commitMsg) const; virtual std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & input) const = 0; + getAccessor(const FetchContext & context, Store & store, const Input & input) const = 0; /** * Is this `InputScheme` part of an experimental feature? @@ -268,7 +269,7 @@ struct InputScheme } virtual std::optional - getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const + getAccessToken(const FetchContext & context, const std::string & host, const std::string & url) const { return {}; } diff --git a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh index 444e0a648f14..98feb69bc2e6 100644 --- a/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh +++ b/src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh @@ -9,6 +9,10 @@ #include +namespace nix { +class SecretResolver; +} + namespace nix::lfs { /** @@ -32,8 +36,9 @@ struct Fetch // derived from git remote url nix::ParsedURL url; + std::shared_ptr secretResolver; - Fetch(git_repository * repo, git_oid rev); + Fetch(git_repository * repo, git_oid rev, std::shared_ptr secretResolver); bool shouldFetch(const CanonPath & path) const; void fetch( const std::string & content, diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index f34f3bfde1b9..e0d30420aae6 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -5,6 +5,8 @@ namespace nix { +class SecretResolver; + namespace fetchers { struct PublicKey; struct Settings; @@ -30,6 +32,7 @@ struct GitAccessorOptions { bool exportIgnore = false; bool smudgeLfs = false; + std::shared_ptr secretResolver; }; struct GitRepo diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh index 4a6a1dff83c6..8824d4703889 100644 --- a/src/libfetchers/include/nix/fetchers/input-cache.hh +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -5,7 +5,7 @@ namespace nix::fetchers { enum class UseRegistries : int; -struct Settings; +struct FetchContext; struct InputCache { @@ -18,7 +18,7 @@ struct InputCache }; CachedResult - getAccessor(const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries); + getAccessor(const FetchContext & context, Store & store, const Input & originalInput, UseRegistries useRegistries); struct CachedInput { diff --git a/src/libfetchers/include/nix/fetchers/registry.hh b/src/libfetchers/include/nix/fetchers/registry.hh index dc7e3edb590e..ab794694b298 100644 --- a/src/libfetchers/include/nix/fetchers/registry.hh +++ b/src/libfetchers/include/nix/fetchers/registry.hh @@ -54,7 +54,7 @@ std::shared_ptr getCustomRegistry(const Settings & settings, const std std::filesystem::path getUserRegistryPath(); -Registries getRegistries(const Settings & settings, Store & store); +Registries getRegistries(const FetchContext & context, Store & store); void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs); @@ -69,6 +69,6 @@ enum class UseRegistries : int { * use the registries for which the filter function returns true. */ std::pair -lookupInRegistries(const Settings & settings, Store & store, const Input & input, UseRegistries useRegistries); +lookupInRegistries(const FetchContext & context, Store & store, const Input & input, UseRegistries useRegistries); } // namespace nix::fetchers diff --git a/src/libfetchers/include/nix/fetchers/tarball.hh b/src/libfetchers/include/nix/fetchers/tarball.hh index 84aa8e5adc6e..29b40dfd34f2 100644 --- a/src/libfetchers/include/nix/fetchers/tarball.hh +++ b/src/libfetchers/include/nix/fetchers/tarball.hh @@ -16,6 +16,7 @@ struct SourceAccessor; namespace nix::fetchers { struct Settings; +struct FetchContext; struct DownloadFileResult { @@ -27,7 +28,7 @@ struct DownloadFileResult DownloadFileResult downloadFile( Store & store, - const Settings & settings, + const FetchContext & context, const VerbatimURL & url, const std::string & name, const Headers & headers = {}); @@ -44,6 +45,6 @@ struct DownloadTarballResult * Download and import a tarball into the Git cache. The result is the * Git tree hash of the root directory. */ -ref downloadTarball(Store & store, const Settings & settings, const std::string & url); +ref downloadTarball(Store & store, const FetchContext & context, const std::string & url); } // namespace nix::fetchers diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 0baa727727ac..ca70485dff16 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -126,7 +126,7 @@ struct IndirectInputScheme : InputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & input) const override + getAccessor(const FetchContext & context, Store & store, const Input & input) const override { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 3fe96d8503bc..59833d9203eb 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" @@ -6,22 +7,22 @@ namespace nix::fetchers { InputCache::CachedResult InputCache::getAccessor( - const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries) + const FetchContext & context, Store & store, const Input & originalInput, UseRegistries useRegistries) { auto fetched = lookup(originalInput); Input resolvedInput = originalInput; if (!fetched) { if (originalInput.isDirect()) { - auto [accessor, lockedInput] = originalInput.getAccessor(settings, store); + auto [accessor, lockedInput] = originalInput.getAccessor(context, store); fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); } else { if (useRegistries != UseRegistries::No) { - auto [res, extraAttrs] = lookupInRegistries(settings, store, originalInput, useRegistries); + auto [res, extraAttrs] = lookupInRegistries(context, store, originalInput, useRegistries); resolvedInput = std::move(res); fetched = lookup(resolvedInput); if (!fetched) { - auto [accessor, lockedInput] = resolvedInput.getAccessor(settings, store); + auto [accessor, lockedInput] = resolvedInput.getAccessor(context, store); fetched.emplace( CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs}); } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 4176ec29de1f..0c749aa23116 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -395,8 +395,9 @@ struct MercurialInputScheme : InputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + getAccessor(const FetchContext & context, Store & store, const Input & _input) const override { + auto & settings = context.settings; Input input(_input); auto storePath = fetchToStore(settings, store, input); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 187bd6936d34..802550062bdb 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -139,8 +139,9 @@ struct PathInputScheme : InputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + getAccessor(const FetchContext & context, Store & store, const Input & _input) const override { + auto & settings = context.settings; Input input(_input); auto path = getStrAttr(input.attrs, "path"); diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 513b8b843cf0..de3f46e39a11 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -136,9 +136,15 @@ void overrideRegistry(const Input & from, const Input & to, const Attrs & extraA getFlagRegistry()->add(from, to, extraAttrs); } -static std::shared_ptr getGlobalRegistry(const Settings & settings, Store & store) +static std::shared_ptr getGlobalRegistry(const FetchContext & context, Store & store) { + /* Like the user and system registries, this is read once per process: + `lookupInRegistries()` is called for every indirect flake input, and + downloading and re-rooting the registry on each of those would mean + an HTTP request per input on a cold cache. The context of the first + caller is the one that performs the fetch. */ static auto reg = [&]() { + auto & settings = context.settings; auto path = settings.flakeRegistry.get(); if (path == "") { return std::make_shared(Registry::Global); // empty registry @@ -149,7 +155,7 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, St [&] -> SourcePath { std::filesystem::path fsPath{path}; if (!fsPath.is_absolute()) { - auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; + auto storePath = downloadFile(store, context, path, "flake-registry.json").storePath; if (auto store2 = dynamic_cast(&store)) store2->addPermRoot(storePath, (getCacheDir() / "flake-registry.json").string()); return {store.requireStoreObjectAccessor(storePath)}; @@ -163,18 +169,19 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, St return reg; } -Registries getRegistries(const Settings & settings, Store & store) +Registries getRegistries(const FetchContext & context, Store & store) { + auto & settings = context.settings; Registries registries; registries.push_back(getFlagRegistry()); registries.push_back(getUserRegistry(settings)); registries.push_back(getSystemRegistry(settings)); - registries.push_back(getGlobalRegistry(settings, store)); + registries.push_back(getGlobalRegistry(context, store)); return registries; } std::pair -lookupInRegistries(const Settings & settings, Store & store, const Input & _input, UseRegistries useRegistries) +lookupInRegistries(const FetchContext & context, Store & store, const Input & _input, UseRegistries useRegistries) { Attrs extraAttrs; int n = 0; @@ -189,7 +196,7 @@ lookupInRegistries(const Settings & settings, Store & store, const Input & _inpu if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); - for (auto & registry : getRegistries(settings, store)) { + for (auto & registry : getRegistries(context, store)) { if (useRegistries == UseRegistries::Limited && !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global)) continue; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index fdf6c8ad1167..c0e1d49c8edb 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -14,11 +14,12 @@ namespace nix::fetchers { DownloadFileResult downloadFile( Store & store, - const Settings & settings, + const FetchContext & context, const VerbatimURL & url, const std::string & name, const Headers & headers) { + auto & settings = context.settings; // FIXME: check store Cache::Key key{ @@ -48,7 +49,7 @@ DownloadFileResult downloadFile( request.expectedETag = getStrAttr(cached->value, "etag"); FileTransferResult res; try { - res = getFileTransfer()->download(request); + res = getFileTransfer()->download(FileTransferContext{context.secretResolver}, request); } catch (FileTransferError & e) { if (cached) { warn("%s; using cached version", e.message()); @@ -105,8 +106,9 @@ DownloadFileResult downloadFile( } static DownloadTarballResult downloadTarball_( - const Settings & settings, const std::string & urlS, const Headers & headers, const std::string & displayPrefix) + const FetchContext & context, const std::string & urlS, const Headers & headers, const std::string & displayPrefix) { + auto & settings = context.settings; ParsedURL url = parseURL(urlS); // Some friendly error messages for common mistakes. @@ -160,7 +162,10 @@ static DownloadTarballResult downloadTarball_( auto source = sinkToSource([&](Sink & sink) { FileTransferRequest req(url); req.expectedETag = cached ? getStrAttr(cached->value, "etag") : ""; - getFileTransfer()->download(std::move(req), sink, [_res](FileTransferResult r) { *_res->lock() = r; }); + getFileTransfer()->download( + FileTransferContext{context.secretResolver}, std::move(req), sink, [_res](FileTransferResult r) { + *_res->lock() = r; + }); }); // TODO: fall back to cached value if download fails. @@ -222,7 +227,7 @@ static DownloadTarballResult downloadTarball_( return attrsToResult(infoAttrs); } -ref downloadTarball(Store & store, const Settings & settings, const std::string & url) +ref downloadTarball(Store & store, const FetchContext & context, const std::string & url) { /* Go through Input::getAccessor() to ensure that the resulting accessor has a fingerprint. */ @@ -232,7 +237,7 @@ ref downloadTarball(Store & store, const Settings & settings, co auto input = Input::fromAttrs(std::move(attrs)); - return input.getAccessor(settings, store).first; + return input.getAccessor(context, store).first; } // An input scheme corresponding to a curl-downloadable resource. @@ -421,7 +426,7 @@ struct FileInputScheme : CurlInputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + getAccessor(const FetchContext & context, Store & store, const Input & _input) const override { auto input(_input); @@ -429,7 +434,7 @@ struct FileInputScheme : CurlInputScheme the Nix store directly, since there is little deduplication benefit in using the Git cache for single big files like tarballs. */ - auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName()); + auto file = downloadFile(store, context, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store.queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); @@ -487,11 +492,12 @@ struct TarballInputScheme : CurlInputScheme } std::pair, Input> - getAccessor(const Settings & settings, Store & store, const Input & _input) const override + getAccessor(const FetchContext & context, Store & store, const Input & _input) const override { + auto & settings = context.settings; auto input(_input); - auto result = downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); + auto result = downloadTarball_(context, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); if (result.immutableUrl) { auto immutableInput = Input::fromURL(*result.immutableUrl); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 86310cc2f2be..99ab98961fce 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -369,7 +369,7 @@ static Flake getFlake( { // Fetch a lazy tree first. auto cachedInput = - state.inputCache->getAccessor(state.fetchSettings, *state.store, originalRef.input, useRegistries); + state.inputCache->getAccessor(state.fetchContext, *state.store, originalRef.input, useRegistries); auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir); auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir); @@ -386,7 +386,7 @@ static Flake getFlake( // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. newLockedRef.input.attrs.erase("narHash"); auto cachedInput2 = state.inputCache->getAccessor( - state.fetchSettings, *state.store, newLockedRef.input, fetchers::UseRegistries::No); + state.fetchContext, *state.store, newLockedRef.input, fetchers::UseRegistries::No); cachedInput.accessor = cachedInput2.accessor; lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } @@ -744,7 +744,7 @@ LockedFlake lockFlake( return {*resolvedPath, *input.ref}; } else { auto cachedInput = state.inputCache->getAccessor( - state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); + state.fetchContext, *state.store, input.ref->input, useRegistriesInputs); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 6e1ec3287cf3..2e0015e7d7f3 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -64,10 +64,10 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef) return str; } -FlakeRef -FlakeRef::resolve(const fetchers::Settings & fetchSettings, Store & store, fetchers::UseRegistries useRegistries) const +FlakeRef FlakeRef::resolve( + const fetchers::FetchContext & fetchContext, Store & store, fetchers::UseRegistries useRegistries) const { - auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries); + auto [input2, extraAttrs] = lookupInRegistries(fetchContext, store, input, useRegistries); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } @@ -280,9 +280,9 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) } std::pair, FlakeRef> -FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const +FlakeRef::lazyFetch(const fetchers::FetchContext & fetchContext, Store & store) const { - auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store); + auto [accessor, lockedInput] = input.getAccessor(fetchContext, store); return {accessor, FlakeRef(std::move(lockedInput), subdir)}; } diff --git a/src/libflake/include/nix/flake/flakeref.hh b/src/libflake/include/nix/flake/flakeref.hh index e63d574e0c91..d59eccb9604d 100644 --- a/src/libflake/include/nix/flake/flakeref.hh +++ b/src/libflake/include/nix/flake/flakeref.hh @@ -74,13 +74,14 @@ struct FlakeRef fetchers::Attrs toAttrs() const; FlakeRef resolve( - const fetchers::Settings & fetchSettings, + const fetchers::FetchContext & fetchContext, Store & store, fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const; static FlakeRef fromAttrs(const fetchers::Attrs & attrs); - std::pair, FlakeRef> lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const; + std::pair, FlakeRef> + lazyFetch(const fetchers::FetchContext & fetchContext, Store & store) const; /** * Canonicalize a flakeref for the purpose of comparing "old" and diff --git a/src/libstore-test-support/https-store.cc b/src/libstore-test-support/https-store.cc index 468515079c7a..d8215fadffbe 100644 --- a/src/libstore-test-support/https-store.cc +++ b/src/libstore-test-support/https-store.cc @@ -10,12 +10,14 @@ void TestHttpBinaryCacheStore::init() BinaryCacheStore::init(); } -ref TestHttpBinaryCacheStoreConfig::openTestStore(ref fileTransfer) const +ref +TestHttpBinaryCacheStoreConfig::openTestStore(ref fileTransfer, const SecretContext & context) const { auto store = make_ref( ref{// FIXME we shouldn't actually need a mutable config std::const_pointer_cast(shared_from_this())}, - fileTransfer); + fileTransfer, + context); store->init(); return store; } @@ -37,8 +39,8 @@ void HttpsBinaryCacheStoreTest::SetUp() cacheDir = tmpDir / "cache"; delTmpDir = std::make_unique(tmpDir); - localCacheStore = - make_ref(cacheDir, LocalBinaryCacheStoreConfig::Params{})->openStore(); + localCacheStore = make_ref(cacheDir, LocalBinaryCacheStoreConfig::Params{}) + ->openStore(SecretContext{}); caCert = tmpDir / "ca.crt"; caKey = tmpDir / "ca.key"; @@ -139,9 +141,10 @@ ref HttpsBinaryCacheStoreTest::makeConfig() return res; } -ref HttpsBinaryCacheStoreTest::openStore(ref config) +ref +HttpsBinaryCacheStoreTest::openStore(ref config, const SecretContext & context) { - return config->openTestStore(ref{testFileTransfer}); + return config->openTestStore(ref{testFileTransfer}, context); } } // namespace nix::testing diff --git a/src/libstore-test-support/include/nix/store/tests/https-store.hh b/src/libstore-test-support/include/nix/store/tests/https-store.hh index 9d4db058e5a4..1454cc46ccc1 100644 --- a/src/libstore-test-support/include/nix/store/tests/https-store.hh +++ b/src/libstore-test-support/include/nix/store/tests/https-store.hh @@ -28,10 +28,11 @@ public: TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete; TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete; - TestHttpBinaryCacheStore(ref config, ref fileTransfer) - : Store{*config} + TestHttpBinaryCacheStore( + ref config, ref fileTransfer, SecretContext context = {}) + : Store{*config, context} , BinaryCacheStore{*config} - , HttpBinaryCacheStore(config, fileTransfer) + , HttpBinaryCacheStore(config, fileTransfer, context) { diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */ } @@ -48,7 +49,8 @@ public: { } - ref openTestStore(ref fileTransfer) const; + ref + openTestStore(ref fileTransfer, const SecretContext & context = {}) const; }; class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest @@ -89,7 +91,8 @@ protected: virtual std::vector serverArgs(); ref makeConfig(); - ref openStore(ref config); + ref + openStore(ref config, const SecretContext & context = {}); }; class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest diff --git a/src/libstore-test-support/include/nix/store/tests/meson.build b/src/libstore-test-support/include/nix/store/tests/meson.build index 8d844d24e7d8..752b4bbb32bd 100644 --- a/src/libstore-test-support/include/nix/store/tests/meson.build +++ b/src/libstore-test-support/include/nix/store/tests/meson.build @@ -11,5 +11,6 @@ headers = files( 'outputs-spec.hh', 'path.hh', 'protocol.hh', + 'secret-resolver.hh', 'test-main.hh', ) diff --git a/src/libstore-test-support/include/nix/store/tests/secret-resolver.hh b/src/libstore-test-support/include/nix/store/tests/secret-resolver.hh new file mode 100644 index 000000000000..edda9b1e6992 --- /dev/null +++ b/src/libstore-test-support/include/nix/store/tests/secret-resolver.hh @@ -0,0 +1,76 @@ +#pragma once +///@file + +#include "nix/store/secret-resolver.hh" + +#include +#include +#include +#include + +namespace nix::testing { + +class CallbackSecretFile : public SecretFile +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + void anchor() override; + +public: + CallbackSecretFile(std::filesystem::path path, std::function onDestroy = {}) + : filePath(std::move(path)) + , onDestroy(std::move(onDestroy)) + { + } + + ~CallbackSecretFile() override + { + if (onDestroy) + onDestroy(); + } + + const std::filesystem::path & path() const noexcept override + { + return filePath; + } + +private: + std::filesystem::path filePath; + std::function onDestroy; +}; + +/** + * Records what was asked for and answers from a callback. + * + * Not thread-safe, unlike a real resolver: tests drive it from one thread + * and want the request log to stay in call order. + */ +class CallbackSecretResolver : public SecretResolver +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + void anchor() override; + +public: + using Callback = std::function(const SecretRequest &)>; + + explicit CallbackSecretResolver(Callback callback) + : callback(std::move(callback)) + { + } + + std::optional resolve(const SecretRequest & request) override + { + requests.push_back(request); + return callback(request); + } + + std::vector requests; + +private: + Callback callback; +}; + +} // namespace nix::testing diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index ca451db4abce..199e14ee514b 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -39,6 +39,7 @@ sources = files( 'libstore-network.cc', 'outputs-spec.cc', 'path.cc', + 'secret-resolver.cc', 'test-main.cc', ) diff --git a/src/libstore-test-support/secret-resolver.cc b/src/libstore-test-support/secret-resolver.cc new file mode 100644 index 000000000000..12f9b88cc546 --- /dev/null +++ b/src/libstore-test-support/secret-resolver.cc @@ -0,0 +1,9 @@ +#include "nix/store/tests/secret-resolver.hh" + +namespace nix::testing { + +void CallbackSecretFile::anchor() {} + +void CallbackSecretResolver::anchor() {} + +} // namespace nix::testing diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 6c2625d5306a..bbdef0cc5c10 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -7,6 +7,7 @@ #include "nix/store/realisation.hh" #include "nix/store/tests/libstore.hh" +#include "nix/store/tests/secret-resolver.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { @@ -64,6 +65,17 @@ TEST(DummyStore, getStateDir_default) EXPECT_EQ(config.getStateDir(), settings.nixStateDir); } +TEST(DummyStore, openStorePreservesSecretContext) +{ + auto config = make_ref(StoreReference::Params{}); + auto resolver = std::make_shared( + [](const SecretRequest &) { return ResolvedSecret{.value = InlineSecret{"unused"}}; }); + + auto store = config->openStore(SecretContext{.secretResolver = resolver}); + + EXPECT_EQ(store->getSecretContext().secretResolver, resolver); +} + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); diff --git a/src/libstore-tests/filetransfer-netrc.cc b/src/libstore-tests/filetransfer-netrc.cc new file mode 100644 index 000000000000..e2da065a7ca3 --- /dev/null +++ b/src/libstore-tests/filetransfer-netrc.cc @@ -0,0 +1,204 @@ +#include + +#include "nix/store/filetransfer-impl.hh" +#include "nix/store/tests/secret-resolver.hh" +#include "nix/util/file-system.hh" + +namespace nix { + +namespace { + +FileTransferRequest getRequest(std::string url = "https://cache.example.org/nar/abc") +{ + return FileTransferRequest(VerbatimURL{std::move(url)}); +} + +/** A `netrc-file` setting pointing somewhere recognisable. */ +FileTransferSettings settingsWithNetrc(const std::filesystem::path & path) +{ + FileTransferSettings settings; + settings.netrcFile = path; + return settings; +} + +SecretPurpose buildPurpose() +{ + return SecretPurpose{.consumer = "builtin:fetchurl", .operation = "build"}; +} + +} // namespace + +TEST(ResolveNetrcFile, withoutResolverUsesSetting) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + + auto netrc = resolveNetrcFile(FileTransferContext{}, settings, getRequest()); + + EXPECT_EQ(netrc.path, "/etc/nix/netrc"); + /* Nothing to keep alive: the setting names a file we don't own. */ + EXPECT_EQ(netrc.lease, nullptr); +} + +TEST(ResolveNetrcFile, resolverWithoutNetrcFallsBackToSetting) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + auto resolver = + std::make_shared([](const SecretRequest &) { return std::nullopt; }); + + auto netrc = resolveNetrcFile(FileTransferContext{.secretResolver = resolver}, settings, getRequest()); + + EXPECT_EQ(netrc.path, "/etc/nix/netrc"); + EXPECT_EQ(netrc.lease, nullptr); + /* The resolver was consulted; it just had nothing to offer. */ + ASSERT_EQ(resolver->requests.size(), 1u); +} + +TEST(ResolveNetrcFile, resolverFileTakesPrecedenceOverSetting) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + auto resolver = std::make_shared([](const SecretRequest &) { + return ResolvedSecret{.value = make_ref("/run/secrets/netrc")}; + }); + + auto netrc = resolveNetrcFile(FileTransferContext{.secretResolver = resolver}, settings, getRequest()); + + EXPECT_EQ(netrc.path, "/run/secrets/netrc"); + ASSERT_NE(netrc.lease, nullptr); + EXPECT_EQ(netrc.lease->path(), "/run/secrets/netrc"); +} + +TEST(ResolveNetrcFile, scopesRequestToHostAndOperation) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + auto resolver = + std::make_shared([](const SecretRequest &) { return std::nullopt; }); + + auto request = getRequest("https://cache.example.org:8443/nar/abc"); + request.method = HttpMethod::Head; + resolveNetrcFile(FileTransferContext{.secretResolver = resolver}, settings, request); + + ASSERT_EQ(resolver->requests.size(), 1u); + const auto & secretRequest = resolver->requests.at(0); + EXPECT_EQ(secretRequest.name, "netrc"); + /* curl reads netrc from disk, so an inline value would be useless. */ + EXPECT_EQ(secretRequest.representation, SecretRepresentation::MaterialisedFile); + EXPECT_EQ(secretRequest.purpose.consumer, "file-transfer"); + EXPECT_EQ(secretRequest.purpose.operation, "download"); + /* The port is not part of a netrc machine name. */ + EXPECT_EQ(secretRequest.purpose.host, std::optional{"cache.example.org"}); +} + +TEST(ResolveNetrcFile, unparseableUrlLeavesHostUnscoped) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + auto resolver = + std::make_shared([](const SecretRequest &) { return std::nullopt; }); + + resolveNetrcFile(FileTransferContext{.secretResolver = resolver}, settings, getRequest("not a url")); + + ASSERT_EQ(resolver->requests.size(), 1u); + EXPECT_EQ(resolver->requests.at(0).purpose.host, std::nullopt); +} + +TEST(ResolveNetrcFile, inlineSecretIsRejected) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + auto resolver = std::make_shared( + [](const SecretRequest &) { return ResolvedSecret{.value = InlineSecret{"machine example.org"}}; }); + + EXPECT_THROW(resolveNetrcFile(FileTransferContext{.secretResolver = resolver}, settings, getRequest()), Error); +} + +TEST(ResolveNetrcFile, leaseOutlivesResolutionAndIsReleasedWithIt) +{ + auto settings = settingsWithNetrc("/etc/nix/netrc"); + bool released = false; + auto resolver = std::make_shared([&](const SecretRequest &) { + return ResolvedSecret{ + .value = make_ref("/run/secrets/netrc", [&] { released = true; })}; + }); + + { + auto netrc = resolveNetrcFile(FileTransferContext{.secretResolver = resolver}, settings, getRequest()); + /* The materialisation must survive the resolve() call itself: curl + reads the file long after we hand it the path. */ + EXPECT_FALSE(released); + EXPECT_EQ(netrc.path, "/run/secrets/netrc"); + } + + EXPECT_TRUE(released); +} + +TEST(ResolveNetrcData, resolverValueTakesPrecedenceOverSetting) +{ + auto settings = settingsWithNetrc("/definitely/not/a/netrc"); + auto resolver = std::make_shared( + [](const SecretRequest &) { return ResolvedSecret{.value = InlineSecret{"machine example.org"}}; }); + + auto data = resolveNetrcData(resolver, settings, buildPurpose()); + + ASSERT_TRUE(data); + EXPECT_EQ(*data, "machine example.org"); + + ASSERT_EQ(resolver->requests.size(), 1u); + const auto & request = resolver->requests.at(0); + EXPECT_EQ(request.name, "netrc"); + /* The bytes have to cross into a sandbox, so a leased file is no use. */ + EXPECT_EQ(request.representation, SecretRepresentation::Inline); + EXPECT_EQ(request.purpose.consumer, "builtin:fetchurl"); + /* One netrc serves every URL the build tries, so it is not host-scoped. */ + EXPECT_EQ(request.purpose.host, std::nullopt); +} + +TEST(ResolveNetrcData, resolverWithoutNetrcFallsBackToSettingFile) +{ + AutoDelete tmpDir(createTempDir()); + auto netrcPath = tmpDir.path() / "netrc"; + writeFile(netrcPath, "machine fallback.example.org", 0600); + + auto settings = settingsWithNetrc(netrcPath); + auto resolver = + std::make_shared([](const SecretRequest &) { return std::nullopt; }); + + auto data = resolveNetrcData(resolver, settings, buildPurpose()); + + ASSERT_TRUE(data); + EXPECT_EQ(*data, "machine fallback.example.org"); + /* The resolver was asked; the setting is only the fallback. */ + ASSERT_EQ(resolver->requests.size(), 1u); +} + +TEST(ResolveNetrcData, unreadableSettingYieldsNothing) +{ + auto settings = settingsWithNetrc("/definitely/not/a/netrc"); + + EXPECT_EQ(resolveNetrcData(nullptr, settings, buildPurpose()), std::nullopt); +} + +TEST(ResolveNetrcData, explicitlyEmptyResolverValueTakesPrecedenceOverSetting) +{ + AutoDelete tmpDir(createTempDir()); + auto netrcPath = tmpDir.path() / "netrc"; + writeFile(netrcPath, "machine fallback.example.org", 0600); + + auto settings = settingsWithNetrc(netrcPath); + auto resolver = std::make_shared( + [](const SecretRequest &) { return ResolvedSecret{.value = InlineSecret{""}}; }); + + auto data = resolveNetrcData(resolver, settings, buildPurpose()); + + ASSERT_TRUE(data); + EXPECT_TRUE(data->empty()); +} + +TEST(ResolveNetrcData, materialisedFileIsRejected) +{ + auto settings = settingsWithNetrc("/definitely/not/a/netrc"); + auto resolver = std::make_shared([](const SecretRequest &) { + return ResolvedSecret{.value = make_ref("/run/secrets/netrc")}; + }); + + EXPECT_THROW(resolveNetrcData(resolver, settings, buildPurpose()), Error); +} + +} // namespace nix diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index 926f4dc38741..0760e4d074bc 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -4,6 +4,8 @@ namespace nix { +static_assert(requires(LegacySSHStore & store, std::shared_ptr evalStore) { store.getBuilder(evalStore); }); + TEST(LegacySSHStore, storeDir_absolutePath) { LegacySSHStoreConfig config{ diff --git a/src/libstore-tests/local-fs-store.cc b/src/libstore-tests/local-fs-store.cc index 22f7ca4084f4..5d8b2f20c7a2 100644 --- a/src/libstore-tests/local-fs-store.cc +++ b/src/libstore-tests/local-fs-store.cc @@ -25,7 +25,7 @@ struct TestLocalFSStoreConfig : LocalFSStoreConfig { } - ref openStore() const override + ref openStore(const SecretContext & context) const override { unreachable(); } diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 95af20784c8e..49929a31f297 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -46,6 +46,7 @@ sources = files( 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-netrc.cc', 'filetransfer-request.cc', 'filetransfer-retry.cc', 'http-binary-cache-store.cc', diff --git a/src/libstore-tests/store-open.cc b/src/libstore-tests/store-open.cc index cadeb3f040a4..22c065f70213 100644 --- a/src/libstore-tests/store-open.cc +++ b/src/libstore-tests/store-open.cc @@ -4,6 +4,7 @@ #include "nix/store/store-reference.hh" #include "nix/store/local-store.hh" #include "nix/store/globals.hh" +#include "nix/store/tests/secret-resolver.hh" #include "nix/util/file-system.hh" #include "nix/util/finally.hh" @@ -59,4 +60,21 @@ TEST(StoreOpen, resolveStoreConfig_auto_withParams) EXPECT_EQ(localConfig->getStateDir(), stateDir); } +TEST(StoreOpen, defaultSubstitutersDoNotRetainOperationResolver) +{ + std::weak_ptr weakResolver; + + { + auto resolver = std::make_shared( + [](const SecretRequest &) -> std::optional { return std::nullopt; }); + weakResolver = resolver; + + /* Unit tests configure no substituters. The cache must still not keep + its operation-scoped key alive after this call returns. */ + EXPECT_TRUE(getDefaultSubstituters(SecretContext{.secretResolver = resolver}).empty()); + } + + EXPECT_TRUE(weakResolver.expired()); +} + } // namespace nix diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index 32122f5c2b86..24e524d0c532 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -4,6 +4,9 @@ namespace nix { +static_assert(requires(RemoteStore & store, std::shared_ptr evalStore) { store.getBuilder(evalStore); }); +static_assert(requires(UDSRemoteStore & store, std::shared_ptr evalStore) { store.getBuilder(evalStore); }); + TEST(UDSRemoteStore, storeDir_absolutePath) { std::filesystem::path storeDir = diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index a288bee9e287..9d32fc6335ef 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -77,7 +77,7 @@ TEST_F(WorkerSubstitutionTest, singleStoreObject) ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter)); // Create a worker with our custom substituter - Worker worker{*dummyStore, *dummyStore}; + Worker worker{*dummyStore, *dummyStore, {}}; // Override the substituters to use our dummy store substituter ref substituerAsStore = substituter; @@ -149,7 +149,7 @@ TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject) ASSERT_FALSE(dummyStore->isValidPath(mainPath)); // Create a worker with our custom substituter - Worker worker{*dummyStore, *dummyStore}; + Worker worker{*dummyStore, *dummyStore, {}}; // Override the substituters to use our dummy store substituter ref substituterAsStore = substituter; @@ -235,7 +235,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); // Create a worker with our custom substituter - Worker worker{*dummyStore, *dummyStore}; + Worker worker{*dummyStore, *dummyStore, {}}; // Override the substituters to use our dummy store substituter ref substituterAsStore = substituter; @@ -404,7 +404,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) ASSERT_FALSE(dummyStore->queryRealisation(resolvedRootDrvOutput)); // Create a worker with our custom substituter - Worker worker{*dummyStore, *dummyStore}; + Worker worker{*dummyStore, *dummyStore, {}}; // Override the substituters to use our dummy store substituter ref substituterAsStore = substituter; diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 621e28eb553b..059b17ab072d 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -911,7 +911,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( * daemon thread. Ideally we should reuse the same * Worker to share scheduling state. */ - Worker freshWorker{goal.worker.store, goal.worker.evalStore}; + Worker freshWorker{goal.worker.store, goal.worker.evalStore, {}}; auto builder = makeRestrictedBuilder(freshWorker, context); daemon::processConnection( store, std::move(from), std::move(to), NotTrusted, recursiveFlag, builder.get_ptr()); @@ -957,6 +957,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( .defaultPathsInChroot = std::move(defaultPathsInChroot), .systemFeatures = worker.store.config.systemFeatures.get(), .desugaredEnv = std::move(desugaredEnv), + .secretResolver = worker.buildContext.secretResolver, }; /* If we have to wait and retry (see below), then `builder` will diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6ff1ef12f745..02aff79968f9 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -16,7 +16,7 @@ namespace nix { -Worker::Worker(Store & store, Store & evalStore) +Worker::Worker(Store & store, Store & evalStore, SecretContext buildContext) /* Can't use make_ref, because the constructor is private. */ : wakerState(ref(new Waker{})) , act(*logger, actRealise) @@ -27,9 +27,12 @@ Worker::Worker(Store & store, Store & evalStore) #endif , store(store) , evalStore(evalStore) + , buildContext(std::move(buildContext)) , settings(nix::settings.getWorkerSettings()) - , getSubstituters{[] { - return nix::settings.getWorkerSettings().useSubstitutes ? getDefaultSubstituters() : std::list>{}; + , getSubstituters{[resolver = this->buildContext.secretResolver] { + return nix::settings.getWorkerSettings().useSubstitutes + ? getDefaultSubstituters(SecretContext{.secretResolver = resolver}) + : std::list>{}; }} { #ifdef _WIN32 diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 30d1c9a6d95e..972897b638f6 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -4,21 +4,83 @@ #include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/util/file-system.hh" +#include "nix/util/util.hh" namespace nix { -static void builtinFetchurl(const BuiltinBuilderContext & ctx) +namespace { + +/** + * The netrc that the parent process copied into the sandbox, served to this + * fetch's transfers. + * + * A build has no broker to lease from, so the lease here is plain ownership: + * the file is written once, shared by every transfer of the fetch, and + * unlinked once the last of them lets go of it. + */ +class SandboxNetrcFile : public SecretFile +{ +public: + SandboxNetrcFile(std::filesystem::path path, std::string_view data) + : filePath(std::move(path)) + { + writeFile(filePath, data, 0600); + } + + ~SandboxNetrcFile() override + { + try { + deletePath(filePath); + } catch (...) { + ignoreExceptionInDestructor(); + } + } + + const std::filesystem::path & path() const noexcept override + { + return filePath; + } + +private: + std::filesystem::path filePath; +}; + +class SandboxNetrcResolver : public SecretResolver { - /* Make the host's netrc data available. Too bad curl requires - this to be stored in a file. It would be nice if we could just - pass a pointer to the data. */ - if (ctx.netrcData != "") { - fileTransferSettings.netrcFile = ctx.tmpDirInSandbox / "netrc"; - writeFile(fileTransferSettings.netrcFile.get(), ctx.netrcData, 0600); +public: + SandboxNetrcResolver(std::filesystem::path path, std::string_view data) + : netrc(make_ref(std::move(path), data)) + { + } + + std::optional resolve(const SecretRequest & request) override + { + if (request.name != "netrc") + return std::nullopt; + if (request.representation != SecretRepresentation::MaterialisedFile) + throw Error("the sandboxed netrc can only be served as a file"); + return ResolvedSecret{.value = netrc}; } +private: + ref netrc; +}; + +} // namespace + +static void builtinFetchurl(const BuiltinBuilderContext & ctx) +{ + /* Make the host's netrc data available to this fetch's transfers. Too + bad curl requires this to be stored in a file. It would be nice if we + could just pass a pointer to the data. */ + FileTransferContext transferContext; + if (ctx.netrcData) + transferContext.secretResolver = + std::make_shared(ctx.tmpDirInSandbox / "netrc", *ctx.netrcData); + + /* Same for the CA bundle: the sandbox has no certificates of its own, + so every request below is pointed at the parent's copy. */ auto caFilePath = ctx.tmpDirInSandbox / "ca-certificates.crt"; - fileTransferSettings.caFile = std::optional{caFilePath}; writeFile(caFilePath, ctx.caFileData, 0600); auto out = get(ctx.drv.outputs, "out"); @@ -41,6 +103,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) auto source = sinkToSource([&](Sink & sink) { FileTransferRequest request(VerbatimURL{url}); request.decompress = false; + request.caFile = caFilePath; #if NIX_WITH_AWS_AUTH // Use pre-resolved credentials if available @@ -56,7 +119,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? CompressionAlgo::xz : CompressionAlgo::none, sink); - fileTransfer->download(std::move(request), *decompressor); + fileTransfer->download(transferContext, std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 4d48566f04e6..d061297bfd77 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -124,9 +124,9 @@ class WholeStoreViewAccessor : public SourceAccessor } // namespace -ref DummyStoreConfig::openStore() const +ref DummyStoreConfig::openStore(const SecretContext & context) const { - return openDummyStore(); + return openDummyStore(context); } bool DummyStoreConfig::getReadOnly() const @@ -151,8 +151,8 @@ struct DummyStoreImpl : DummyStore */ ref wholeStoreView = make_ref(); - DummyStoreImpl(ref config) - : Store{*config} + DummyStoreImpl(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} , DummyStore{config} { wholeStoreView->setPathDisplay(config->storeDir); @@ -394,9 +394,9 @@ struct DummyStoreImpl : DummyStore void DummyStoreImpl::anchor() {} -ref DummyStore::Config::openDummyStore() const +ref DummyStore::Config::openDummyStore(const SecretContext & context) const { - return make_ref(ref{shared_from_this()}); + return make_ref(ref{shared_from_this()}, context); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index e1b9eaeca4b0..8c2044493b16 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -4,6 +4,7 @@ #include "nix/util/config-global.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" +#include "nix/util/file-system.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" @@ -134,6 +135,21 @@ struct curlMultiError final : CloneableError } }; +/** + * Host that `request` authenticates against, for scoping the netrc lookup. + */ +std::optional netrcHost(const FileTransferRequest & request) +{ + try { + if (auto & authority = request.uri.parsed().authority) + return authority->host; + } catch (BadURL &) { + /* Leave it to curl to report the malformed URL. An unscoped lookup + is the right fallback: it is what the setting would have given. */ + } + return std::nullopt; +} + /* Check if the linked libcurl was built with HTTP3 support. */ bool curlSupportsHttp3() { @@ -143,6 +159,63 @@ bool curlSupportsHttp3() } // namespace +NetrcFile resolveNetrcFile( + const FileTransferContext & context, const FileTransferSettings & settings, const FileTransferRequest & request) +{ + if (context.secretResolver) { + auto secret = context.secretResolver->resolve( + SecretRequest{ + .name = "netrc", + .representation = SecretRepresentation::MaterialisedFile, + .purpose = + { + .consumer = "file-transfer", + .operation = std::string(request.operation()), + .host = netrcHost(request), + }, + }); + if (secret) { + auto * file = std::get_if>(&secret->value); + if (!file) + throw Error( + "secret resolver returned the 'netrc' secret inline, but curl can only read one from a file"); + return {.path = (*file)->path(), .lease = *file}; + } + } + + return {.path = settings.netrcFile.get()}; +} + +std::optional resolveNetrcData( + const std::shared_ptr & secretResolver, + const FileTransferSettings & settings, + const SecretPurpose & purpose) +{ + if (secretResolver) { + auto secret = secretResolver->resolve( + SecretRequest{ + .name = "netrc", + .representation = SecretRepresentation::Inline, + .purpose = purpose, + }); + if (secret) { + auto * inlineSecret = std::get_if(&secret->value); + if (!inlineSecret) + throw Error( + "secret resolver materialised the 'netrc' secret as a file, " + "but it can only be passed on as data here"); + return inlineSecret->value; + } + } + + try { + return readFile(settings.netrcFile.get()); + } catch (SystemError &) { + /* No netrc configured, which is the common case. */ + return std::nullopt; + } +} + struct curlFileTransfer : public FileTransfer { const FileTransferSettings & settings; @@ -158,6 +231,13 @@ struct curlFileTransfer : public FileTransfer { curlFileTransfer & fileTransfer; FileTransferRequest request; + + /** + * netrc handed to curl, resolved once per transfer so that retries + * reuse one lease rather than taking a fresh one each attempt. + */ + NetrcFile netrc; + FileTransferResult result; std::unique_ptr _act; Callback callback; @@ -245,10 +325,12 @@ struct curlFileTransfer : public FileTransfer TransferItem( curlFileTransfer & fileTransfer, + const FileTransferContext & context, const FileTransferRequest & request, Callback && callback) : fileTransfer(fileTransfer) , request(request) + , netrc(resolveNetrcFile(context, fileTransfer.settings, request)) , callback(std::move(callback)) , finalSink([this](std::string_view data) { if (errorSink) { @@ -649,9 +731,18 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_SEEKDATA, this); } + /* A bundle named by the request wins over the `ssl-cert-file` setting: + builtin:fetchurl carries the host's copy into a sandbox that has no + certificates of its own. */ + const std::filesystem::path * caFile = nullptr; + if (request.caFile) + caFile = &*request.caFile; + else if (auto & configuredCaFile = fileTransfer.settings.caFile.get()) + caFile = &configuredCaFile->path(); + /* Note: libcurl copies string arguments, so temporaries from .string().c_str() are safe. See the comment near CURLOPT_SSLKEY below. */ - if (auto & caFile = fileTransfer.settings.caFile.get()) + if (caFile) curl_easy_setopt(req, CURLOPT_CAINFO, caFile->string().c_str()); #ifdef _WIN32 /* Use native windows certificate store when the option is not specified explicitly. */ @@ -677,7 +768,7 @@ struct curlFileTransfer : public FileTransfer /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().string().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, netrc.path.string().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); if (writtenToSink) @@ -1196,16 +1287,19 @@ struct curlFileTransfer : public FileTransfer return ItemHandle(item.get_ptr()); } - ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback callback) override + ItemHandle enqueueFileTransfer( + const FileTransferContext & context, + const FileTransferRequest & request, + Callback callback) override { /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */ if (request.uri.scheme() == "s3") { auto modifiedRequest = request; modifiedRequest.setupForS3(); - return enqueueItem(make_ref(*this, std::move(modifiedRequest), std::move(callback))); + return enqueueItem(make_ref(*this, context, std::move(modifiedRequest), std::move(callback))); } - return enqueueItem(make_ref(*this, request, std::move(callback))); + return enqueueItem(make_ref(*this, context, request, std::move(callback))); } void unpauseTransfer(std::weak_ptr item) @@ -1300,9 +1394,21 @@ void FileTransferRequest::setupForS3() } std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) +{ + return enqueueFileTransfer(FileTransferContext{}, request); +} + +FileTransfer::ItemHandle +FileTransfer::enqueueFileTransfer(const FileTransferRequest & request, Callback callback) +{ + return enqueueFileTransfer(FileTransferContext{}, request, std::move(callback)); +} + +std::future +FileTransfer::enqueueFileTransfer(const FileTransferContext & context, const FileTransferRequest & request) { auto promise = std::make_shared>(); - enqueueFileTransfer(request, {[promise](std::future fut) { + enqueueFileTransfer(context, request, {[promise](std::future fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -1314,22 +1420,47 @@ std::future FileTransfer::enqueueFileTransfer(const FileTran FileTransferResult FileTransfer::download(const FileTransferRequest & request) { - return enqueueFileTransfer(request).get(); + return download(FileTransferContext{}, request); +} + +FileTransferResult FileTransfer::download(const FileTransferContext & context, const FileTransferRequest & request) +{ + return enqueueFileTransfer(context, request).get(); } FileTransferResult FileTransfer::upload(const FileTransferRequest & request) +{ + return upload(FileTransferContext{}, request); +} + +FileTransferResult FileTransfer::upload(const FileTransferContext & context, const FileTransferRequest & request) { /* Note: this method is the same as download, but helps in readability */ - return enqueueFileTransfer(request).get(); + return enqueueFileTransfer(context, request).get(); } FileTransferResult FileTransfer::deleteResource(const FileTransferRequest & request) { - return enqueueFileTransfer(request).get(); + return deleteResource(FileTransferContext{}, request); +} + +FileTransferResult +FileTransfer::deleteResource(const FileTransferContext & context, const FileTransferRequest & request) +{ + return enqueueFileTransfer(context, request).get(); } void FileTransfer::download( FileTransferRequest && request, Sink & sink, std::function resultCallback) +{ + download(FileTransferContext{}, std::move(request), sink, std::move(resultCallback)); +} + +void FileTransfer::download( + const FileTransferContext & context, + FileTransferRequest && request, + Sink & sink, + std::function resultCallback) { /* Note: we can't call 'sink' via request.dataCallback, because that would cause the sink to execute on the fileTransfer @@ -1389,7 +1520,7 @@ void FileTransfer::download( }; auto handle = enqueueFileTransfer( - request, {[_state, resultCallback{std::move(resultCallback)}](std::future fut) { + context, request, {[_state, resultCallback{std::move(resultCallback)}](std::future fut) { auto state(_state->lock()); state->quit = true; try { diff --git a/src/libstore/freebsd/build/freebsd-derivation-builder.cc b/src/libstore/freebsd/build/freebsd-derivation-builder.cc index f46ebf55ed91..cbe10400b6b1 100644 --- a/src/libstore/freebsd/build/freebsd-derivation-builder.cc +++ b/src/libstore/freebsd/build/freebsd-derivation-builder.cc @@ -363,11 +363,7 @@ void ChrootFreeBSDDerivationBuilder::startChild() { int jid; - RunChildArgs args{ -#if NIX_WITH_AWS_AUTH - .awsCredentials = preResolveAwsCredentials(), -#endif - }; + auto args = makeRunChildArgs(); if (derivationType.isSandboxed()) { jid = jail_setv( diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index d881e696b756..102249acd934 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -59,8 +59,8 @@ std::string HttpBinaryCacheStoreConfig::doc() ; } -HttpBinaryCacheStore::HttpBinaryCacheStore(ref config, ref fileTransfer) - : Store{*config} // TODO it will actually mutate the configuration +HttpBinaryCacheStore::HttpBinaryCacheStore(ref config, ref fileTransfer, SecretContext context) + : Store{*config, std::move(context)} // TODO it will actually mutate the configuration , BinaryCacheStore{*config} , fileTransfer{fileTransfer} , config{config} @@ -172,7 +172,7 @@ bool HttpBinaryCacheStore::fileExists(const std::string & path) try { FileTransferRequest request(makeRequest(path)); request.method = HttpMethod::Head; - fileTransfer->download(request); + fileTransfer->download(FileTransferContext{secretContext.secretResolver}, request); return true; } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the @@ -202,7 +202,7 @@ void HttpBinaryCacheStore::upload( req.data = {sizeHint, source}; req.mimeType = mimeType; - fileTransfer->upload(req); + fileTransfer->upload(FileTransferContext{secretContext.secretResolver}, req); } void HttpBinaryCacheStore::upsertFile( @@ -282,7 +282,7 @@ void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink) checkEnabled(); auto request(makeRequest(path)); try { - fileTransfer->download(std::move(request), sink); + fileTransfer->download(FileTransferContext{secretContext.secretResolver}, std::move(request), sink); } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile( @@ -301,19 +301,21 @@ void HttpBinaryCacheStore::getFile(const std::string & path, CallbackenqueueFileTransfer(request, {[callbackPtr, this](std::future result) { - try { - (*callbackPtr)(std::move(result.get().data)); - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound - || e.error == FileTransfer::Forbidden) - return (*callbackPtr)({}); - maybeDisable(); - callbackPtr->rethrow(); - } catch (...) { - callbackPtr->rethrow(); - } - }}); + fileTransfer->enqueueFileTransfer( + FileTransferContext{secretContext.secretResolver}, + request, + {[callbackPtr, this](std::future result) { + try { + (*callbackPtr)(std::move(result.get().data)); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + return (*callbackPtr)({}); + maybeDisable(); + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); } catch (...) { callbackPtr->rethrow(); @@ -324,7 +326,8 @@ void HttpBinaryCacheStore::getFile(const std::string & path, Callback HttpBinaryCacheStore::getNixCacheInfo() { try { - auto result = fileTransfer->download(makeRequest(cacheInfoFile)); + auto result = + fileTransfer->download(FileTransferContext{secretContext.secretResolver}, makeRequest(cacheInfoFile)); return result.data; } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound) @@ -347,17 +350,18 @@ std::optional HttpBinaryCacheStore::isTrustedClient() return std::nullopt; } -ref HttpBinaryCacheStore::Config::openStore(ref fileTransfer) const +ref HttpBinaryCacheStore::Config::openStore(const SecretContext & context, ref fileTransfer) const { return make_ref( ref{// FIXME we shouldn't actually need a mutable config std::const_pointer_cast(shared_from_this())}, - fileTransfer); + fileTransfer, + context); } -ref HttpBinaryCacheStoreConfig::openStore() const +ref HttpBinaryCacheStoreConfig::openStore(const SecretContext & context) const { - return openStore(getFileTransfer()); + return openStore(context, getFileTransfer()); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 088644a3eb32..6ad9e0b5af75 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -17,6 +17,8 @@ namespace nix { +class SecretResolver; + /** * Rethrow the current exception as a subclass of `Error`. */ @@ -118,6 +120,9 @@ struct DerivationBuilderParams StringSet systemFeatures; DesugaredEnv desugaredEnv; + + /** Resolver owned by the build operation, never by global settings. */ + std::shared_ptr secretResolver; }; /** diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index f30a3146be4e..3dc957554aa7 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -75,9 +75,10 @@ struct HookInstance; class LocalBuilder : public Builder { public: - LocalBuilder(ref store, ref evalStore) + LocalBuilder(ref store, ref evalStore, SecretContext context) : store(store) - , evalStore(evalStore) {}; + , evalStore(evalStore) + , context(std::move(context)) {}; /* Builder interface — see `Builder` for documentation. */ @@ -95,11 +96,12 @@ private: */ inline std::shared_ptr getWorker() { - return std::make_shared(*store, *evalStore); + return std::make_shared(*store, *evalStore, context); } ref store; ref evalStore; + SecretContext context; }; /** @@ -235,6 +237,9 @@ public: Store & store; Store & evalStore; + /** Dependencies scoped to the current LocalBuilder operation. */ + const SecretContext buildContext; + const WorkerSettings & settings; /** @@ -269,7 +274,7 @@ public: */ bool tryBuildHook = true; - Worker(Store & store, Store & evalStore); + Worker(Store & store, Store & evalStore, SecretContext buildContext); ~Worker(); /** diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index e2caba3f1839..dc6472a6e197 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -14,7 +14,7 @@ struct BuiltinBuilderContext { const BasicDerivation & drv; std::map outputs; - std::string netrcData; + std::optional netrcData; std::string caFileData; Strings hashedMirrors; std::filesystem::path tmpDirInSandbox; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index bec77a6bee68..0a1a805f0812 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -56,7 +56,9 @@ public: boost::concurrent_flat_map> buildTrace; DummyStore(ref config) - : Store{*config} + /* Abstract, so the most-derived store initialises the virtual `Store` + base and this argument is never the one that survives. */ + : Store{*config, SecretContext{}} , config(config) { } diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index c8a212c75603..96efa868d375 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -56,9 +56,9 @@ public: /** * Same as `openStore`, just with a more precise return type. */ - ref openDummyStore() const; + ref openDummyStore(const SecretContext & context = {}) const; - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override { diff --git a/src/libstore/include/nix/store/filetransfer-impl.hh b/src/libstore/include/nix/store/filetransfer-impl.hh index bac68baf55bd..2e9e3faaa1ef 100644 --- a/src/libstore/include/nix/store/filetransfer-impl.hh +++ b/src/libstore/include/nix/store/filetransfer-impl.hh @@ -9,12 +9,67 @@ #include #include #include +#include #include +#include #include #include +#include + +#include "nix/store/filetransfer.hh" namespace nix { +/** + * The netrc file curl should read credentials from, together with the lease + * that keeps a resolver-provided one alive. + */ +struct NetrcFile +{ + std::filesystem::path path; + + /** + * Holds the resolver's materialisation open. Null when `path` came from + * the `netrc-file` setting, which nothing needs to keep alive. + */ + std::shared_ptr lease; +}; + +/** + * Pick the netrc file for one transfer. + * + * A resolver owning the transfer is asked first, so a broker can hand back + * credentials scoped to the host being contacted rather than the whole of + * the user's netrc. Without a resolver, or when it holds no netrc, the + * `netrc-file` setting applies exactly as before. curl treats a nonexistent + * path as "no netrc", so the unconfigured case needs no handling here. + * + * @throws Error if the resolver answers with an inline secret, which curl + * has no way to consume. + */ +NetrcFile resolveNetrcFile( + const FileTransferContext & context, const FileTransferSettings & settings, const FileTransferRequest & request); + +/** + * The netrc contents for a consumer that cannot be handed a file, such as a + * sandboxed build that has to carry the bytes across a fork. + * + * Precedence matches resolveNetrcFile(): the resolver first, then the + * `netrc-file` setting, then nothing at all. Unlike the file case there is + * no host to scope by, since one netrc has to serve every URL the consumer + * goes on to try. + * + * An engaged empty string is an explicit empty override. `std::nullopt` + * means that neither the resolver nor the configured file supplied data. + * + * @throws Error if the resolver answers with a materialised file, which + * cannot be passed on as data. + */ +std::optional resolveNetrcData( + const std::shared_ptr & secretResolver, + const FileTransferSettings & settings, + const SecretPurpose & purpose); + /** * Clamped exponential growth: base * 2^(attempt-1), capped at ceil. * Shift is clamped at 31 and the intermediate is widened to uint64_t diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 38f13af5d23c..fd12cb518594 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -14,6 +14,7 @@ #include "nix/util/url.hh" #include "nix/store/config.hh" +#include "nix/store/secret-resolver.hh" #if NIX_WITH_AWS_AUTH # include "nix/store/aws-creds.hh" #endif @@ -23,6 +24,12 @@ namespace nix { const std::filesystem::path & nixConfDir(); +/** Dependencies that may vary between otherwise shared file transfers. */ +struct FileTransferContext +{ + std::shared_ptr secretResolver; +}; + class FileTransferSettings : public Config { static std::optional getDefaultSSLCertFile(); @@ -263,6 +270,13 @@ struct FileTransferRequest std::optional retryMaxDelayMs; std::optional retryAttempts; + /** + * Optional path to a file of CA certificates, used to verify the server + * instead of the ones named by the `ssl-cert-file` setting. Only used for + * TLS-based protocols. + */ + std::optional caFile; + /** * Optional path to the client certificate in "PEM" format. Only used for TLS-based protocols. */ @@ -328,6 +342,27 @@ struct FileTransferRequest */ std::string displayUri() const; + /** + * Stable name for the kind of transfer, used as + * `SecretPurpose::operation` so a secret broker can scope credentials + * to it. Unlike `verb()` this is not a log message: rewording a + * progress line must not change what a broker matches on. + */ + std::string_view operation() const + { + switch (method) { + case HttpMethod::Head: + case HttpMethod::Get: + return "download"; + case HttpMethod::Put: + case HttpMethod::Post: + return "upload"; + case HttpMethod::Delete: + return "delete"; + } + unreachable(); + } + /** * Returns the method description for logging purposes. */ @@ -438,8 +473,12 @@ public: * the download. The future may throw a FileTransferError * exception. */ - virtual ItemHandle - enqueueFileTransfer(const FileTransferRequest & request, Callback callback) = 0; + virtual ItemHandle enqueueFileTransfer( + const FileTransferContext & context, + const FileTransferRequest & request, + Callback callback) = 0; + + ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback callback); /** * Unpause a transfer that has been previously paused by a dataCallback. @@ -448,21 +487,30 @@ public: std::future enqueueFileTransfer(const FileTransferRequest & request); + std::future + enqueueFileTransfer(const FileTransferContext & context, const FileTransferRequest & request); + /** * Synchronously download a file. */ FileTransferResult download(const FileTransferRequest & request); + FileTransferResult download(const FileTransferContext & context, const FileTransferRequest & request); + /** * Synchronously upload a file. */ FileTransferResult upload(const FileTransferRequest & request); + FileTransferResult upload(const FileTransferContext & context, const FileTransferRequest & request); + /** * Synchronously delete a resource. */ FileTransferResult deleteResource(const FileTransferRequest & request); + FileTransferResult deleteResource(const FileTransferContext & context, const FileTransferRequest & request); + /** * Download a file, writing its data to a sink. The sink will be * invoked on the thread of the caller. @@ -470,6 +518,12 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); + void download( + const FileTransferContext & context, + FileTransferRequest && request, + Sink & sink, + std::function resultCallback = {}); + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 6557b9efcaa1..2d66c9455369 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -83,9 +83,9 @@ public: static std::string doc(); - ref openStore(ref fileTransfer) const; + ref openStore(const SecretContext & context, ref fileTransfer) const; - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; }; @@ -112,7 +112,8 @@ public: ref config; - HttpBinaryCacheStore(ref config, ref fileTransfer = getFileTransfer()); + HttpBinaryCacheStore( + ref config, ref fileTransfer = getFileTransfer(), SecretContext context = {}); void init() override; diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index bf1532b8ba25..6a1c8b76d44b 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -59,7 +59,7 @@ public: static std::string doc(); - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; }; @@ -80,7 +80,7 @@ public: SSHMaster master; - LegacySSHStore(ref); + LegacySSHStore(ref, SecretContext secretContext); ref openConnection(); @@ -139,7 +139,9 @@ public: public: - ref getBuilder(std::shared_ptr evalStore) override; + using Store::getBuilder; + + ref getBuilder(const SecretContext & context, std::shared_ptr evalStore) override; ref getFSAccessor(bool requireValidPath) override { diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 181b33e4bdf8..fae05d6bf300 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -36,7 +36,7 @@ public: static std::string doc(); - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; }; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 60cde8380366..2a6d20c996f7 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -90,7 +90,7 @@ public: static std::string doc(); - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; @@ -120,7 +120,7 @@ struct LocalOverlayStore : virtual LocalStore ref config; - LocalOverlayStore(ref); + LocalOverlayStore(ref, SecretContext secretContext); private: void anchor() override; diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 36be56379ac7..ae0068eeb051 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -172,7 +172,7 @@ public: static std::string doc(); - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; }; @@ -263,6 +263,8 @@ public: */ LocalStore(ref params); + LocalStore(ref params, SecretContext secretContext); + ~LocalStore(); /** diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index aa966c27c4c7..d29a5eeb2467 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -86,6 +86,7 @@ headers = [ config_pub_h ] + files( 'restricted-store.hh', 's3-binary-cache-store.hh', 's3-url.hh', + 'secret-resolver.hh', 'serve-protocol-connection.hh', 'serve-protocol-impl.hh', 'serve-protocol.hh', diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index b357b78f9421..2b1b929b0af5 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -126,7 +126,9 @@ public: void queryRealisationUncached( const DrvOutput &, Callback> callback) noexcept override; - ref getBuilder(std::shared_ptr evalStore) override; + using Store::getBuilder; + + ref getBuilder(const SecretContext & context, std::shared_ptr evalStore) override; void addTempRoot(const StorePath & path) override; diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index f4a068e4473d..5cd6972fcba3 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -150,7 +150,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig std::string getHumanReadableURI() const override; - ref openStore() const override; + ref openStore(const SecretContext & context) const override; }; } // namespace nix diff --git a/src/libstore/include/nix/store/secret-resolver.hh b/src/libstore/include/nix/store/secret-resolver.hh new file mode 100644 index 000000000000..a87cf98f5b94 --- /dev/null +++ b/src/libstore/include/nix/store/secret-resolver.hh @@ -0,0 +1,117 @@ +#pragma once +///@file + +#include "nix/util/types.hh" +#include "nix/util/ref.hh" + +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + +enum class SecretRepresentation { + Inline, + MaterialisedFile, +}; + +struct SecretPurpose +{ + std::string consumer; + std::string operation; + std::optional host; + std::optional path; + + auto operator<=>(const SecretPurpose &) const = default; +}; + +struct SecretRequest +{ + std::string name; + SecretRepresentation representation; + SecretPurpose purpose; + + auto operator<=>(const SecretRequest &) const = default; +}; + +struct InlineSecret +{ + std::string value; +}; + +/** + * A materialised secret file and the lease that keeps it alive. + * + * Implementations release the broker-side lease and remove any associated + * materialisation from their destructor. Consumers must retain this object for + * as long as they use `path()`; a bare path must never outlive the object. + */ +class SecretFile +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + +public: + virtual ~SecretFile() = default; + + virtual const std::filesystem::path & path() const noexcept = 0; +}; + +struct ResolvedSecret +{ + std::variant> value; + std::optional expiresAt; +}; + +/** + * Resolve named secrets for one explicitly owned operation context. + * + * Implementations may keep instance-local transport or provider state, but + * callers must not rely on a process-global resolver or cache. + * + * One resolver serves a whole operation, and an operation is not one thread: + * a single store copy has as many transfers in flight as it has connections, + * each resolving on its own thread. Implementations must therefore make + * `resolve` safe to call concurrently on the same instance. + */ +class SecretResolver +{ +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor(); + +public: + virtual ~SecretResolver() = default; + + /** + * Resolve `request`, or return `std::nullopt` when this resolver knows + * of no secret under that name. + * + * Absence is an ordinary answer rather than a failure. Some consumers + * (netrc lookups, say) are expected to carry on without the secret, so + * they must be able to tell "nothing is provisioned" apart from "the + * broker could not be reached". Implementations throw only for the latter. + */ + virtual std::optional resolve(const SecretRequest & request) = 0; +}; + +/** + * The secret-resolving authority handed to one operation, or none. + * + * File transfers, builds, and store opens differ in how they use this + * authority, not in the authority itself. A single context avoids repacking + * structurally identical resolver holders at every layer boundary. + */ +struct SecretContext +{ + std::shared_ptr secretResolver; +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 324e85eb60f4..914196c35635 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -40,7 +40,7 @@ public: static std::string doc(); - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; }; @@ -71,7 +71,7 @@ public: return ExperimentalFeature::MountedSSHStore; } - ref openStore() const override; + ref openStore(const SecretContext & context) const override; }; } // namespace nix diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 06122cc746cd..9f7763fcd2d8 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/store/secret-resolver.hh" #include "nix/store/outputs-spec.hh" #include "nix/store/path.hh" #include "nix/store/derived-path.hh" @@ -357,7 +358,7 @@ public: * Open a store of the type corresponding to this configuration * type. */ - virtual ref openStore() const = 0; + virtual ref openStore(const SecretContext & context) const = 0; /** * Render the config back to a `StoreReference`. It should round-trip @@ -453,7 +454,26 @@ protected: std::shared_ptr diskCache; - Store(const Store::Config & config); + /** + * Authority this store may use for requests it makes itself, and that it + * propagates to substituters it opens. + * + * Taken at construction: `Store` is a virtual base, so the most-derived + * store states its authority once, and there is no window in which a store + * is usable without having stated it. + */ + const SecretContext secretContext; + +public: + /** Read-only view for code that opens substituters on this store's behalf. */ + const SecretContext & getSecretContext() const + { + return secretContext; + } + +protected: + + Store(const Store::Config & config, SecretContext secretContext); public: /** @@ -470,8 +490,20 @@ public: * @param evalStore If provided and different from this store, * derivation files will be copied from the eval store to this * store before building. + * + * Uses the authority supplied when this store was opened. + * + * @note Not virtual: it delegates to the `SecretContext` overload, which is + * the one implementations override. Overriding only this one would leave + * every context-aware call site bypassing the override. + */ + ref getBuilder(std::shared_ptr evalStore = nullptr); + + /** + * Get a builder with dependencies scoped to this build operation. + * Remote stores must not forward the resolver to another trust domain. */ - virtual ref getBuilder(std::shared_ptr evalStore = nullptr); + virtual ref getBuilder(const SecretContext & context, std::shared_ptr evalStore = nullptr); /** * Follow symlinks until we end up with a path in the Nix store. diff --git a/src/libstore/include/nix/store/store-open.hh b/src/libstore/include/nix/store/store-open.hh index ef7d81675eca..a1d54b0ff158 100644 --- a/src/libstore/include/nix/store/store-open.hh +++ b/src/libstore/include/nix/store/store-open.hh @@ -23,24 +23,44 @@ ref resolveStoreConfig(StoreReference && storeURI); /** * @return a Store object to access the Nix store denoted by * ‘uri’ (slight misnomer...). + * + * The `SecretContext` overloads hand the opened store the authority of the + * opening process, so that stores which make requests themselves (the HTTP and + * S3 binary caches) can resolve configured credentials. Callers that + * hold no such authority pass an empty context explicitly. */ +ref openStore(const SecretContext & context, StoreReference && storeURI); + ref openStore(StoreReference && storeURI); /** * Opens the store at `uri`, where `uri` is in the format expected by * `StoreReference::parse` */ +ref openStore( + const SecretContext & context, + const std::string & uri, + const StoreReference::Params & extraParams = StoreReference::Params()); + ref openStore(const std::string & uri, const StoreReference::Params & extraParams = StoreReference::Params()); /** * Short-hand which opens the default store, according to global settings */ +ref openStore(const SecretContext & context); + ref openStore(); /** * @return the default substituter stores, defined by the * ‘substituters’ option and various legacy options. + * + * @note The list is cached for the most recently used resolver. A caller with + * different authority rebuilds it, so substituter stores never inherit the + * resolver selected by an unrelated caller. */ +std::list> getDefaultSubstituters(const SecretContext & context); + std::list> getDefaultSubstituters(); } // namespace nix diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 8359f8fdf45b..c1bb02e32748 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -54,7 +54,7 @@ public: return {"unix"}; } - ref openStore() const override; + ref openStore(const SecretContext & context) const override; StoreReference getReference() const override; }; @@ -69,7 +69,7 @@ public: ref config; - UDSRemoteStore(ref); + UDSRemoteStore(ref, SecretContext secretContext); ref getFSAccessor(bool requireValidPath = true) override { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1765c5af747a..95298cd173d6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -90,8 +90,8 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection void LegacySSHStore::anchor() {} -LegacySSHStore::LegacySSHStore(ref config) - : Store{*config} +LegacySSHStore::LegacySSHStore(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} , config{config} , connections( make_ref>( @@ -387,8 +387,9 @@ LegacySSHBuilder::buildPathsWithResults(const std::vector & reqs, B return results; } -ref LegacySSHStore::getBuilder(std::shared_ptr evalStore) +ref LegacySSHStore::getBuilder(const SecretContext &, std::shared_ptr evalStore) { + /* Secret resolution is local to the process that performs the build. */ if (evalStore && evalStore.get() != this) throw Error("building on an SSH store is incompatible with '--eval-store'"); return make_ref( @@ -465,9 +466,9 @@ std::optional LegacySSHStore::isTrustedClient() return std::nullopt; } -ref LegacySSHStore::Config::openStore() const +ref LegacySSHStore::Config::openStore(const SecretContext & context) const { - return make_ref(ref{shared_from_this()}); + return make_ref(ref{shared_from_this()}, context); } static RegisterStoreImplementation regLegacySSHStore; diff --git a/src/libstore/linux/build/linux-derivation-builder.cc b/src/libstore/linux/build/linux-derivation-builder.cc index 525b3ffa1632..5c0e0ee3425c 100644 --- a/src/libstore/linux/build/linux-derivation-builder.cc +++ b/src/libstore/linux/build/linux-derivation-builder.cc @@ -505,11 +505,7 @@ void ChrootLinuxDerivationBuilder::prepareSandbox() void ChrootLinuxDerivationBuilder::startChild() { - RunChildArgs args{ -#if NIX_WITH_AWS_AUTH - .awsCredentials = preResolveAwsCredentials(), -#endif - }; + auto args = makeRunChildArgs(); /* Set up private namespaces for the build: diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 4d79f613e4a5..0e2721ec2117 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -60,8 +60,8 @@ struct LocalBinaryCacheStore : virtual BinaryCacheStore ref config; - LocalBinaryCacheStore(ref config) - : Store{*config} + LocalBinaryCacheStore(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} , BinaryCacheStore{*config} , config{config} { @@ -147,11 +147,12 @@ void LocalBinaryCacheStoreConfig::anchor() {} void LocalBinaryCacheStore::anchor() {} -ref LocalBinaryCacheStoreConfig::openStore() const +ref LocalBinaryCacheStoreConfig::openStore(const SecretContext & context) const { auto store = make_ref( ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + std::const_pointer_cast(shared_from_this())}, + context); store->init(); return store; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index a6fd24bf34e6..1bfe4abfc249 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -25,7 +25,9 @@ LocalFSStoreConfig::LocalFSStoreConfig(const std::filesystem::path & rootDir, co } LocalFSStore::LocalFSStore(const Config & config) - : Store{static_cast(*this)} + /* Abstract, so the most-derived store initialises the virtual `Store` base + and this argument is never the one that survives. */ + : Store{static_cast(*this), SecretContext{}} , config{config} { } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index d3b0ccc77a73..8c2480f1944c 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -21,10 +21,10 @@ std::string LocalOverlayStoreConfig::doc() ; } -ref LocalOverlayStoreConfig::openStore() const +ref LocalOverlayStoreConfig::openStore(const SecretContext & context) const { return make_ref( - ref{std::dynamic_pointer_cast(shared_from_this())}); + ref{std::dynamic_pointer_cast(shared_from_this())}, context); } StoreReference LocalOverlayStoreConfig::getReference() const @@ -42,12 +42,13 @@ std::filesystem::path LocalOverlayStoreConfig::toUpperPath(const StorePath & pat return upperLayer.get() / path.to_string(); } -LocalOverlayStore::LocalOverlayStore(ref config) - : Store{*config} - , LocalFSStore{*config} - , LocalStore{static_cast>(config)} +LocalOverlayStore::LocalOverlayStore(ref config, SecretContext secretContext) + /* Copied, not moved: the lower store is opened with the same authority. */ + : Store{*config, secretContext} + , LocalFSStore{*config} /* The `Store` base is initialised above; this argument is discarded. */ + , LocalStore{static_cast>(config), SecretContext{}} , config{config} - , lowerStore(openStore(config->lowerStoreUri.get()).dynamic_pointer_cast()) + , lowerStore(openStore(secretContext, config->lowerStoreUri.get()).dynamic_pointer_cast()) { if (!config->upperLayer.isOverridden()) throw Error("overlay store at %s requires the 'upper-layer' setting", PathFmt(config->realStoreDir.get())); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index edacae79c4e2..a32a41f3eba1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -91,9 +91,9 @@ std::filesystem::path LocalBuildStoreConfig::getBuildDir() const : AbsolutePath{stateDir.get() / "builds"}; } -ref LocalStore::Config::openStore() const +ref LocalStore::Config::openStore(const SecretContext & context) const { - return make_ref(ref{shared_from_this()}); + return make_ref(ref{shared_from_this()}, context); } bool LocalStoreConfig::getDefaultRequireSigs() @@ -123,7 +123,12 @@ struct LocalStore::State::Stmts }; LocalStore::LocalStore(ref config) - : Store{*config} + : LocalStore(std::move(config), {}) +{ +} + +LocalStore::LocalStore(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} , LocalFSStore{*config} , config{config} , _state(make_ref>()) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d0248b423b43..bb50e5eaaa2b 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -328,6 +328,7 @@ sources = files( 'restricted-store.cc', 's3-binary-cache-store.cc', 's3-url.cc', + 'secret-resolver.cc', 'serve-protocol-connection.cc', 'serve-protocol.cc', 'sqlite.cc', diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 67822e170bf0..c0cf473a9052 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -95,7 +95,7 @@ querySubstitutablePathInfosAsync(Store & store, const StorePathCAMap & paths, Su co_await forEachAsync(paths, [&store, &infos](auto path) -> asio::awaitable { std::optional lastStoresException = std::nullopt; - for (auto & sub : getDefaultSubstituters()) { + for (auto & sub : getDefaultSubstituters(store.getSecretContext())) { if (lastStoresException.has_value()) { logError(lastStoresException->info()); lastStoresException.reset(); @@ -248,7 +248,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) continue; bool found = false; - for (auto & sub : getDefaultSubstituters()) { + for (auto & sub : getDefaultSubstituters(secretContext)) { /* TODO: Asyncify this. */ auto realisation = sub->queryRealisation({drvPath, outputName}); if (!realisation) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4c56dc8305c7..a22bcb44d99a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -36,7 +36,9 @@ void RemoteStoreConfig::anchor() {} /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) - : Store{config} + /* Abstract, so the most-derived store initialises the virtual `Store` base + and this argument is never the one that survives. */ + : Store{config, SecretContext{}} , config{config} , connections( make_ref>( @@ -739,8 +741,10 @@ void RemoteBuilder::repairPath(const StorePath & path) throw Unsupported("operation 'repairPath' is not supported by store '%s'", store->config.getHumanReadableURI()); } -ref RemoteStore::getBuilder(std::shared_ptr evalStore) +ref RemoteStore::getBuilder(const SecretContext &, std::shared_ptr evalStore) { + /* A resolver is process-local authority. The remote endpoint must create + its own build context instead of receiving ours over the store protocol. */ return make_ref( ref(std::dynamic_pointer_cast(shared_from_this())), std::move(evalStore)); } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 68d38525eb17..24c35f05d46b 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -56,7 +56,9 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor RestrictionContext & goal; RestrictedStore(ref config, ref next, RestrictionContext & goal) - : Store{*config} + /* No authority by design: a recursive-nix store must not resolve + secrets on behalf of the build that owns it. */ + : Store{*config, SecretContext{}} , LocalFSStore{*config} , config{config} , next(next) @@ -162,7 +164,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor return NotTrusted; } - ref getBuilder(std::shared_ptr evalStore) override + ref getBuilder(const SecretContext & context, std::shared_ptr evalStore) override { unreachable(); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 9abc73882c37..4ce7b99bfcc1 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -26,8 +26,8 @@ static constexpr uint64_t AWS_MAX_PART_COUNT = 10000; class S3BinaryCacheStore : public virtual HttpBinaryCacheStore { public: - S3BinaryCacheStore(ref config) - : Store{*config} + S3BinaryCacheStore(ref config, SecretContext context) + : Store{*config, context} , BinaryCacheStore{*config} , HttpBinaryCacheStore{config} , s3Config{config} @@ -324,7 +324,7 @@ std::string S3BinaryCacheStore::createMultipartUpload( std::move(headers->begin(), headers->end(), std::back_inserter(req.headers)); } - auto result = fileTransfer->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(FileTransferContext{secretContext.secretResolver}, req).get(); std::regex uploadIdRegex("([^<]+)"); std::smatch match; @@ -355,7 +355,7 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, req.data = {payload}; req.mimeType = "application/octet-stream"; - auto result = fileTransfer->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(FileTransferContext{secretContext.secretResolver}, req).get(); if (result.etag.empty()) { throw Error("S3 UploadPart response missing ETag for part %d", partNumber); @@ -376,7 +376,7 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_ req.uri = VerbatimURL(url); req.method = HttpMethod::Delete; - (void) fileTransfer->enqueueFileTransfer(req).get(); + (void) fileTransfer->enqueueFileTransfer(FileTransferContext{secretContext.secretResolver}, req).get(); } catch (...) { ignoreExceptionInDestructor(); } @@ -409,7 +409,7 @@ void S3BinaryCacheStore::completeMultipartUpload( req.data = {payload}; req.mimeType = "text/xml"; - (void) fileTransfer->enqueueFileTransfer(req).get(); + (void) fileTransfer->enqueueFileTransfer(FileTransferContext{secretContext.secretResolver}, req).get(); debug("S3 multipart upload completed: %d parts uploaded for '%s'", partEtags.size(), key); } @@ -483,11 +483,11 @@ std::string S3BinaryCacheStoreConfig::doc() ; } -ref S3BinaryCacheStoreConfig::openStore() const +ref S3BinaryCacheStoreConfig::openStore(const SecretContext & context) const { auto sharedThis = std::const_pointer_cast( std::static_pointer_cast(shared_from_this())); - return make_ref(ref{sharedThis}); + return make_ref(ref{sharedThis}, context); } static RegisterStoreImplementation registerS3BinaryCacheStore; diff --git a/src/libstore/secret-resolver.cc b/src/libstore/secret-resolver.cc new file mode 100644 index 000000000000..8b30d8ab6269 --- /dev/null +++ b/src/libstore/secret-resolver.cc @@ -0,0 +1,9 @@ +#include "nix/store/secret-resolver.hh" + +namespace nix { + +void SecretFile::anchor() {} + +void SecretResolver::anchor() {} + +} // namespace nix diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 4ecf2780e7b2..d5ed2698a121 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -51,8 +51,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ ref config; - SSHStore(ref config) - : Store{*config} + SSHStore(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} , RemoteStore{*config} , config{config} , master(config->createSSHMaster( @@ -152,10 +152,10 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore public: using Config = MountedSSHStoreConfig; - MountedSSHStore(ref config) - : Store{*config} - , RemoteStore{*config} - , SSHStore{config} + MountedSSHStore(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} + , RemoteStore{*config} /* The `Store` base is initialised above; this argument is discarded. */ + , SSHStore{config, SecretContext{}} , LocalFSStore{*config} { extraRemoteProgramArgs = { @@ -211,14 +211,15 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore void MountedSSHStore::anchor() {} -ref SSHStore::Config::openStore() const +ref SSHStore::Config::openStore(const SecretContext & context) const { - return make_ref(ref{shared_from_this()}); + return make_ref(ref{shared_from_this()}, context); } -ref MountedSSHStore::Config::openStore() const +ref MountedSSHStore::Config::openStore(const SecretContext & context) const { - return make_ref(ref{std::dynamic_pointer_cast(shared_from_this())}); + return make_ref( + ref{std::dynamic_pointer_cast(shared_from_this())}, context); } ref SSHStore::openConnection() diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 72f288ffa968..fbc42d577280 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -150,10 +150,15 @@ std::pair StoreDirConfig::toStorePath(std::string_view pat } ref Store::getBuilder(std::shared_ptr evalStore) +{ + return getBuilder(secretContext, std::move(evalStore)); +} + +ref Store::getBuilder(const SecretContext & context, std::shared_ptr evalStore) { auto store = ref(shared_from_this()); auto evalStoreRef = evalStore ? ref(std::move(evalStore)) : store; - return make_ref(store, evalStoreRef); + return make_ref(store, evalStoreRef, context); } std::filesystem::path Store::followLinksToStore(std::string_view _path) const @@ -410,13 +415,14 @@ StringSet Store::Config::getDefaultSystemFeatures() return res; } -Store::Store(const Store::Config & config) +Store::Store(const Store::Config & config, SecretContext secretContext) : StoreDirConfig{config} , config{config} , pathInfoCache( config.pathInfoCacheSize ? std::make_shared((size_t) config.pathInfoCacheSize) : nullptr) + , secretContext{std::move(secretContext)} { assertLibStoreInitialized(); } @@ -515,7 +521,7 @@ StorePathSet Store::querySubstitutablePaths(const StorePathSet & paths) StorePathSet res; - for (auto & sub : getDefaultSubstituters()) { + for (auto & sub : getDefaultSubstituters(secretContext)) { if (remaining.empty()) break; if (sub->storeDir != storeDir) diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 2879a5be4024..87fcda137f05 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -1,3 +1,4 @@ +#include "nix/util/sync.hh" #include "nix/store/store-registration.hh" #include "nix/store/store-open.hh" #include "nix/store/local-store.hh" @@ -6,26 +7,42 @@ #include "nix/util/environment-variables.hh" #include +#include namespace nix { +ref openStore(const SecretContext & context) +{ + return openStore(context, StoreReference{settings.storeUri.get()}); +} + ref openStore() { - return openStore(StoreReference{settings.storeUri.get()}); + return openStore(SecretContext{}); +} + +ref openStore(const SecretContext & context, const std::string & uri, const Store::Config::Params & extraParams) +{ + return openStore(context, StoreReference::parse(uri, extraParams)); } ref openStore(const std::string & uri, const Store::Config::Params & extraParams) { - return openStore(StoreReference::parse(uri, extraParams)); + return openStore(SecretContext{}, uri, extraParams); } -ref openStore(StoreReference && storeURI) +ref openStore(const SecretContext & context, StoreReference && storeURI) { - auto store = resolveStoreConfig(std::move(storeURI))->openStore(); + auto store = resolveStoreConfig(std::move(storeURI))->openStore(context); store->init(); return store; } +ref openStore(StoreReference && storeURI) +{ + return openStore(SecretContext{}, std::move(storeURI)); +} + ref resolveStoreConfig(StoreReference && storeURI) { auto & params = storeURI.params; @@ -51,7 +68,7 @@ ref resolveStoreConfig(StoreReference && storeURI) { } - ref openStore() const override + ref openStore(const SecretContext & context) const override { unreachable(); } @@ -111,32 +128,56 @@ ref resolveStoreConfig(StoreReference && storeURI) return storeConfig; } -std::list> getDefaultSubstituters() +std::list> getDefaultSubstituters(const SecretContext & context) { - static auto stores([]() { - std::list> stores; - - std::set done; - - auto addStore = [&](const StoreReference & ref) { - if (!done.insert(ref).second) - return; - try { - stores.push_back(openStore(StoreReference{ref})); - } catch (Error & e) { - logWarning(e.info()); - } - }; - - for (const auto & ref : settings.getWorkerSettings().substituters.get()) - addStore(ref); - - stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); - + /* Opening substituters is expensive, so preserve the process-wide cache + for the process-wide, resolver-free context. A resolver belongs to one + operation and must be released with it, so neither it nor stores that + retain it may be placed in this static cache. */ + using Cache = std::optional>>; + + static Sync cache; + + if (!context.secretResolver) { + auto cached(cache.lock()); + if (cached->has_value()) + return cached->value(); + } + + /* Opening a store can do network I/O (a binary cache fetches + `nix-cache-info` in `init()`), so build the list without the lock + held. A concurrent caller may do the same work; the first to install + its result wins and the rest is discarded. */ + std::list> stores; + std::set done; + + auto addStore = [&](const StoreReference & ref) { + if (!done.insert(ref).second) + return; + try { + stores.push_back(openStore(context, StoreReference{ref})); + } catch (Error & e) { + logWarning(e.info()); + } + }; + + for (const auto & ref : settings.getWorkerSettings().substituters.get()) + addStore(ref); + + stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); + + if (context.secretResolver) return stores; - }()); - return stores; + auto cached(cache.lock()); + if (!cached->has_value()) + cached->emplace(std::move(stores)); + return cached->value(); +} + +std::list> getDefaultSubstituters() +{ + return getDefaultSubstituters(SecretContext{}); } Implementations::Map & Implementations::registered() diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index c4137df7c6aa..520f8e64d8f9 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -55,8 +55,8 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) { } -UDSRemoteStore::UDSRemoteStore(ref config) - : Store{*config} +UDSRemoteStore::UDSRemoteStore(ref config, SecretContext secretContext) + : Store{*config, std::move(secretContext)} , LocalFSStore{*config} , RemoteStore{*config} , config{config} @@ -111,9 +111,9 @@ void UDSRemoteStore::addIndirectRoot(const std::filesystem::path & path) readInt(conn->from); } -ref UDSRemoteStore::Config::openStore() const +ref UDSRemoteStore::Config::openStore(const SecretContext & context) const { - return make_ref(ref{shared_from_this()}); + return make_ref(ref{shared_from_this()}, context); } static RegisterStoreImplementation regUDSRemoteStore; diff --git a/src/libstore/unix/build/derivation-builder-impl.hh b/src/libstore/unix/build/derivation-builder-impl.hh index 353840b7673f..3935076d1065 100644 --- a/src/libstore/unix/build/derivation-builder-impl.hh +++ b/src/libstore/unix/build/derivation-builder-impl.hh @@ -364,15 +364,26 @@ protected: void writeBuilderFile(const std::string & name, std::string_view contents); /** - * Arguments passed to runChild(). + * Arguments passed to runChild(). Everything here is gathered before + * the fork, because the child is in no position to go looking for it. */ struct RunChildArgs { + /** + * netrc contents for builtin:fetchurl. An engaged empty value is an + * explicit empty netrc and must not fall back to `netrc-file`. + */ + std::optional netrcData; #if NIX_WITH_AWS_AUTH std::optional awsCredentials; #endif }; + /** + * Gather everything runChild() needs from the parent process. + */ + RunChildArgs makeRunChildArgs(); + /** * Run the builder's process. */ diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 22bf645610a2..6f89b6078261 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -20,6 +20,7 @@ #include "nix/store/build/derivation-env-desugar.hh" #include "nix/util/terminal.hh" #include "nix/store/filetransfer.hh" +#include "nix/store/filetransfer-impl.hh" #include #include @@ -639,13 +640,31 @@ std::optional DerivationBuilderImpl::preResolveAwsCredentials() } #endif -void DerivationBuilderImpl::startChild() +DerivationBuilderImpl::RunChildArgs DerivationBuilderImpl::makeRunChildArgs() { - RunChildArgs args{ + return RunChildArgs{ + /* Only builtin:fetchurl reads a netrc, and only the parent can ask a + resolver for one: the child must not talk to a broker, and by the + time it could the sandbox may have taken the file away. + + The lookup is left unscoped by host. One netrc has to serve the + derivation's URL and every hashed mirror tried ahead of it, so + narrowing it to a single machine would break those fallbacks. */ + .netrcData = drv.isBuiltin() && drv.builder == "builtin:fetchurl" + ? resolveNetrcData( + secretResolver, + fileTransferSettings, + SecretPurpose{.consumer = "builtin:fetchurl", .operation = "build"}) + : std::nullopt, #if NIX_WITH_AWS_AUTH .awsCredentials = preResolveAwsCredentials(), #endif }; +} + +void DerivationBuilderImpl::startChild() +{ + auto args = makeRunChildArgs(); pid = startProcess([this, args = std::move(args)]() { openSlave(); @@ -949,9 +968,12 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) /* Make the contents of netrc and the CA certificate bundle available to builtin:fetchurl (which may run under a - different uid and/or in a sandbox). */ + different uid and/or in a sandbox). The netrc came from the + parent, which is the only side that can ask a secret resolver + for it. */ BuiltinBuilderContext ctx{ .drv = drv, + .netrcData = std::move(args.netrcData), .hashedMirrors = settings.getLocalSettings().hashedMirrors, .tmpDirInSandbox = tmpDirInSandbox(), #if NIX_WITH_AWS_AUTH @@ -960,11 +982,6 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) }; if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") { - try { - ctx.netrcData = readFile(fileTransferSettings.netrcFile.get()); - } catch (SystemError &) { - } - if (auto & caFile = fileTransferSettings.caFile.get()) try { ctx.caFileData = readFile(*caFile); diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index eb73c6c915a4..ef85042e3567 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -45,7 +45,9 @@ struct CmdFlakePrefetchInputs : FlakeCommand if (auto lockedNode = dynamic_cast(&node)) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); - auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, *store).first; + auto accessor = + lockedNode->lockedRef.input.getAccessor(fetchers::FetchContext{fetchSettings, {}}, *store) + .first; fetchToStore( fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); } catch (Error & e) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 96de12623a61..4ba2fd2dfc05 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1078,7 +1078,8 @@ struct CmdFlakeClone : FlakeCommand if (destDir.empty()) throw Error("missing flag '--dest'"); - getFlakeRef().resolve(fetchSettings, *store).input.clone(fetchSettings, *store, destDir); + auto fetchContext = fetchers::FetchContext{fetchSettings, {}}; + getFlakeRef().resolve(fetchContext, *store).input.clone(fetchContext, *store, destDir); } }; @@ -1118,7 +1119,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs auto getStorePath = [&](const FlakeRef & lockedRef) { return dryRun ? lockedRef.input.computeStorePath(*store) - : std::get(lockedRef.input.fetchToStore(fetchSettings, *store)); + : std::get( + lockedRef.input.fetchToStore(fetchers::FetchContext{fetchSettings, {}}, *store)); }; auto storePath = getStorePath(flake.flake.lockedRef); @@ -1526,8 +1528,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON void run(ref store) override { auto originalRef = getFlakeRef(); - auto resolvedRef = originalRef.resolve(fetchSettings, *store); - auto [accessor, lockedRef] = resolvedRef.lazyFetch(getEvalState()->fetchSettings, *store); + auto resolvedRef = originalRef.resolve(getEvalState()->fetchContext, *store); + auto [accessor, lockedRef] = resolvedRef.lazyFetch(getEvalState()->fetchContext, *store); auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); auto hash = store->queryPathInfo(storePath)->narHash; diff --git a/src/nix/main.cc b/src/nix/main.cc index 4b0bf4cc2ee8..248d5647d2f4 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -249,7 +249,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto statePtr = std::make_shared( LookupPath{}, openStore(StoreReference{.variant = StoreReference::Specified{.scheme = "dummy"}}), - fetchSettings, + fetchers::FetchContext{fetchSettings, {}}, evalSettings); auto & state = *statePtr; @@ -458,7 +458,7 @@ void mainWrapped(int argc, char ** argv) auto statePtr = std::make_shared( LookupPath{}, openStore(StoreReference{.variant = StoreReference::Specified{.scheme = "dummy"}}), - fetchSettings, + fetchers::FetchContext{fetchSettings, {}}, evalSettings); auto & state = *statePtr; auto builtinsJson = nlohmann::json::object(); diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index 1268163967a9..23eea5d767ac 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -30,7 +30,8 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, void run(ref srcStore, StorePaths && storePaths) override { - auto dstStore = !dstUri ? openStore() : openStore(StoreReference{*dstUri}); + auto dstStore = !dstUri ? openStore(srcStore->getSecretContext()) + : openStore(srcStore->getSecretContext(), StoreReference{*dstUri}); auto remappings = makeContentAddressed(*srcStore, *dstStore, StorePathSet(storePaths.begin(), storePaths.end())); diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index c84c98a96923..03c9233f2971 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -320,7 +320,8 @@ static void main_nix_build(int argc, char ** argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(StoreReference{*myArgs.evalStoreUrl}) : store; - auto state = std::make_shared(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); + auto state = std::make_shared( + myArgs.lookupPath, evalStore, fetchers::FetchContext{fetchSettings, {}}, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index 9b42182ef827..d72a67643ea2 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -8,6 +8,7 @@ #include "nix/expr/eval-settings.hh" // for defexpr #include "nix/util/os-string.hh" #include "nix/util/users.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/tarball.hh" #include "self-exe.hh" #include "man-pages.hh" @@ -134,7 +135,8 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(*store, fetchSettings, url, std::string(baseNameOf(url))); + auto result = fetchers::downloadFile( + *store, fetchers::FetchContext{fetchSettings, {}}, url, std::string(baseNameOf(url))); url = result.effectiveUrl; bool unpacked = false; @@ -155,10 +157,14 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - result = fetchers::downloadFile(*store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz"); + result = fetchers::downloadFile( + *store, fetchers::FetchContext{fetchSettings, {}}, url + "/nixexprs.tar.xz", "nixexprs.tar.xz"); } catch (FileTransferError & e) { - result = - fetchers::downloadFile(*store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2"); + result = fetchers::downloadFile( + *store, + fetchers::FetchContext{fetchSettings, {}}, + url + "/nixexprs.tar.bz2", + "nixexprs.tar.bz2"); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression diff --git a/src/nix/nix-copy-closure/nix-copy-closure.cc b/src/nix/nix-copy-closure/nix-copy-closure.cc index db797a745c6c..ff79a13b47b2 100644 --- a/src/nix/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix/nix-copy-closure/nix-copy-closure.cc @@ -55,8 +55,8 @@ static int main_nix_copy_closure(int argc, char ** argv) for SSH reference parsing? */ make_ref(ParsedURL::Authority::parse(sshHost), LegacySSHStoreConfig::Params{}); remoteConfig->compress |= gzip; - auto to = toMode ? remoteConfig->openStore() : openStore(); - auto from = toMode ? openStore() : remoteConfig->openStore(); + auto to = toMode ? remoteConfig->openStore(SecretContext{}) : openStore(); + auto from = toMode ? openStore() : remoteConfig->openStore(SecretContext{}); RealisedPath::Set storePaths2; for (auto & path : storePaths) diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 671fa46f45f5..bbc85f68ab3b 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -1512,8 +1512,8 @@ static int main_nix_env(int argc, char ** argv) if (op != opVersion) { auto store = openStore(); - globals.state = - std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); + globals.state = std::shared_ptr( + new EvalState(myArgs.lookupPath, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index b180472c1275..609ed16af9d1 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -169,7 +169,8 @@ static int main_nix_instantiate(int argc, char ** argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(StoreReference{*myArgs.evalStoreUrl}) : store; - auto state = std::make_shared(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); + auto state = std::make_shared( + myArgs.lookupPath, evalStore, fetchers::FetchContext{fetchSettings, {}}, evalSettings, store); state->repair = myArgs.repair; const Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index bceb8c3bb3db..89ed94c6b975 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -203,7 +203,8 @@ static int main_nix_prefetch_url(int argc, char ** argv) setLogFormat("bar"); auto store = openStore(); - auto state = std::make_shared(myArgs.lookupPath, store, fetchSettings, evalSettings); + auto state = std::make_shared( + myArgs.lookupPath, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings); const Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 8150ddf1d4ef..effbec7eb415 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -65,7 +65,7 @@ struct CmdRegistryList : StoreCommand { using namespace fetchers; - auto registries = getRegistries(fetchSettings, *store); + auto registries = getRegistries(fetchers::FetchContext{fetchSettings, {}}, *store); for (auto & registry : registries) { for (auto & entry : registry->entries) { @@ -186,8 +186,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand auto registry = getRegistry(); auto ref = parseFlakeRef(url); auto lockedRef = parseFlakeRef(locked); - auto resolvedInput = lockedRef.resolve(fetchSettings, *store).input; - auto resolved = resolvedInput.getAccessor(fetchSettings, *store).second; + auto resolvedInput = lockedRef.resolve(fetchers::FetchContext{fetchSettings, {}}, *store).input; + auto resolved = resolvedInput.getAccessor(fetchers::FetchContext{fetchSettings, {}}, *store).second; if (!resolved.isLocked(fetchSettings)) warn("flake '%s' is not locked", resolved.to_string()); fetchers::Attrs extraAttrs; @@ -227,7 +227,7 @@ struct CmdRegistryResolve : StoreCommand { for (auto & url : urls) { auto ref = parseFlakeRef(url); - auto resolved = ref.resolve(fetchSettings, *store); + auto resolved = ref.resolve(fetchers::FetchContext{fetchSettings, {}}, *store); logger->cout("%s", resolved.to_string()); } } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index bcc577317983..61abb8ff057a 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -44,7 +44,7 @@ struct CmdCopySigs : StorePathsCommand // FIXME: factor out commonality with MixVerify. std::vector> substituters; for (auto & s : substituterUris) - substituters.push_back(openStore(StoreReference{s})); + substituters.push_back(openStore(store->getSecretContext(), StoreReference{s})); ThreadPool pool{fileTransferSettings.httpConnections}; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index cb55e41935f3..86145f5bdb87 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -372,7 +372,7 @@ static void daemonLoop( } // Handle the connection. - auto store = storeConfig->openStore(); + auto store = storeConfig->openStore(SecretContext{}); store->init(); processConnection( std::move(store), @@ -486,7 +486,7 @@ static void runDaemon( std::visit( overloaded{ [&](StdIO) { - auto store = storeConfig->openStore(); + auto store = storeConfig->openStore(SecretContext{}); store->init(); std::shared_ptr remoteStore; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a99331966c4d..1990fe2f235c 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -198,7 +198,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest(parseURL(upgradeSettings.storePathUrl.get())); auto res = getFileTransfer()->download(req); - auto state = std::make_shared(LookupPath{}, store, fetchSettings, evalSettings); + auto state = + std::make_shared(LookupPath{}, store, fetchers::FetchContext{fetchSettings, {}}, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); const Bindings & bindings = Bindings::emptyBindings; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 27a349062e41..821040d65dfb 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -65,7 +65,7 @@ struct CmdVerify : StorePathsCommand { std::vector> substituters; for (auto & s : substituterUris) - substituters.push_back(openStore(StoreReference{s})); + substituters.push_back(openStore(store->getSecretContext(), StoreReference{s})); auto publicKeys = getDefaultPublicKeys(); diff --git a/tests/functional/flakes/bundle.sh b/tests/functional/flakes/bundle.sh index 0689d94220a3..0305fadaa174 100755 --- a/tests/functional/flakes/bundle.sh +++ b/tests/functional/flakes/bundle.sh @@ -17,6 +17,7 @@ cat < flake.nix default = simple; }; packages.$system.default = import ./simple.nix; + packages.$system.functionPackage = {}: import ./simple.nix; apps.$system.default = { type = "app"; program = "\${import ./simple.nix}/hello"; @@ -29,6 +30,7 @@ nix build .# nix bundle --bundler .# .# nix bundle --bundler .#bundlers."$system".default .#packages."$system".default nix bundle --bundler .#bundlers."$system".simple .#packages."$system".default +nix bundle --bundler .#bundlers."$system".simple .#packages."$system".functionPackage nix bundle --bundler .#bundlers."$system".default .#apps."$system".default nix bundle --bundler .#bundlers."$system".simple .#apps."$system".default diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh index 75a2c8cacbf9..6afff7d9e090 100755 --- a/tests/functional/flakes/eval-cache.sh +++ b/tests/functional/flakes/eval-cache.sh @@ -48,3 +48,38 @@ nix build --no-link "$flake1Dir#stack-depth" expect 1 nix build "$flake1Dir#ifd" --option allow-import-from-derivation false 2>&1 \ | grepQuiet 'error: cannot build .* during evaluation because the option '\''allow-import-from-derivation'\'' is disabled' nix build --no-link "$flake1Dir#ifd" + +# Commands that auto-call the installable ('nix search', 'nix run', ...) must +# still use the cache: the trace fires while the cache is cold, and never again. +flake2Dir="$TEST_ROOT/eval-cache-auto-call-flake" + +createGitRepo "$flake2Dir" "" +cp ../simple.nix ../simple.builder.sh "${config_nix}" "$flake2Dir/" +git -C "$flake2Dir" add simple.nix simple.builder.sh config.nix + +cat >"$flake2Dir/flake.nix" <" = "real auto-call attribute"; + cached = mkDerivation { + name = "cached"; + buildCommand = '' + echo true > \$out + ''; + }; + }; + }; +} +EOF + +git -C "$flake2Dir" add flake.nix +git -C "$flake2Dir" commit -m "Init" + +# A real attribute with the old pseudo-entry name must occupy a distinct cache +# slot. Prime that real attribute first, then exercise the auto-call cache. +[[ $(nix eval --raw --no-write-lock-file "$flake2Dir#packages.$system.\"\"") == \ + "real auto-call attribute" ]] +nix search --no-write-lock-file "$flake2Dir" ^ 2>&1 | grepQuiet "evaluating packages" +nix search --no-write-lock-file "$flake2Dir" ^ 2>&1 | grepQuietInverse "evaluating packages" diff --git a/tests/functional/flakes/run.sh b/tests/functional/flakes/run.sh index d3e4549dc5cf..edbc1c8008b0 100755 --- a/tests/functional/flakes/run.sh +++ b/tests/functional/flakes/run.sh @@ -18,11 +18,16 @@ cat < flake.nix type = "app"; program = "\${(import ./shell-hello.nix).hello}/bin/hello"; }; + apps.$system.functionApp = {}: { + type = "app"; + program = "\${(import ./shell-hello.nix).hello}/bin/hello"; + }; }; } EOF nix run --no-write-lock-file .#appAsApp nix run --no-write-lock-file .#pkgAsPkg +nix run --no-write-lock-file .#functionApp ! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" ! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" @@ -87,4 +92,3 @@ nix run --no-write-lock-file -- . myarg1 myarg2 2>&1 | grepQuiet "ARGS: myarg1 m # And verify that a non-installable first argument causes an error expectStderr 1 nix run --no-write-lock-file -- myarg1 myarg2 | grepQuiet "error.*myarg1" - diff --git a/tests/functional/search.sh b/tests/functional/search.sh index ffcbebf3edf7..fca1358ea370 100755 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -46,3 +46,16 @@ e=$'\x1b' # grep doesn't support \e, \033 or even \x1b (( $(nix search -f search.nix foo ^ --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) (( $(nix search -f search.nix foo ^ -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) [[ $(nix search -f search.nix '' ^ -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] + +# Flake installables with function-valued package sets are auto-called. +flakeDir="$TEST_HOME/function-flake" +mkdir "$flakeDir" +cp search.nix "$config_nix" "$flakeDir" +cat > "$flakeDir/flake.nix" <