From 996e21523d00d44573a8a03f9baa9f1db4070c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sat, 5 Sep 2026 18:05:39 -0500 Subject: [PATCH 1/2] Create CBOR representation of derivations Add deterministic CBOR import and export with arbitrary byte support for builder, arguments, and environment names and values. Share derivation field definitions with the JSON codec while preserving the existing JSON format and the original structured-attribute JSON bytes. Distinguish byte 0xff from EOF in the ATerm parser so stored derivations round-trip correctly. Document the CBOR schema and normalization rules, and test byte preservation, malformed inputs, and derivation identity. Assisted-by: Codex (GPT-6) --- doc/manual/source/SUMMARY.md.in | 1 + .../source/protocols/derivation-cbor.md | 94 ++++ doc/manual/source/store/derivation/index.md | 4 +- src/libstore-tests/derivation/cbor.cc | 337 +++++++++++++ .../derivation/external-formats.cc | 19 + src/libstore-tests/meson.build | 1 + src/libstore/derivation/aterm.cc | 5 +- src/libstore/derivation/cbor.cc | 461 ++++++++++++++++++ src/libstore/derivation/json.cc | 225 +++------ src/libstore/derivation/structured.hh | 182 +++++++ .../include/nix/store/derivation/cbor.hh | 25 + src/libstore/include/nix/store/meson.build | 1 + src/libstore/meson.build | 1 + src/nix/derivation-add.cc | 22 +- src/nix/derivation-add.md | 4 + src/nix/derivation-show.cc | 21 + src/nix/derivation-show.md | 7 + tests/functional/derivation-json.sh | 44 ++ 18 files changed, 1288 insertions(+), 166 deletions(-) create mode 100644 doc/manual/source/protocols/derivation-cbor.md create mode 100644 src/libstore-tests/derivation/cbor.cc create mode 100644 src/libstore/derivation/cbor.cc create mode 100644 src/libstore/derivation/structured.hh create mode 100644 src/libstore/include/nix/store/derivation/cbor.hh diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 612a13c77024..792fe1897449 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -143,6 +143,7 @@ - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) + - [Derivation CBOR format](protocols/derivation-cbor.md) - [Nix32 Encoding](protocols/nix32.md) - [C API](c-api.md) - [Glossary](glossary.md) diff --git a/doc/manual/source/protocols/derivation-cbor.md b/doc/manual/source/protocols/derivation-cbor.md new file mode 100644 index 000000000000..d7a26d03a334 --- /dev/null +++ b/doc/manual/source/protocols/derivation-cbor.md @@ -0,0 +1,94 @@ +# Derivation CBOR format + +CBOR version 1 is an experimental interchange representation of existing +Nix derivations. It supports arbitrary bytes in the builder, arguments, and +environment names and values, including strings that cannot be represented +by the current derivation JSON format. + +`nix derivation show --cbor` writes a single derivation when given one +derivation without `--recursive`. With multiple derivations or `--recursive`, +it writes a collection. `nix derivation add --cbor` reads a single derivation. + +Importing uses the existing derivation validation, storage, and hashing +rules. CBOR bytes are not hashed directly. This format does not change +derivation identity or introduce a new addressing scheme. + +## Schema + +A derivation is a CBOR map with the following fields. Field names are text +strings. All fields except `structuredAttrs` are required. + +| Field | CBOR type | Meaning | +| --- | --- | --- | +| `version` | Unsigned integer | Must be `1`. | +| `name` | Text string | Derivation name. | +| `system` | Text string | Build platform. | +| `builder` | Byte string | Builder path. | +| `args` | Array of byte strings | Ordered builder arguments. | +| `env` | Map of byte strings to byte strings | Environment names and values. | +| `outputs` | Map with text keys | Output specifications. | +| `inputs` | Map with text keys | Source and derivation dependencies. | +| `structuredAttrs` | Byte string, optional | Original structured-attribute JSON document. | + +The `inputs` and `outputs` substructures use the field definitions of +[derivation JSON version 4](json/derivation/index.md), with JSON objects +encoded as CBOR maps and their strings encoded as CBOR text strings. +An input node has `outputs` (a set of output names encoded as an array) and +`dynamicOutputs` (a map from output names to further input nodes). +Output variants and experimental-feature requirements are shared between +the JSON and CBOR codecs. Changes to the shared schema must consider each +format's version independently. + +Text strings must contain valid UTF-8. Byte-valued fields always use byte +strings, even when their contents happen to be valid UTF-8. Environment +names are byte strings too; a decoder that only supports text map keys +cannot decode this format in general. + +A collection is a map containing `version` (unsigned integer `1`) and +`derivations` (a map from store-path base names, as text strings, to +derivation maps). Each derivation includes its own `version` field. + +## Structured attributes and identity + +The `structuredAttrs` byte string must contain a JSON object accepted by +Nix's structured-attribute parser. Its bytes are preserved verbatim, +including whitespace, key order, escaping, and number spellings. +Numbers inside it are JSON bytes, not CBOR integers or floating-point values. + +Existing derivation hashes depend on the original JSON spelling. +For example, `{"a":1}` and `{ "a": 1 }` can belong to different derivations +despite describing equivalent JSON values. Importing and exporting CBOR +must preserve this distinction, as required by the +[ATerm identity rules](derivation-aterm.md#canonical-form). + +## Deterministic output + +The encoder follows the +[RFC 8949 length-first deterministic encoding requirements](https://www.rfc-editor.org/rfc/rfc8949.html#section-4.2.3): + +- Integers and lengths use their shortest available encodings. +- All arrays, maps, and strings have definite lengths. +- Map keys are ordered by the length of their encoded key, then by + unsigned bytewise lexicographic order of that encoding. +- Sets, including input source paths and input output names, are encoded + as arrays in ascending bytewise lexical order, without duplicates. +- Argument order and the contents of byte strings are preserved. + +Only unsigned integers, byte strings, text strings, arrays, maps, and +booleans are supported at the CBOR layer. The schema further restricts +their locations; for example, `impure` must be `true` when present. +Tags, negative integers, floats, null, and indefinite lengths are rejected. + +## Import validation + +The reader accepts nonminimal integer and length encodings, unsorted maps, +and unsorted set arrays. Exporting the imported derivation normalizes these +to the deterministic encoding described above. Import does not normalize +the embedded structured-attribute JSON. + +Duplicate map keys, duplicate set entries, missing or unknown fields, +incorrect wire types, invalid UTF-8 text, unsupported versions, truncated +input, and trailing data are rejected. Dynamic input nodes are limited to +256 levels below the initial node, and the CBOR reader also limits nesting. +The normal derivation invariants are checked when adding a derivation to +the store. diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 6b0a516c5299..31cda632671f 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -268,12 +268,14 @@ There are two types of placeholder, corresponding to the two cases where this pr ### Derivation {#derivation-encoding} -There are two formats, documented separately: +The formats are documented separately: - The legacy ["ATerm" format](@docroot@/protocols/derivation-aterm.md) - The experimental, currently under development and changing [JSON format](@docroot@/protocols/json/derivation/index.md) +- The experimental [CBOR interchange format](@docroot@/protocols/derivation-cbor.md), which preserves arbitrary bytes in builder, arguments, and environment fields + Every derivation has a canonical choice of encoding used to serialize it to a store object. This ensures that there is a canonical [store path] used to refer to the derivation, as described in [Referencing derivations](#derivation-path). diff --git a/src/libstore-tests/derivation/cbor.cc b/src/libstore-tests/derivation/cbor.cc new file mode 100644 index 000000000000..00c830c71e6d --- /dev/null +++ b/src/libstore-tests/derivation/cbor.cc @@ -0,0 +1,337 @@ +#include +#include + +#include "nix/store/derivation/cbor.hh" +#include "nix/store/derivation/aterm.hh" +#include "derivation/test-support.hh" + +namespace nix { +namespace { + +using nlohmann::json; +using namespace std::string_literals; +using namespace std::string_view_literals; + +const auto golden = + "\xa8\x63" + "env\xa2\x41z\x41\x00\x42" + "aa\x41\xff\x64" + "args\x81\x41" + "a\x64" + "name\x61n\x66inputs\xa2\x64" + "drvs\xa0\x64srcs\x80\x66system\x61s\x67" + "builder\x41" + "b\x67outputs\xa0\x67version\x01"s; + +json binary(std::string_view value) +{ + return json::binary(std::vector(value.begin(), value.end())); +} + +json fixture() +{ + return { + {"version", 1u}, + {"name", "n"}, + {"system", "s"}, + {"builder", binary("b")}, + {"args", {binary("a")}}, + {"env", {{"aa", binary("\xff")}, {"z", binary("\0"sv)}}}, + {"outputs", json::object()}, + {"inputs", {{"srcs", json::array()}, {"drvs", json::object()}}}, + }; +} + +std::string encodeValue(const json & value) +{ + auto bytes = json::to_cbor(value); + return {bytes.begin(), bytes.end()}; +} + +// nlohmann's decoder cannot represent byte-string map keys. Use its encoder +// for individual items and construct the environment map independently. +std::string encodeMap(const std::vector> & entries) +{ + auto bytes = encodeValue(entries.size()); + bytes[0] |= 0xa0; + for (auto & [key, value] : entries) { + bytes += encodeValue(key); + bytes += value; + } + return bytes; +} + +std::string encode(const json & value) +{ + std::vector> entries; + for (auto & [key, child] : value.items()) { + if (key == "env" && child.is_object()) { + std::vector> env; + for (auto & [name, content] : child.items()) + env.emplace_back(binary(name), encodeValue(content)); + entries.emplace_back(key, encodeMap(env)); + } else + entries.emplace_back(key, encodeValue(child)); + } + return encodeMap(entries); +} + +TEST_F(DerivationTest, CborReferenceGolden) +{ + auto drv = derivation::parseCbor(golden, mockXpSettings); + EXPECT_EQ(drv.name, "n"); + EXPECT_EQ(drv.env.at("aa"), "\xff"s); + EXPECT_EQ(drv.env.at("z"), "\x00"s); + EXPECT_EQ(derivation::toCbor(drv), golden); + EXPECT_EQ( + derivation::unparse(drv, *store), "Derive([],[],[],\"s\",\"b\",[\"a\"],[(\"aa\",\"\xff\"),(\"z\",\"\x00\")])"s); +} + +TEST_F(DerivationTest, CborRejectsTruncationsAndInvalidWireTypes) +{ + for (size_t size = 0; size < golden.size(); ++size) + EXPECT_THROW(derivation::parseCbor(std::string_view(golden).substr(0, size)), Error) << size; + for (auto bytes : + {golden + "\x00"s, + "\xc0" + golden, + "\xbf\xff"s, + "\xa2\x61x\x00\x61x\x00"s, + "\xbb\xff\xff\xff\xff\xff\xff\xff\xff"s}) + EXPECT_THROW(derivation::parseCbor(bytes), Error); +} + +TEST_F(DerivationTest, CborRejectsInvalidSchema) +{ + auto original = fixture(); + for (auto & [key, ignored] : original.items()) { + auto value = original; + value.erase(key); + EXPECT_THROW(derivation::parseCbor(encode(value)), Error) << key; + } + for (auto bad : {json(nullptr), json("text"), json::array({1, 2})}) { + auto value = original; + value["env"]["z"] = bad; + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); + } + for (auto key : {"builder", "args", "env", "name", "system"}) { + auto value = original; + value[key] = std::string_view(key) == "name" || std::string_view(key) == "system" ? binary("x") : json("text"); + EXPECT_THROW(derivation::parseCbor(encode(value)), Error) << key; + } + auto value = original; + value["args"] = {"text"}; + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); + value = original; + value["extra"] = 1; + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); + value = original; + value["version"] = 4; + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); + value = original; + value["inputs"]["srcs"] = {"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep", "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep"}; + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); + value = original; + value["name"] = "\xff"; + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); +} + +TEST_F(DerivationTest, CborNormalizesEncoding) +{ + auto value = fixture(); + EXPECT_EQ(derivation::toCbor(derivation::parseCbor(encode(value))), golden); + auto nonminimal = golden.substr(0, golden.size() - 1) + "\x18\x01"; + EXPECT_EQ(derivation::toCbor(derivation::parseCbor(nonminimal)), golden); + nonminimal = "\xb8\x08" + golden.substr(1); + EXPECT_EQ(derivation::toCbor(derivation::parseCbor(nonminimal)), golden); +} + +TEST_F(DerivationTest, CborEnvironmentKeyTypesAndDuplicates) +{ + auto drv = derivation::parseCbor(golden); + drv.env.clear(); + auto empty = derivation::toCbor(drv); + auto position = empty.find( + "\x63" + "env\xa0"s); + ASSERT_NE(position, std::string::npos); + for (auto env : { + encodeMap({{"text", encodeValue(binary("value"))}}), + encodeMap({{binary("\xff"), encodeValue(binary("a"))}, {binary("\xff"), encodeValue(binary("b"))}}), + encodeMap({{binary("x"), encodeValue("text")}}), + }) { + auto malformed = empty; + malformed.replace(position + 4, 1, env); + EXPECT_THROW(derivation::parseCbor(malformed), Error); + } +} + +TEST_F(DerivationTest, CborArbitraryBytes) +{ + std::string allBytes; + for (unsigned i = 0; i < 256; ++i) + allBytes += static_cast(i); + auto drv = derivation::parseCbor(golden); + drv.builder = allBytes; + drv.args = {"", allBytes, "\xc3\xa9"}; + drv.env = {{"", ""}, {allBytes, allBytes}, {"\x7f", "ascii"}, {"\x80", "non-UTF-8"}}; + auto bytes = derivation::toCbor(drv); + auto decoded = derivation::parseCbor(bytes); + EXPECT_EQ(decoded, drv); + EXPECT_EQ(derivation::unparse(decoded, *store), derivation::unparse(drv, *store)); + EXPECT_EQ(computeStorePath(*store, decoded), computeStorePath(*store, drv)); + EXPECT_EQ(derivation::toCbor(decoded), bytes); + + auto stored = derivation::unparse(decoded, *store); + EXPECT_EQ( + derivation::parse( + *store, std::move(stored), drv.name, derivation::defaultSupportWindowsStoreDir, mockXpSettings), + drv); + + auto expected = fixture(); + expected["builder"] = binary(allBytes); + expected["args"] = {binary(""), binary(allBytes), binary("\xc3\xa9")}; + expected["env"] = json::object(); + for (auto & [name, value] : drv.env) + expected["env"][name] = binary(value); + EXPECT_EQ(derivation::parseCbor(encode(expected)), drv); + + StorePath path{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-bytes.drv"}; + EXPECT_EQ( + derivation::toCbor(std::map{{path, drv}}), + encodeMap({ + {"version", encodeValue(1u)}, + {"derivations", encodeMap({{path.to_string(), bytes}})}, + })); + + // This payload illustrates the JSON wire format's remaining limitation. + EXPECT_THROW(json(drv).dump(), json::type_error); +} + +TEST_F(DerivationTest, CborBinaryKeyOrdering) +{ + auto drv = derivation::parseCbor(golden); + drv.env = {{"\x80", ""}, {"\x7f", ""}, {"aa", ""}}; + auto bytes = derivation::toCbor(drv); + EXPECT_NE( + bytes.find( + "\x63" + "env\xa3\x41\x7f\x40\x41\x80\x40\x42" + "aa\x40"s), + std::string::npos); +} + +TEST_F(DerivationTest, CborDerivationCollection) +{ + auto drv = derivation::parseCbor(golden); + drv.env.clear(); + auto expected = derivation::toCbor(drv); + StorePath first{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-first.drv"}; + StorePath second{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-second.drv"}; + std::map drvs{{first, drv}, {second, drv}}; + auto bytes = derivation::toCbor(drvs); + auto value = json::from_cbor(bytes); + EXPECT_EQ(value.at("version"), 1); + EXPECT_EQ(value.at("derivations").size(), 2u); + for (auto & [path, entry] : value.at("derivations").items()) + EXPECT_EQ(derivation::toCbor(derivation::parseCbor(encode(entry))), expected); + EXPECT_EQ(derivation::toCbor(drvs), bytes); +} + +TEST_F(DynDerivationTest, CborFixtureRoundTrips) +{ + for (auto name : {"simple-derivation", "dyn-dep-derivation"}) { + auto drv = nlohmann::adl_serializer::from_json( + json::parse(readFile(goldenMaster(std::string(name) + ".json"))), mockXpSettings); + auto decoded = derivation::parseCbor(derivation::toCbor(drv), mockXpSettings); + EXPECT_EQ(decoded, drv); + EXPECT_EQ(derivation::unparse(decoded, *store), derivation::unparse(drv, *store)); + } +} + +TEST_F(DerivationTest, CborStructuredAttrs) +{ + auto drv = derivation::parseCbor(golden); + drv.env.clear(); + drv.structuredAttrs = StructuredAttrs{json::parse(R"({"z":-0.0,"a":18446744073709551615})")}; + auto bytes = derivation::toCbor(drv); + auto value = json::from_cbor(bytes); + EXPECT_TRUE(value.at("structuredAttrs").is_binary()); + auto decoded = derivation::parseCbor(bytes); + EXPECT_EQ(decoded, drv); + EXPECT_EQ(derivation::unparse(decoded, *store), derivation::unparse(drv, *store)); + value["structuredAttrs"] = json::binary({'[', ']'}); + EXPECT_THROW(derivation::parseCbor(encode(value)), Error); +} + +TEST_F(DerivationTest, CborStringLengthsAndUtf8) +{ + auto drv = derivation::parseCbor(golden); + for (auto size : {0, 23, 24, 255, 256, 65535, 65536}) { + drv.env = {{std::string(size, '\xff'), std::string(size, '\xff')}}; + drv.args = {std::string(size, '\xff')}; + drv.builder = std::string(size, 'x'); + auto bytes = derivation::toCbor(drv); + EXPECT_EQ(derivation::parseCbor(bytes), drv); + EXPECT_NE(bytes.find(encodeValue(binary(drv.builder))), std::string::npos); + } + drv.name = "\xc3\xa9"; + EXPECT_EQ(derivation::parseCbor(derivation::toCbor(drv)), drv); + drv.name = "\xff"; + EXPECT_THROW(derivation::toCbor(drv), Error); +} + +TEST_F(DerivationTest, CborPreservesStructuredAttrsIdentity) +{ + for (const std::string text : + {R"({ "z": 0, "a": 1 })", + R"({"escaped":"\u0061","number":1e+02,"negativeZero":-0.0})", + "{\n \"a\": 1, \"a\": 2\n}\n"}) { + auto drv = derivation::parseCbor(golden); + drv.env.clear(); + drv.structuredAttrs = StructuredAttrs::parse(text); + auto aterm = derivation::unparse(drv, *store); + auto imported = derivation::parse( + *store, std::string(aterm), drv.name, derivation::defaultSupportWindowsStoreDir, mockXpSettings); + auto bytes = derivation::toCbor(imported); + auto payload = json::from_cbor(bytes).at("structuredAttrs").get_binary(); + EXPECT_EQ(std::string(payload.begin(), payload.end()), text); + auto decoded = derivation::parseCbor(bytes); + EXPECT_EQ(decoded, imported); + EXPECT_EQ(decoded.structuredAttrs->unparse().second, text); + EXPECT_EQ(derivation::unparse(decoded, *store), aterm); + EXPECT_EQ(computeStorePath(*store, decoded), computeStorePath(*store, imported)); + EXPECT_EQ(derivation::toCbor(decoded), bytes); + + auto collection = json::from_cbor( + derivation::toCbor(std::map{{computeStorePath(*store, imported), imported}})); + EXPECT_EQ(collection.at("derivations").begin()->at("structuredAttrs"), json::binary(payload)); + } +} + +TEST_F(DerivationTest, CborOutputVariants) +{ + mockXpSettings.set("experimental-features", "ca-derivations dynamic-derivations impure-derivations"); + for (auto name : + {"inputAddressed", "caFixedFlat", "caFixedNAR", "caFixedText", "caFloating", "deferred", "impure"}) { + auto value = fixture(); + value["outputs"]["out"] = json::parse(readFile(goldenMaster("output-"s + name + ".json"))); + auto drv = derivation::parseCbor(encode(value), mockXpSettings); + EXPECT_EQ(derivation::parseCbor(derivation::toCbor(drv), mockXpSettings), drv); + } +} + +TEST_F(DynDerivationTest, CborDynamicDepthLimit) +{ + auto value = fixture(); + json node = {{"outputs", {"out"}}, {"dynamicOutputs", json::object()}}; + for (unsigned depth = 0; depth < 256; ++depth) + node = {{"outputs", json::array()}, {"dynamicOutputs", {{"out", std::move(node)}}}}; + auto path = "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep.drv"; + value["inputs"]["drvs"][path] = node; + EXPECT_NO_THROW(derivation::parseCbor(encode(value), mockXpSettings)); + value["inputs"]["drvs"][path] = {{"outputs", json::array()}, {"dynamicOutputs", {{"out", node}}}}; + EXPECT_THROW(derivation::parseCbor(encode(value), mockXpSettings), Error); +} + +} // namespace +} // namespace nix diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 45ee3c50f055..b05d06fcdcb5 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -35,6 +35,25 @@ TEST_F(DerivationTest, UnterminatedString) FormatError); } +TEST_F(DerivationTest, ATermArbitraryBytes) +{ + std::string allBytes; + for (unsigned i = 0; i < 256; ++i) + allBytes += static_cast(i); + + Derivation drv; + drv.name = "bytes"; + drv.platform = "s"; + drv.builder = allBytes; + drv.args = {allBytes}; + drv.env = {{allBytes, allBytes}}; + + auto encoded = derivation::unparse(drv, *store); + auto decoded = derivation::parse( + *store, std::move(encoded), drv.name, derivation::defaultSupportWindowsStoreDir, mockXpSettings); + EXPECT_EQ(decoded, drv); +} + /** * A fixed-output derivation states its output path, but that path is a * function of the content address, so a stated path that disagrees is diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 5fb7e5990c60..62d4fe5172b6 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -37,6 +37,7 @@ sources = files( 'common-protocol.cc', 'content-address.cc', 'derivation/advanced-attrs.cc', + 'derivation/cbor.cc', 'derivation/external-formats.cc', 'derivation/full-inputs.cc', 'derivation/invariants.cc', diff --git a/src/libstore/derivation/aterm.cc b/src/libstore/derivation/aterm.cc index 65139e427c40..15750a80c8e2 100644 --- a/src/libstore/derivation/aterm.cc +++ b/src/libstore/derivation/aterm.cc @@ -38,14 +38,15 @@ struct StringViewStream int peek() const { - return remaining.empty() ? EOF : remaining[0]; + return remaining.empty() ? EOF : static_cast(remaining[0]); } int get() { if (remaining.empty()) return EOF; - char c = remaining[0]; + // Keep every byte value distinct from EOF, including 0xff. + unsigned char c = remaining[0]; remaining.remove_prefix(1); return c; } diff --git a/src/libstore/derivation/cbor.cc b/src/libstore/derivation/cbor.cc new file mode 100644 index 000000000000..4ca13546f649 --- /dev/null +++ b/src/libstore/derivation/cbor.cc @@ -0,0 +1,461 @@ +#include "nix/store/derivation/cbor.hh" +#include "nix/util/json-utils.hh" +#include "structured.hh" + +#include +#include + +namespace nix::derivation { +namespace { + +using nlohmann::json; + +[[noreturn]] void invalid(std::string_view message) +{ + throw FormatError("invalid derivation CBOR: %s", message); +} + +void checkText(const std::string & text) +{ + (void) json(text).dump(); +} + +class Reader +{ + std::string_view bytes; + size_t position = 0; + + uint8_t byte() + { + if (position == bytes.size()) + invalid("unexpected end of input"); + return static_cast(bytes[position++]); + } + + uint64_t length(uint8_t info) + { + if (info < 24) + return info; + if (info > 27) + invalid("reserved or indefinite length"); + uint64_t value = 0; + for (unsigned i = 0; i < (1u << (info - 24)); ++i) + value = (value << 8) | byte(); + return value; + } + +public: + explicit Reader(std::string_view bytes) + : bytes(bytes) + { + } + + json read(unsigned depth = 0) + { + if (depth > 520) + invalid("nesting limit exceeded"); + auto initial = byte(); + auto major = initial >> 5; + if (initial == 0xf4 || initial == 0xf5) + return initial == 0xf5; + if (major != 0 && major != 2 && major != 3 && major != 4 && major != 5) + invalid("unsupported CBOR type"); + auto size = length(initial & 31); + if (major == 0) + return size; + if (size > (bytes.size() - position) / (major == 5 ? 2 : 1)) + invalid("length exceeds remaining input"); + if (major == 2 || major == 3) { + auto value = bytes.substr(position, size); + position += size; + if (major == 2) + return json::binary(std::vector(value.begin(), value.end())); + std::string text(value); + checkText(text); + return text; + } + auto value = major == 4 ? json::array() : json::object(); + for (uint64_t i = 0; i < size; ++i) { + if (major == 4) + value.push_back(read(depth + 1)); + else { + auto key = read(depth + 1); + if (!key.is_string()) + invalid("map key must be text"); + auto name = key.get(); + if (value.contains(name)) + invalid("duplicate map key"); + value[name] = read(depth + 1); + } + } + return value; + } + + uint64_t count(uint8_t major) + { + auto initial = byte(); + if ((initial >> 5) != major) + invalid("unexpected CBOR type"); + auto size = length(initial & 31); + if (size > (bytes.size() - position) / (major == 5 ? 2 : 1)) + invalid("length exceeds remaining input"); + return size; + } + + std::string string(uint8_t major) + { + auto size = count(major); + std::string result(bytes.substr(position, size)); + position += size; + if (major == 3) + checkText(result); + return result; + } + + void map(uint8_t keyType, auto consume) + { + auto size = count(5); + std::set keys; + for (uint64_t i = 0; i < size; ++i) { + auto key = string(keyType); + if (!keys.insert(key).second) + invalid("duplicate map key"); + consume(key, *this); + } + } + + /** Locate a field without converting binary map keys to JSON keys. */ + void skip(unsigned depth = 0) + { + if (depth > 520) + invalid("nesting limit exceeded"); + auto initial = byte(); + if (initial == 0xf4 || initial == 0xf5) + return; + auto major = initial >> 5; + if (major != 0 && major != 2 && major != 3 && major != 4 && major != 5) + invalid("unsupported CBOR type"); + auto size = length(initial & 31); + if (major == 0) + return; + if (size > (bytes.size() - position) / (major == 5 ? 2 : 1)) + invalid("length exceeds remaining input"); + if (major == 2 || major == 3) + position += size; + else + for (uint64_t i = 0; i < size * (major == 5 ? 2 : 1); ++i) + skip(depth + 1); + } + + std::string_view item() + { + auto start = position; + skip(); + return bytes.substr(start, position - start); + } + + bool done() const + { + return position == bytes.size(); + } +}; + +void fields( + const json & value, + std::initializer_list required, + std::initializer_list optional = {}) +{ + if (!value.is_object()) + invalid("expected map"); + for (auto key : required) + if (!value.contains(key)) + invalid("missing field"); + for (auto & [key, ignored] : value.items()) + if (std::find(required.begin(), required.end(), key) == required.end() + && std::find(optional.begin(), optional.end(), key) == optional.end()) + invalid("unknown field"); +} + +void stringSet(const json & value) +{ + if (!value.is_array()) + invalid("expected set array"); + std::set seen; + for (auto & entry : value) { + if (!entry.is_string()) + invalid("set entry must be text"); + if (!seen.insert(entry.get()).second) + invalid("duplicate set entry"); + } +} + +void input(const json & value, unsigned depth = 0) +{ + if (depth > 256) + invalid("dynamic input nesting exceeds 256 levels"); + fields(value, {"outputs", "dynamicOutputs"}); + stringSet(value.at("outputs")); + for (auto & [name, child] : getObject(value.at("dynamicOutputs"))) + input(child, depth + 1); +} + +void writeHead(std::string & bytes, uint8_t major, uint64_t size) +{ + if (size < 24) { + bytes += static_cast((major << 5) | size); + return; + } + unsigned width = size <= 0xff ? 1 : size <= 0xffff ? 2 : size <= 0xffffffff ? 4 : 8; + bytes += static_cast((major << 5) | (width == 1 ? 24 : width == 2 ? 25 : width == 4 ? 26 : 27)); + for (unsigned i = width; i; --i) + bytes += static_cast(size >> ((i - 1) * 8)); +} + +void writeValue(std::string & bytes, const json & value) +{ + if (value.is_object()) { + std::vector keys; + for (auto & [key, ignored] : value.items()) + keys.push_back(key); + std::sort(keys.begin(), keys.end(), [](auto & a, auto & b) { + return a.size() != b.size() ? a.size() < b.size() : a < b; + }); + writeHead(bytes, 5, keys.size()); + for (auto & key : keys) { + writeValue(bytes, key); + writeValue(bytes, value.at(key)); + } + } else if (value.is_array()) { + writeHead(bytes, 4, value.size()); + for (auto & entry : value) + writeValue(bytes, entry); + } else if (value.is_string()) { + auto & text = value.get_ref(); + checkText(text); + writeHead(bytes, 3, text.size()); + bytes += text; + } else if (value.is_binary()) { + auto & data = value.get_binary(); + writeHead(bytes, 2, data.size()); + bytes.append(data.begin(), data.end()); + } else if (value.is_boolean()) + bytes += value.get() ? '\xf5' : '\xf4'; + else if (value.is_number_unsigned()) + writeHead(bytes, 0, value.get()); + else + invalid("unsupported value"); +} + +void checkOutputs(const Full & drv) +{ + for (auto & [name, output] : drv.outputs) { + if (auto fixed = std::get_if(&output.raw)) { + auto algo = fixed->ca.hash.algo; + if (fixed->ca.method == ContentAddressMethod::Raw::Text && algo != HashAlgorithm::SHA256) + invalid("text content addressing requires SHA-256"); + if (fixed->ca.method == ContentAddressMethod::Raw::Git && algo != HashAlgorithm::SHA1 + && algo != HashAlgorithm::SHA256) + invalid("Git content addressing requires SHA-1 or SHA-256"); + } + } +} + +using EncodedMap = std::vector>; + +std::string encodeString(std::string_view value, uint8_t major) +{ + std::string bytes; + writeHead(bytes, major, value.size()); + bytes += value; + return bytes; +} + +std::string encodeMap(EncodedMap entries) +{ + // RFC 8949 length-first deterministic ordering, comparing unsigned bytes. + std::sort(entries.begin(), entries.end(), [](auto & a, auto & b) { + if (a.first.size() != b.first.size()) + return a.first.size() < b.first.size(); + return std::lexicographical_compare( + a.first.begin(), a.first.end(), b.first.begin(), b.first.end(), [](unsigned char x, unsigned char y) { + return x < y; + }); + }); + std::string bytes; + writeHead(bytes, 5, entries.size()); + for (auto & [key, value] : entries) { + bytes += key; + bytes += value; + } + return bytes; +} + +struct CborWriter +{ + EncodedMap entries; + + void encoded(const char * key, std::string value) + { + entries.emplace_back(encodeString(key, 3), std::move(value)); + } + + void value(const char * key, json value) + { + if (std::string_view(key) == "inputs") { + auto & sources = value.at("srcs"); + std::sort(sources.begin(), sources.end()); + for (auto & [path, node] : getObject(value.at("drvs"))) + input(node); + } + std::string bytes; + writeValue(bytes, value); + encoded(key, std::move(bytes)); + } + + void bytes(const char * key, const std::string & value) + { + encoded(key, encodeString(value, 2)); + } + + void byteStrings(const char * key, const Strings & values) + { + std::string bytes; + writeHead(bytes, 4, values.size()); + for (auto & value : values) + bytes += encodeString(value, 2); + encoded(key, std::move(bytes)); + } + + void byteMap(const char * key, const StringPairs & values) + { + EncodedMap entries; + for (auto & [name, value] : values) + entries.emplace_back(encodeString(name, 2), encodeString(value, 2)); + encoded(key, encodeMap(std::move(entries))); + } + + void attrs(const char * key, const StructuredAttrs & value) + { + bytes(key, value.unparse().second); + } +}; + +struct CborReader +{ + std::map fields; + std::set used; + + explicit CborReader(std::string_view bytes) + { + Reader reader(bytes); + reader.map(3, [&](const auto & key, auto & reader) { fields.emplace(key, reader.item()); }); + if (!reader.done()) + invalid("trailing data"); + auto version = value("version"); + if (!version.is_number_unsigned() || version != expectedCborVersion) + invalid("unsupported version"); + } + + Reader field(const char * key) + { + auto i = fields.find(key); + if (i == fields.end()) + invalid("missing field"); + used.insert(key); + return Reader(i->second); + } + + json value(const char * key) + { + auto reader = field(key); + return reader.read(); + } + + std::string bytes(const char * key) + { + auto reader = field(key); + return reader.string(2); + } + + Strings byteStrings(const char * key) + { + auto reader = field(key); + auto size = reader.count(4); + Strings values; + for (uint64_t i = 0; i < size; ++i) + values.push_back(reader.string(2)); + return values; + } + + StringPairs byteMap(const char * key) + { + auto reader = field(key); + StringPairs values; + reader.map(2, [&](const auto & name, auto & reader) { values.emplace(name, reader.string(2)); }); + return values; + } + + std::optional attrs(const char * key) + { + if (fields.contains(key)) + return StructuredAttrs::parse(bytes(key)); + return std::nullopt; + } + + void finish() + { + if (used.size() != fields.size()) + invalid("unknown field"); + } +}; + +} // namespace + +std::string toCbor(const Full & drv) +{ + try { + checkOutputs(drv); + CborWriter writer; + writer.value("version", expectedCborVersion); + structured::write(drv, writer); + return encodeMap(std::move(writer.entries)); + } catch (json::exception & e) { + invalid(e.what()); + } +} + +std::string toCbor(const std::map & drvs) +{ + EncodedMap entries; + for (auto & [path, drv] : drvs) + entries.emplace_back(encodeString(path.to_string(), 3), toCbor(drv)); + CborWriter writer; + writer.value("version", expectedCborVersion); + writer.encoded("derivations", encodeMap(std::move(entries))); + return encodeMap(std::move(writer.entries)); +} + +Full parseCbor(std::string_view bytes, const ExperimentalFeatureSettings & xpSettings) +{ + try { + CborReader reader(bytes); + auto inputs = reader.value("inputs"); + fields(inputs, {"srcs", "drvs"}); + stringSet(inputs.at("srcs")); + for (auto & [path, node] : getObject(inputs.at("drvs"))) + input(node); + auto outputs = reader.value("outputs"); + for (auto & [name, output] : getObject(outputs)) { + if (output.contains("impure") && output.at("impure") != json(true)) + invalid("impure must be true"); + } + auto drv = structured::read>(reader, xpSettings); + reader.finish(); + checkOutputs(drv); + return drv; + } catch (json::exception & e) { + invalid(e.what()); + } +} + +} // namespace nix::derivation diff --git a/src/libstore/derivation/json.cc b/src/libstore/derivation/json.cc index 04ca28c75c9f..552271b1eae2 100644 --- a/src/libstore/derivation/json.cc +++ b/src/libstore/derivation/json.cc @@ -1,4 +1,5 @@ #include "nix/store/derivations.hh" +#include "structured.hh" #include "nix/store/derivation/full-inputs.hh" #include "nix/store/store-api.hh" #include "nix/util/json-utils.hh" @@ -104,192 +105,96 @@ nix::DerivationOutput adl_serializer::from_json( } } -static void inputsToJson(json & res, const nix::StorePathSet & inputs) -{ - res = nlohmann::json::array(); - for (auto & input : inputs) - res.emplace_back(input); -} - -static void inputsToJson(json & res, const nix::derivation::FullInputs & inputs) -{ - using namespace nix; - res = nlohmann::json::object(); - - inputsToJson(res["srcs"], inputs.srcs); +namespace { - auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { - auto value = nlohmann::json::object(); - value["outputs"] = inputNode.value; - { - auto next = nlohmann::json::object(); - for (auto & [outputId, childNode] : inputNode.childMap) - next[outputId] = doInput(childNode); - value["dynamicOutputs"] = std::move(next); - } - return value; - }; - - auto & inputDrvsObj = res["drvs"]; - inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : inputs.drvs.map) - inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); -} - -static void inputsToJson(json & res, const std::set & inputs) +struct JsonWriter { - using namespace nix::derivation; - inputsToJson(res, FullInputs::fromSet(inputs)); -} - -template -void adl_serializer>::to_json( - json & res, const nix::derivation::Derivation & d) -{ - using namespace nix; - res = nlohmann::json::object(); - - res["name"] = d.name; - res["version"] = expectedJsonVersionDerivation; + json & object; + void value(const char * key, const json & value) { - nlohmann::json & outputsObj = res["outputs"]; - outputsObj = nlohmann::json::object(); - for (auto & [outputName, output] : d.outputs) - outputsObj[outputName] = output; + object[key] = value; } - inputsToJson(res["inputs"], d.inputs); + void bytes(const char * key, const std::string & value) + { + object[key] = value; + } - res["system"] = d.platform; - res["builder"] = d.builder; - res["args"] = d.args; - res["env"] = d.env; + void byteStrings(const char * key, const nix::Strings & value) + { + object[key] = value; + } - if (d.structuredAttrs) - res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; -} + void byteMap(const char * key, const nix::StringPairs & value) + { + object[key] = value; + } -template -static Inputs inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings); + void attrs(const char * key, const nix::StructuredAttrs & value) + { + object[key] = value.structuredAttrs; + } +}; -template<> -nix::StorePathSet inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings &) +struct JsonReader { - using namespace nix; - StorePathSet inputSrcs; - for (auto & input : getArray(inputsJson)) - inputSrcs.insert(input); - return inputSrcs; -} + const json::object_t & object; -template<> -nix::derivation::FullInputs inputsFromJson( - const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings) -{ - using namespace nix; - using namespace derivation; + const json & value(const char * key) + { + return nix::valueAt(object, key); + } - auto inputsObj = getObject(inputsJson); - FullInputs inputs; + std::string bytes(const char * key) + { + return nix::getString(value(key)); + } - try { - for (auto & input : getArray(valueAt(inputsObj, "srcs"))) - inputs.srcs.insert(input); - } catch (Error & e) { - e.addTrace({}, "while reading key 'srcs'"); - throw; + nix::Strings byteStrings(const char * key) + { + return nix::getStringList(value(key)); } - try { - auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { - auto & json = getObject(_json); - DerivedPathMap::ChildNode node; - node.value = getStringSet(valueAt(json, "outputs")); - for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { - xpSettings.require( - Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); - node.childMap[outputId] = doInput(childNode); - } - return node; - }; - for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(inputsObj, "drvs"))) - inputs.drvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); - } catch (Error & e) { - e.addTrace({}, "while reading key 'drvs'"); - throw; + nix::StringPairs byteMap(const char * key) + { + return nix::getStringMap(value(key)); } - return inputs; -} + std::optional attrs(const char * key) + { + if (auto value = nix::get(object, key)) + return nix::StructuredAttrs{*value}; + return std::nullopt; + } +}; + +} // namespace -template<> -std::set inputsFromJson>( - const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings) +template +void adl_serializer>::to_json( + json & res, const nix::derivation::Derivation & drv) { - using namespace nix::derivation; - return inputsFromJson(inputsJson, xpSettings).toSet(); + res = json::object(); + res["version"] = nix::expectedJsonVersionDerivation; + JsonWriter writer{res}; + nix::derivation::structured::write(drv, writer); } template nix::derivation::Derivation adl_serializer>::from_json( - const json & _json, const nix::ExperimentalFeatureSettings & xpSettings) + const json & object, const nix::ExperimentalFeatureSettings & xpSettings) { using namespace nix; - using namespace derivation; - - auto & json = getObject(_json); - { - auto version = getUnsigned(valueAt(json, "version")); - if (version != expectedJsonVersionDerivation) - throw Error( - "Unsupported derivation JSON format version %d, only format version %d is currently supported.", - version, - expectedJsonVersionDerivation); - } - - return derivation::Derivation{ - .outputs = - [&] { - Outputs<> outputs; - try { - for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) - outputs.insert_or_assign( - outputName, adl_serializer::from_json(output, xpSettings)); - } catch (Error & e) { - e.addTrace({}, "while reading key 'outputs'"); - throw; - } - return outputs; - }(), - .inputs = - [&] { - try { - return inputsFromJson(valueAt(json, "inputs"), xpSettings); - } catch (Error & e) { - e.addTrace({}, "while reading key 'inputs'"); - throw; - } - }(), - .platform = getString(valueAt(json, "system")), - .builder = getString(valueAt(json, "builder")), - .args = getStringList(valueAt(json, "args")), - .env = - [&] { - try { - return getStringMap(valueAt(json, "env")); - } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); - throw; - } - }(), - .structuredAttrs = [&]() -> std::optional { - if (auto structuredAttrs = get(json, "structuredAttrs")) - return StructuredAttrs{*structuredAttrs}; - return std::nullopt; - }(), - .name = getString(valueAt(json, "name")), - }; + auto & fields = getObject(object); + auto version = getUnsigned(valueAt(fields, "version")); + if (version != expectedJsonVersionDerivation) + throw Error( + "Unsupported derivation JSON format version %d, only format version %d is currently supported.", + version, + expectedJsonVersionDerivation); + JsonReader reader{fields}; + return derivation::structured::read(reader, xpSettings); } template struct adl_serializer; diff --git a/src/libstore/derivation/structured.hh b/src/libstore/derivation/structured.hh new file mode 100644 index 000000000000..ef62a7f53c74 --- /dev/null +++ b/src/libstore/derivation/structured.hh @@ -0,0 +1,182 @@ +#pragma once + +#include "nix/store/derivations.hh" +#include "nix/store/derivation/full-inputs.hh" +#include "nix/util/json-utils.hh" + +#include + +namespace nix::derivation::structured { + +using nlohmann::adl_serializer; +using nlohmann::json; + +/** + * Shared derivation fields for JSON and CBOR. The wire codecs distinguish + * text from arbitrary bytes, including byte-valued map keys. Only the + * text-only input and output substructures use a JSON value as a carrier. + * Changes to this schema must consider both wire format versions. + */ +inline void inputsToJson(json & res, const nix::StorePathSet & inputs) +{ + res = nlohmann::json::array(); + for (auto & input : inputs) + res.emplace_back(input); +} + +inline void inputsToJson(json & res, const nix::derivation::FullInputs & inputs) +{ + using namespace nix; + res = nlohmann::json::object(); + + inputsToJson(res["srcs"], inputs.srcs); + + auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + + auto & inputDrvsObj = res["drvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputs.drvs.map) + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); +} + +inline void inputsToJson(json & res, const std::set & inputs) +{ + using namespace nix::derivation; + inputsToJson(res, FullInputs::fromSet(inputs)); +} + +template +Inputs inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings); + +template<> +inline nix::StorePathSet +inputsFromJson(const json & inputsJson, const nix::ExperimentalFeatureSettings &) +{ + using namespace nix; + StorePathSet inputSrcs; + for (auto & input : getArray(inputsJson)) + inputSrcs.insert(input); + return inputSrcs; +} + +template<> +inline nix::derivation::FullInputs inputsFromJson( + const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings) +{ + using namespace nix; + using namespace derivation; + + auto inputsObj = getObject(inputsJson); + FullInputs inputs; + + try { + for (auto & input : getArray(valueAt(inputsObj, "srcs"))) + inputs.srcs.insert(input); + } catch (Error & e) { + e.addTrace({}, "while reading key 'srcs'"); + throw; + } + + try { + auto doInput = [&](this const auto & doInput, const auto & _json) -> DerivedPathMap::ChildNode { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { + xpSettings.require( + Xp::DynamicDerivations, [&] { return fmt("dynamic output '%s' in JSON", outputId); }); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(inputsObj, "drvs"))) + inputs.drvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'drvs'"); + throw; + } + + return inputs; +} + +template<> +inline std::set inputsFromJson>( + const json & inputsJson, const nix::ExperimentalFeatureSettings & xpSettings) +{ + using namespace nix::derivation; + return inputsFromJson(inputsJson, xpSettings).toSet(); +} + +template +void write(const Derivation & drv, Writer & writer) +{ + writer.value("name", drv.name); + json outputs = json::object(); + for (auto & [name, output] : drv.outputs) + outputs[name] = output; + writer.value("outputs", outputs); + json inputs; + inputsToJson(inputs, drv.inputs); + writer.value("inputs", inputs); + writer.value("system", drv.platform); + writer.bytes("builder", drv.builder); + writer.byteStrings("args", drv.args); + writer.byteMap("env", drv.env); + if (drv.structuredAttrs) + writer.attrs("structuredAttrs", *drv.structuredAttrs); +} + +template +Derivation read(Reader & reader, const ExperimentalFeatureSettings & xpSettings) +{ + return Derivation{ + .outputs = + [&] { + Outputs<> outputs; + try { + auto value = reader.value("outputs"); + for (auto & [name, output] : getObject(value)) + outputs.insert_or_assign(name, adl_serializer::from_json(output, xpSettings)); + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; + } + return outputs; + }(), + .inputs = + [&] { + try { + return inputsFromJson(reader.value("inputs"), xpSettings); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + }(), + .platform = getString(reader.value("system")), + .builder = reader.bytes("builder"), + .args = reader.byteStrings("args"), + .env = + [&] { + try { + return reader.byteMap("env"); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + }(), + .structuredAttrs = reader.attrs("structuredAttrs"), + .name = getString(reader.value("name")), + }; +} + +} // namespace nix::derivation::structured diff --git a/src/libstore/include/nix/store/derivation/cbor.hh b/src/libstore/include/nix/store/derivation/cbor.hh new file mode 100644 index 000000000000..1c73e04cae73 --- /dev/null +++ b/src/libstore/include/nix/store/derivation/cbor.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "nix/store/derivations.hh" + +namespace nix::derivation { + +inline constexpr unsigned expectedCborVersion = 1; + +/** + * Deterministic CBOR interchange encoding. Builder, arguments, and both + * environment names and values are byte strings. Structured attributes + * retain their original JSON bytes. See protocols/derivation-cbor.md. + */ +std::string toCbor(const Full & drv); + +/** Encode a collection, with store-path base names as text map keys. */ +std::string toCbor(const std::map & drvs); + +/** + * Read a single derivation, accepting nonminimal encodings and unordered + * maps and sets. Call the normal derivation validation before storing it. + */ +Full parseCbor(std::string_view bytes, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +} // namespace nix::derivation diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index af82f8dea449..48af813b3970 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -35,6 +35,7 @@ headers = [ config_pub_h ] + files( 'daemon.hh', 'derivation-options.hh', 'derivation/aterm.hh', + 'derivation/cbor.hh', 'derivation/full-inputs.hh', 'derivation/masked.hh', 'derivation/output.hh', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 1a66d4bf08fa..35311f8bb74e 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -281,6 +281,7 @@ sources = files( 'daemon.cc', 'derivation-options.cc', 'derivation/aterm.cc', + 'derivation/cbor.cc', 'derivation/full-inputs.cc', 'derivation/json.cc', 'derivation/masked.cc', diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 04e2f0438bbd..6dc3d62fd780 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -4,6 +4,7 @@ #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" #include "nix/store/derivations.hh" +#include "nix/store/derivation/cbor.hh" #include "nix/store/globals.hh" #include @@ -13,6 +14,17 @@ namespace nix { struct CmdAddDerivation : MixDryRun, StoreCommand { + bool cbor = false; + + CmdAddDerivation() + { + addFlag({ + .longName = "cbor", + .description = "Read a derivation in CBOR format version 1 from standard input.", + .handler = {&cbor, true}, + }); + } + std::string description() override { return "add a store derivation"; @@ -32,9 +44,13 @@ struct CmdAddDerivation : MixDryRun, StoreCommand void run(ref store) override { - auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); - - auto drv = derivation::parseJsonAndValidate(*store, json); + auto bytes = drainFD(STDIN_FILENO); + auto drv = cbor ? derivation::parseCbor(bytes) + : derivation::parseJsonAndValidate(*store, nlohmann::json::parse(bytes)); + if (cbor) { + derivation::fillInOutputPaths(drv, *store); + derivation::checkInvariants(drv, *store); + } auto drvPath = (dryRun || settings.readOnlyMode) ? computeStorePath(*store, drv) : store->writeDerivation(drv, NoRepair); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index 69a821d4ee02..bcc06f25262c 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -15,4 +15,8 @@ a Nix expression evaluates. `nix derivation add` takes a single derivation in the JSON format. See [the manual](@docroot@/protocols/json/derivation/index.md) for a documentation of this format. +With `--cbor`, it reads a single derivation in the +[CBOR format](@docroot@/protocols/derivation-cbor.md), which preserves arbitrary +bytes in the builder, arguments, and environment names and values. + )"" diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 5880c6c8f13b..f809b24f5040 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -5,6 +5,7 @@ #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" #include "nix/store/derivations.hh" +#include "nix/store/derivation/cbor.hh" #include using json = nlohmann::json; @@ -14,9 +15,15 @@ namespace nix { struct CmdShowDerivation : InstallablesCommand, MixPrintJSON { bool recursive = false; + bool cbor = false; CmdShowDerivation() { + addFlag({ + .longName = "cbor", + .description = "Write derivations in CBOR format version 1.", + .handler = {&cbor, true}, + }); addFlag({ .longName = "recursive", .shortName = 'r', @@ -46,12 +53,26 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON { auto drvPaths = Installable::toDerivations(store, installables, true); + if (cbor && !recursive && drvPaths.size() == 1) { + writeFull(STDOUT_FILENO, derivation::toCbor(store->readDerivation(*drvPaths.begin()))); + return; + } + if (recursive) { StorePathSet closure; store->computeFSClosure(drvPaths, closure); drvPaths = std::move(closure); } + if (cbor) { + std::map drvs; + for (auto & drvPath : drvPaths) + if (drvPath.isDerivation()) + drvs.emplace(drvPath, store->readDerivation(drvPath)); + writeFull(STDOUT_FILENO, derivation::toCbor(drvs)); + return; + } + json jsonRoot = json::object(); for (auto & drvPath : drvPaths) { diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 6948b5ba72e6..374783c740df 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -51,6 +51,13 @@ By default, this command only shows top-level derivations, but with `nix derivation show` outputs a JSON map of [store path]s to derivations in JSON format. See [the manual](@docroot@/protocols/json/derivation/index.md) for a documentation of this format. +With `--cbor`, it writes the +[CBOR format](@docroot@/protocols/derivation-cbor.md), which preserves arbitrary +bytes in the builder, arguments, and environment names and values. +A single derivation without `--recursive` can be piped directly to +`nix derivation add --cbor`. Multiple derivations or `--recursive` produce +a collection. + [store path]: @docroot@/store/store-path.md )"" diff --git a/tests/functional/derivation-json.sh b/tests/functional/derivation-json.sh index d2518b6960e1..a84a033fda85 100755 --- a/tests/functional/derivation-json.sh +++ b/tests/functional/derivation-json.sh @@ -10,6 +10,50 @@ nix derivation show "$drvPath" | jq '.derivations[]' > "$TEST_HOME/simple.json" drvPath2=$(nix derivation add < "$TEST_HOME/simple.json") [[ "$drvPath" = "$drvPath2" ]] +nix derivation show --cbor "$drvPath" > "$TEST_HOME/simple.cbor" +drvPathCbor=$(nix derivation add --cbor < "$TEST_HOME/simple.cbor") +[[ "$drvPath" = "$drvPathCbor" ]] +[[ "$drvPath" = "$(nix derivation add --cbor --dry-run < "$TEST_HOME/simple.cbor")" ]] +nix derivation show --cbor "$drvPathCbor" > "$TEST_HOME/simple-roundtrip.cbor" +cmp "$TEST_HOME/simple.cbor" "$TEST_HOME/simple-roundtrip.cbor" +nix derivation show --cbor --recursive "$drvPath" > "$TEST_HOME/recursive.cbor" +test -s "$TEST_HOME/recursive.cbor" +depDrvPath=$(nix-instantiate dependencies.nix) +nix derivation show --cbor --recursive "$depDrvPath" > "$TEST_HOME/dependencies.cbor" +nix derivation show --cbor "$drvPath" "$depDrvPath" > "$TEST_HOME/multiple.cbor" +nix derivation show --cbor "$depDrvPath" "$drvPath" > "$TEST_HOME/multiple-reversed.cbor" +cmp "$TEST_HOME/multiple.cbor" "$TEST_HOME/multiple-reversed.cbor" +expectStderr 1 nix derivation add --cbor < "$TEST_HOME/simple.json" | grepQuiet 'CBOR' + +verbatimDrv=$(nix-instantiate --expr 'with import ./config.nix; mkDerivation { + name = "verbatim-structured-attrs"; + __json = "{ \"z\": 0, \"a\": 1 }"; +}') +nix derivation show --cbor "$verbatimDrv" > "$TEST_HOME/verbatim.cbor" +[[ "$verbatimDrv" = "$(nix derivation add --cbor < "$TEST_HOME/verbatim.cbor")" ]] +nix derivation show --cbor "$verbatimDrv" > "$TEST_HOME/verbatim-roundtrip.cbor" +cmp "$TEST_HOME/verbatim.cbor" "$TEST_HOME/verbatim-roundtrip.cbor" + +# Invalid UTF-8 must survive in every byte-valued field, including env keys. +printf '\200\377' > "$TEST_HOME/arbitrary-bytes" +bytesDrv=$(nix-instantiate --argstr bytesFile "$TEST_HOME/arbitrary-bytes" --expr ' + { bytesFile }: + let bytes = builtins.readFile bytesFile; in + derivation { + name = "arbitrary-bytes"; + system = (import ./config.nix).system; + builder = "/builder-${bytes}"; + args = [ "" bytes ]; + "${bytes}" = bytes; + payload = bytes; + } +') +nix derivation show --cbor "$bytesDrv" > "$TEST_HOME/bytes.cbor" +[[ "$bytesDrv" = "$(nix derivation add --cbor < "$TEST_HOME/bytes.cbor")" ]] +[[ "$bytesDrv" = "$(nix derivation add --cbor --dry-run < "$TEST_HOME/bytes.cbor")" ]] +nix derivation show --cbor "$bytesDrv" > "$TEST_HOME/bytes-roundtrip.cbor" +cmp "$TEST_HOME/bytes.cbor" "$TEST_HOME/bytes-roundtrip.cbor" + # Derivation is input addressed, all outputs have a path jq -e '.outputs | .[] | has("path")' < "$TEST_HOME/simple.json" From 72dc19ced48e1aff5e1bf55868cdf08bc8843d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sat, 5 Sep 2026 18:19:45 -0500 Subject: [PATCH 2/2] Test CBOR normalization preserves derivation identity --- src/libstore-tests/derivation/cbor.cc | 39 +++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/libstore-tests/derivation/cbor.cc b/src/libstore-tests/derivation/cbor.cc index 00c830c71e6d..3a350c4fdc47 100644 --- a/src/libstore-tests/derivation/cbor.cc +++ b/src/libstore-tests/derivation/cbor.cc @@ -1,3 +1,4 @@ +#include #include #include @@ -136,13 +137,41 @@ TEST_F(DerivationTest, CborRejectsInvalidSchema) } TEST_F(DerivationTest, CborNormalizesEncoding) +{ + auto expected = derivation::parseCbor(golden); + auto expectedPath = computeStorePath(*store, expected); + for (auto & bytes : { + encode(fixture()), // Maps ordered lexically instead of by encoded key length. + golden.substr(0, golden.size() - 1) + "\x18\x01"s, + golden.substr(0, golden.size() - 1) + "\x19\x00\x01"s, + golden.substr(0, golden.size() - 1) + "\x1a\x00\x00\x00\x01"s, + golden.substr(0, golden.size() - 1) + "\x1b\x00\x00\x00\x00\x00\x00\x00\x01"s, + "\xb8\x08" + golden.substr(1), + }) { + ASSERT_NE(bytes, golden); + auto drv = derivation::parseCbor(bytes); + EXPECT_EQ(derivation::toCbor(drv), golden); + EXPECT_EQ(computeStorePath(*store, drv), expectedPath); + } +} + +TEST_F(DerivationTest, CborNormalizesSetsPreservingIdentity) { auto value = fixture(); - EXPECT_EQ(derivation::toCbor(derivation::parseCbor(encode(value))), golden); - auto nonminimal = golden.substr(0, golden.size() - 1) + "\x18\x01"; - EXPECT_EQ(derivation::toCbor(derivation::parseCbor(nonminimal)), golden); - nonminimal = "\xb8\x08" + golden.substr(1); - EXPECT_EQ(derivation::toCbor(derivation::parseCbor(nonminimal)), golden); + value["inputs"]["srcs"] = {"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-a", "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-z"}; + auto & outputs = value["inputs"]["drvs"]["c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep.drv"]; + outputs = {{"outputs", {"a", "z"}}, {"dynamicOutputs", json::object()}}; + auto expected = derivation::parseCbor(encode(value)); + auto canonical = derivation::toCbor(expected); + auto expectedPath = computeStorePath(*store, expected); + + std::reverse(value["inputs"]["srcs"].begin(), value["inputs"]["srcs"].end()); + std::reverse(outputs["outputs"].begin(), outputs["outputs"].end()); + auto bytes = encode(value); + ASSERT_NE(bytes, canonical); + auto drv = derivation::parseCbor(bytes); + EXPECT_EQ(derivation::toCbor(drv), canonical); + EXPECT_EQ(computeStorePath(*store, drv), expectedPath); } TEST_F(DerivationTest, CborEnvironmentKeyTypesAndDuplicates)