From d690cbfb584a2d2067957b5ae36e7780a51ecd3d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 4 Sep 2026 23:36:48 +0300 Subject: [PATCH 1/4] libfetchers: Disable GIT_OPT_ENABLE_STRICT_HASH_VERIFICATION See the comment for reasoning. Re-validiating all objects we read is probably out of scope for nix, just to catch occasional odb corruption. --- src/libfetchers/git-utils.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 58e5dc42f09..0576bbd6d90 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -176,6 +176,12 @@ static void initLibGit2() std::call_once(initialized, []() { if (git_libgit2_init() < 0) throw GitError("initialising libgit2"); + + /* Nuke the "hashing on all reads" behavior, since that can lead to bad + performance https://github.com/libgit2/libgit2/issues/4951. It's a + compromise of course, but one that is mostly in line with git cli and + like how we don't recalculate narHash when reading from a store. */ + git_libgit2_opts(GIT_OPT_ENABLE_STRICT_HASH_VERIFICATION, 0); }); } From 21dafd187672d1e1d936a443db36be8821b46a74 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 5 Sep 2026 01:11:29 +0300 Subject: [PATCH 2/4] libfetchers: Patch libgit2 to use zlib-ng Will be upstreamed, just WIP for visibility now. --- packaging/dependencies.nix | 28 +- packaging/patches/0001-zlib-ng-support.patch | 379 +++++++++++++++++++ 2 files changed, 394 insertions(+), 13 deletions(-) create mode 100644 packaging/patches/0001-zlib-ng-support.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 34fdf8d9768..2ec60d95157 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -101,20 +101,22 @@ scope: { ]; }); - libgit2 = - if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then - pkgs.libgit2 - else - # Grab newer libgit2. - pkgs.libgit2.overrideAttrs rec { - version = "1.9.4"; - src = pkgs.fetchFromGitHub { - owner = "libgit2"; - repo = "libgit2"; - tag = "v${version}"; - hash = "sha256-ZKUiz3pdFE2SKxh53X2oyr7hs32Njj5YVA0OXDXz7h0="; - }; + libgit2 = pkgs.libgit2.overrideAttrs ( + finalAttrs: prevAttrs: { + version = "2.0.0-rc.1"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + rev = "ae45d0d168f7e8dbfdb8c623589cb51caac96ab3"; + hash = "sha256-3sbqHm37SOwBeFgtjI2DLN6kx1F7G2N1m6rRIkqDXNI="; }; + patches = prevAttrs.patches or [ ] ++ [ + ./patches/0001-zlib-ng-support.patch + ]; + cmakeFlags = prevAttrs.cmakeFlags ++ [ "-DUSE_COMPRESSION=zlib-ng" ]; + buildInputs = prevAttrs.buildInputs ++ [ pkgs.zlib-ng ]; + } + ); # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = diff --git a/packaging/patches/0001-zlib-ng-support.patch b/packaging/patches/0001-zlib-ng-support.patch new file mode 100644 index 00000000000..48c8be5ab4f --- /dev/null +++ b/packaging/patches/0001-zlib-ng-support.patch @@ -0,0 +1,379 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 87b33a9bc..19ec37c3c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -35,7 +35,7 @@ option(BUILD_FUZZERS "Build the fuzz targets" + set(USE_AUTH_NEGOTIATE "" CACHE STRING "Enable Negotiate (SPNEGO) authentication support. One of GSSAPI or win32.") + # set(USE_XDIFF "" CACHE STRING "Specifies the xdiff implementation; either system or builtin.") + set(USE_REGEX "" CACHE STRING "Selects regex provider. One of regcomp_l, pcre2, pcre, regcomp, or builtin.") +- set(USE_COMPRESSION "" CACHE STRING "Selects compression backend. Either builtin or zlib.") ++ set(USE_COMPRESSION "" CACHE STRING "Selects compression backend. Either builtin, zlib or zlib-ng.") + set(USE_NSEC "" CACHE STRING "Enable nanosecond precision timestamps. One of ON, OFF, or a specific provider: mtimespec, mtim, mtime, or win32. (Defaults to ON).") + + if(APPLE) +diff --git a/cmake/SelectCompression.cmake b/cmake/SelectCompression.cmake +index d0a4b5664..764af5197 100644 +--- a/cmake/SelectCompression.cmake ++++ b/cmake/SelectCompression.cmake +@@ -32,6 +32,19 @@ elseif(USE_COMPRESSION STREQUAL "zlib") + endif() + + set(GIT_COMPRESSION_ZLIB 1) ++elseif(USE_COMPRESSION STREQUAL "zlib-ng") ++ find_package(zlib-ng) ++ find_package(ZLIB) # For reftable library ++ ++ if(NOT zlib-ng_FOUND) ++ message(FATAL_ERROR "system zlib-ng was requested but not found") ++ endif() ++ ++ if(NOT ZLIB_FOUND) ++ message(FATAL_ERROR "system zlib was requested but not found") ++ endif() ++ ++ set(GIT_COMPRESSION_ZLIB_NG 1) + elseif(USE_COMPRESSION STREQUAL "builtin") + set(GIT_COMPRESSION_BUILTIN 1) + endif() +@@ -45,6 +58,18 @@ if(GIT_COMPRESSION_ZLIB) + list(APPEND LIBGIT2_PC_REQUIRES "zlib") + endif() + add_feature_info("Compression" ON "using system zlib") ++elseif(GIT_COMPRESSION_ZLIB_NG) ++ list(APPEND LIBGIT2_SYSTEM_INCLUDES ${zlib-ng_INCLUDE_DIR}) ++ list(APPEND LIBGIT2_SYSTEM_LIBS "z-ng") # FIXME ++ list(APPEND LIBGIT2_SYSTEM_INCLUDES ${ZLIB_INCLUDE_DIRS}) ++ list(APPEND LIBGIT2_SYSTEM_LIBS ${ZLIB_LIBRARIES}) ++ if(APPLE OR CMAKE_SYSTEM_NAME MATCHES "FreeBSD") ++ list(APPEND LIBGIT2_PC_LIBS "-lz") ++ else() ++ list(APPEND LIBGIT2_PC_REQUIRES "zlib") ++ endif() ++ list(APPEND LIBGIT2_PC_REQUIRES "zlib-ng;zlib") ++ add_feature_info("Compression" ON "using system zlib-ng") + elseif(GIT_COMPRESSION_BUILTIN) + add_subdirectory("${PROJECT_SOURCE_DIR}/deps/zlib" "${PROJECT_BINARY_DIR}/deps/zlib") + list(APPEND LIBGIT2_DEPENDENCY_INCLUDES "${PROJECT_SOURCE_DIR}/deps/zlib") +diff --git a/src/libgit2/indexer.c b/src/libgit2/indexer.c +index 68f595696..580628e65 100644 +--- a/src/libgit2/indexer.c ++++ b/src/libgit2/indexer.c +@@ -345,14 +345,14 @@ static int crc_object(uint32_t *crc_out, git_mwindow_file *mwf, off64_t start, o + unsigned int left, len; + git_mwindow *w = NULL; + +- crc = crc32(0L, Z_NULL, 0); ++ crc = GIT_ZLIB_SYMBOL(crc32)(0L, Z_NULL, 0); + while (size) { + ptr = git_mwindow_open(mwf, &w, start, (size_t)size, &left); + if (ptr == NULL) + return -1; + + len = min(left, (unsigned int)size); +- crc = crc32(crc, ptr, len); ++ crc = GIT_ZLIB_SYMBOL(crc32)(crc, ptr, len); + size -= len; + start += len; + git_mwindow_close(&w); +@@ -606,7 +606,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start) + + git_oid_cpy(&pentry->id, &oid); + git_oid_cpy(&entry->oid, &oid); +- entry->crc = crc32(0L, Z_NULL, 0); ++ entry->crc = GIT_ZLIB_SYMBOL(crc32)(0L, Z_NULL, 0); + + entry_size = (size_t)(idx->off - entry_start); + if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0) +@@ -1008,7 +1008,7 @@ static int inject_object(git_indexer *idx, git_oid *id) + entry = git__calloc(1, sizeof(*entry)); + GIT_ERROR_CHECK_ALLOC(entry); + +- entry->crc = crc32(0L, Z_NULL, 0); ++ entry->crc = GIT_ZLIB_SYMBOL(crc32)(0L, Z_NULL, 0); + + /* Write out the object header */ + if ((error = git_packfile__object_header(&hdr_len, hdr, len, git_odb_object_type(obj))) < 0 || +@@ -1016,7 +1016,7 @@ static int inject_object(git_indexer *idx, git_oid *id) + goto cleanup; + + idx->pack->mwf.size += hdr_len; +- entry->crc = crc32(entry->crc, hdr, (uInt)hdr_len); ++ entry->crc = GIT_ZLIB_SYMBOL(crc32)(entry->crc, hdr, (uInt)hdr_len); + + if ((error = git_zstream_deflatebuf(&buf, data, len)) < 0) + goto cleanup; +@@ -1026,7 +1026,7 @@ static int inject_object(git_indexer *idx, git_oid *id) + goto cleanup; + + idx->pack->mwf.size += buf.size; +- entry->crc = htonl(crc32(entry->crc, (unsigned char *)buf.ptr, (uInt)buf.size)); ++ entry->crc = htonl(GIT_ZLIB_SYMBOL(crc32)(entry->crc, (unsigned char *)buf.ptr, (uInt)buf.size)); + git_str_dispose(&buf); + + /* Write a fake trailer so the pack functions play ball */ +diff --git a/src/libgit2/libgit2.c b/src/libgit2/libgit2.c +index 081a25087..0c8a8ea81 100644 +--- a/src/libgit2/libgit2.c ++++ b/src/libgit2/libgit2.c +@@ -233,6 +233,8 @@ const char *git_libgit2_feature_backend(git_feature_t feature) + case GIT_FEATURE_COMPRESSION: + #if defined(GIT_COMPRESSION_ZLIB) + return "zlib"; ++#elif defined(GIT_COMPRESSION_ZLIB_NG) ++ return "zlib-ng"; + #elif defined(GIT_COMPRESSION_BUILTIN) + return "builtin"; + #else +diff --git a/src/libgit2/odb.c b/src/libgit2/odb.c +index 3b2306c3c..72bae5088 100644 +--- a/src/libgit2/odb.c ++++ b/src/libgit2/odb.c +@@ -7,7 +7,6 @@ + + #include "odb.h" + +-#include + #include "git2/object.h" + #include "git2/sys/odb_backend.h" + #include "futils.h" +diff --git a/src/libgit2/odb_loose.c b/src/libgit2/odb_loose.c +index 0b3a51732..ed0ebcc7c 100644 +--- a/src/libgit2/odb_loose.c ++++ b/src/libgit2/odb_loose.c +@@ -7,7 +7,6 @@ + + #include "common.h" + +-#include + #include "git2/object.h" + #include "git2/sys/odb_backend.h" + #include "futils.h" +diff --git a/src/libgit2/odb_pack.c b/src/libgit2/odb_pack.c +index 6fbc1952a..0ccce7c06 100644 +--- a/src/libgit2/odb_pack.c ++++ b/src/libgit2/odb_pack.c +@@ -7,7 +7,6 @@ + + #include "common.h" + +-#include + #include "git2/repository.h" + #include "git2/indexer.h" + #include "git2/sys/odb_backend.h" +diff --git a/src/util/filebuf.c b/src/util/filebuf.c +index 7afb76b88..a307dacd9 100644 +--- a/src/util/filebuf.c ++++ b/src/util/filebuf.c +@@ -117,7 +117,11 @@ void git_filebuf_cleanup(git_filebuf *file) + /* use the presence of z_buf to decide if we need to deflateEnd */ + if (file->z_buf) { + git__free(file->z_buf); ++#ifdef GIT_COMPRESSION_ZLIB_NG ++ zng_deflateEnd(&file->zs); ++#else + deflateEnd(&file->zs); ++#endif + } + + if (file->path_original) +@@ -158,7 +162,7 @@ static int write_normal(git_filebuf *file, void *source, size_t len) + + static int write_deflate(git_filebuf *file, void *source, size_t len) + { +- z_stream *zs = &file->zs; ++ GIT_ZLIB_STREAM *zs = &file->zs; + + if (len > 0 || file->flush_mode == Z_FINISH) { + zs->next_in = source; +@@ -170,7 +174,7 @@ static int write_deflate(git_filebuf *file, void *source, size_t len) + zs->next_out = file->z_buf; + zs->avail_out = (uInt)file->buf_size; + +- if (deflate(zs, file->flush_mode) == Z_STREAM_ERROR) { ++ if (GIT_ZLIB_SYMBOL(deflate)(zs, file->flush_mode) == Z_STREAM_ERROR) { + file->last_error = BUFERR_ZLIB; + return -1; + } +@@ -319,7 +323,7 @@ int git_filebuf_open_withsize(git_filebuf *file, const char *path, int flags, mo + /* If we are deflating on-write, */ + if (compression != 0) { + /* Initialize the ZLib stream */ +- if (deflateInit(&file->zs, compression) != Z_OK) { ++ if (GIT_ZLIB_SYMBOL(deflateInit)(&file->zs, compression) != Z_OK) { + git_error_set(GIT_ERROR_ZLIB, "failed to initialize zlib"); + goto cleanup; + } +diff --git a/src/util/filebuf.h b/src/util/filebuf.h +index e23b9ed2a..c4cb4cb02 100644 +--- a/src/util/filebuf.h ++++ b/src/util/filebuf.h +@@ -11,7 +11,7 @@ + + #include "futils.h" + #include "hash.h" +-#include ++#include "zstream.h" + + #ifdef GIT_THREADS + # define GIT_FILEBUF_THREADS +@@ -42,7 +42,7 @@ struct git_filebuf { + unsigned char *buffer; + unsigned char *z_buf; + +- z_stream zs; ++ GIT_ZLIB_STREAM zs; + int flush_mode; + + size_t buf_size, buf_pos; +diff --git a/src/util/git2_features.h.in b/src/util/git2_features.h.in +index e890670bb..8e8f5ae14 100644 +--- a/src/util/git2_features.h.in ++++ b/src/util/git2_features.h.in +@@ -32,6 +32,7 @@ + + #cmakedefine GIT_COMPRESSION_BUILTIN 1 + #cmakedefine GIT_COMPRESSION_ZLIB 1 ++#cmakedefine GIT_COMPRESSION_ZLIB_NG 1 + + #cmakedefine GIT_NSEC 1 + #cmakedefine GIT_NSEC_MTIM 1 +diff --git a/src/util/git2_util.h b/src/util/git2_util.h +index 556671e6e..9803bd315 100644 +--- a/src/util/git2_util.h ++++ b/src/util/git2_util.h +@@ -27,6 +27,9 @@ typedef struct git_str git_str; + #endif + #define GIT_INLINE(type) static GIT_INLINE_KEYWORD type + ++#define GIT_CONCAT_RAW(a, b) a ## b ++#define GIT_CONCAT(a, b) GIT_CONCAT_RAW(a, b) ++ + /** Support for gcc/clang __has_builtin intrinsic */ + #ifndef __has_builtin + # define __has_builtin(x) 0 +diff --git a/src/util/zstream.c b/src/util/zstream.c +index efc0883c9..7eabef36d 100644 +--- a/src/util/zstream.c ++++ b/src/util/zstream.c +@@ -7,8 +7,6 @@ + + #include "zstream.h" + +-#include +- + #include "str.h" + + #define ZSTREAM_BUFFER_SIZE (1024 * 1024) +@@ -39,26 +37,26 @@ int git_zstream_init(git_zstream *zstream, git_zstream_t type) + zstream->type = type; + + if (zstream->type == GIT_ZSTREAM_INFLATE) +- zstream->zerr = inflateInit(&zstream->z); ++ zstream->zerr = GIT_ZLIB_SYMBOL(inflateInit)(&zstream->z); + else +- zstream->zerr = deflateInit(&zstream->z, Z_DEFAULT_COMPRESSION); ++ zstream->zerr = GIT_ZLIB_SYMBOL(deflateInit)(&zstream->z, Z_DEFAULT_COMPRESSION); + return zstream_seterr(zstream); + } + + void git_zstream_free(git_zstream *zstream) + { + if (zstream->type == GIT_ZSTREAM_INFLATE) +- inflateEnd(&zstream->z); ++ GIT_ZLIB_SYMBOL(inflateEnd)(&zstream->z); + else +- deflateEnd(&zstream->z); ++ GIT_ZLIB_SYMBOL(deflateEnd)(&zstream->z); + } + + void git_zstream_reset(git_zstream *zstream) + { + if (zstream->type == GIT_ZSTREAM_INFLATE) +- inflateReset(&zstream->z); ++ GIT_ZLIB_SYMBOL(inflateReset)(&zstream->z); + else +- deflateReset(&zstream->z); ++ GIT_ZLIB_SYMBOL(deflateReset)(&zstream->z); + zstream->in = NULL; + zstream->in_len = 0; + zstream->zerr = Z_STREAM_END; +@@ -120,9 +118,9 @@ int git_zstream_get_output_chunk( + + /* compress next chunk */ + if (zstream->type == GIT_ZSTREAM_INFLATE) +- zstream->zerr = inflate(&zstream->z, zstream->flush); ++ zstream->zerr = GIT_ZLIB_SYMBOL(inflate)(&zstream->z, zstream->flush); + else +- zstream->zerr = deflate(&zstream->z, zstream->flush); ++ zstream->zerr = GIT_ZLIB_SYMBOL(deflate)(&zstream->z, zstream->flush); + + if (zstream_seterr(zstream)) + return -1; +diff --git a/src/util/zstream.h b/src/util/zstream.h +index d78b11291..001036d4f 100644 +--- a/src/util/zstream.h ++++ b/src/util/zstream.h +@@ -9,7 +9,15 @@ + + #include "git2_util.h" + ++#ifdef GIT_COMPRESSION_ZLIB_NG ++#include ++# define GIT_ZLIB_SYMBOL(name) GIT_CONCAT(zng_, name) ++# define GIT_ZLIB_STREAM zng_stream ++#else + #include ++# define GIT_ZLIB_SYMBOL(name) name ++# define GIT_ZLIB_STREAM z_stream ++#endif + + #include "str.h" + +@@ -19,7 +27,7 @@ typedef enum { + } git_zstream_t; + + typedef struct { +- z_stream z; ++ GIT_ZLIB_STREAM z; + git_zstream_t type; + const char *in; + size_t in_len; +diff --git a/tests/libgit2/core/features.c b/tests/libgit2/core/features.c +index 780d29853..5891be3bc 100644 +--- a/tests/libgit2/core/features.c ++++ b/tests/libgit2/core/features.c +@@ -200,6 +200,8 @@ void test_core_features__backends(void) + cl_assert_equal_s("builtin", compression); + #elif defined(GIT_COMPRESSION_ZLIB) + cl_assert_equal_s("zlib", compression); ++#elif defined(GIT_COMPRESSION_ZLIB_NG) ++ cl_assert_equal_s("zlib-ng", compression); + #else + cl_assert(0); + #endif +diff --git a/tests/util/zstream.c b/tests/util/zstream.c +index ea7ffb1d0..9481676e7 100644 +--- a/tests/util/zstream.c ++++ b/tests/util/zstream.c +@@ -10,7 +10,7 @@ static void assert_zlib_equal_( + const void *compressed, size_t c_len, + const char *msg, const char *file, const char *func, int line) + { +- z_stream stream; ++ GIT_ZLIB_STREAM stream; + char *expanded = git__calloc(1, e_len + INFLATE_EXTRA); + cl_assert(expanded); + +@@ -20,9 +20,9 @@ static void assert_zlib_equal_( + stream.next_in = (Bytef *)compressed; + stream.avail_in = (uInt)c_len; + +- cl_assert(inflateInit(&stream) == Z_OK); +- cl_assert(inflate(&stream, Z_FINISH)); +- inflateEnd(&stream); ++ cl_assert(GIT_ZLIB_SYMBOL(inflateInit)(&stream) == Z_OK); ++ cl_assert(GIT_ZLIB_SYMBOL(inflate)(&stream, Z_FINISH)); ++ GIT_ZLIB_SYMBOL(inflateEnd)(&stream); + + clar__assert_equal( + file, func, line, msg, 1, From 52bff94dc2693c555d695c8e104c64cb0ea5ba62 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 4 Sep 2026 23:03:09 +0000 Subject: [PATCH 3/4] libfetchers: Don't try to find deltas when unpacking to the tarball cache This is by far the most expensive part of unpacking now: 49.35% nix libgit2.so.2.0.0 [.] git_delta_create_from_index 8.56% nix libgit2.so.2.0.0 [.] sha1_compression_states 5.89% nix libz.so.1.3.2 [.] longest_match 5.24% nix libz.so.1.3.2 [.] inflate_fast 4.80% nix libz.so.1.3.2 [.] deflate_slow 4.44% nix libgit2.so.2.0.0 [.] ubc_check 1.78% nix libz.so.1.3.2 [.] pqdownheap.constprop.0 1.53% nix libz.so.1.3.2 [.] compress_block 1.27% nix libgit2.so.2.0.0 [.] git_delta_index_init --- packaging/dependencies.nix | 1 + packaging/patches/0002-memory-config.patch | 22 +++++++++++ src/libfetchers/git-utils.cc | 39 ++++++++++++++++++- .../include/nix/fetchers/git-utils.hh | 6 +++ 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/0002-memory-config.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 2ec60d95157..2fc7c6cdcbf 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -112,6 +112,7 @@ scope: { }; patches = prevAttrs.patches or [ ] ++ [ ./patches/0001-zlib-ng-support.patch + ./patches/0002-memory-config.patch ]; cmakeFlags = prevAttrs.cmakeFlags ++ [ "-DUSE_COMPRESSION=zlib-ng" ]; buildInputs = prevAttrs.buildInputs ++ [ pkgs.zlib-ng ]; diff --git a/packaging/patches/0002-memory-config.patch b/packaging/patches/0002-memory-config.patch new file mode 100644 index 00000000000..ce1ff61acab --- /dev/null +++ b/packaging/patches/0002-memory-config.patch @@ -0,0 +1,22 @@ +diff --git a/include/git2/sys/config.h b/include/git2/sys/config.h +index dcce18e6f..8945a3285 100644 +--- a/include/git2/sys/config.h ++++ b/include/git2/sys/config.h +@@ -200,7 +200,7 @@ GIT_EXTERN(int) git_config_backend_memory_options_init( + * @param opts the options to initialize this backend with, or NULL + * @return 0 on success or an error code + */ +-extern int git_config_backend_from_string( ++GIT_EXTERN(int) git_config_backend_from_string( + git_config_backend **out, + const char *cfg, + size_t len, +@@ -216,7 +216,7 @@ extern int git_config_backend_from_string( + * @param opts the options to initialize this backend with, or NULL + * @return 0 on success or an error code + */ +-extern int git_config_backend_from_values( ++GIT_EXTERN(int) git_config_backend_from_values( + git_config_backend **out, + const char **values, + size_t len, diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 0576bbd6d90..9a37cb6aa8d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -142,6 +143,11 @@ typedef std::unique_ptr> typedef std::unique_ptr> StatusList; typedef std::unique_ptr> Remote; typedef std::unique_ptr> GitConfig; +typedef std::unique_ptrfree(backend); + })> + GitConfigBackend; typedef std::unique_ptr> ConfigIterator; typedef std::unique_ptr> ObjectDb; typedef std::unique_ptr> PackBuilder; @@ -363,6 +369,28 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_repository_open(Setter(repo), path.string().c_str())) throw GitError("opening Git repository %s", PathFmt(path)); + GitConfig config; + if (git_repository_config(Setter(config), *this)) + throw GitError("getting Git repository config"); + + /* Create an in-memory configuration so that we can set config options without modifying the + config file on-disk. */ + git_config_backend_memory_options configOpts = GIT_CONFIG_BACKEND_MEMORY_OPTIONS_INIT; + configOpts.backend_type = "nix"; + + std::vector configValues; + if (options.dontFindDeltas) + configValues.push_back("pack.deltacachesize=1"); + + GitConfigBackend memBackend; + if (git_config_backend_from_values(Setter(memBackend), configValues.data(), configValues.size(), &configOpts)) + throw GitError("creating an in-memory Git config"); + + if (git_config_add_backend(config.get(), memBackend.get(), GIT_CONFIG_LEVEL_APP, *this, /*force=*/false)) + throw GitError("adding the in-memory Git configuration backend"); + + memBackend.release(); + ObjectDb odb; if (options.packfilesOnly) { /* Create a fresh object database because by default the repo also @@ -1706,7 +1734,16 @@ ref Settings::getTarballCache() const * for optimal packfiles. */ static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2"; - return GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true}); + return GitRepo::openRepo( + repoDir, + { + .create = true, + .bare = true, + .packfilesOnly = true, + /* Tarball unpacking is not expected to benefit from deltas much, + compared to how much CPU times it takes to find. */ + .dontFindDeltas = true, + }); } } // namespace fetchers diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index f34f3bfde1b..26239498078 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -41,6 +41,12 @@ struct GitRepo bool create = false; bool bare = false; bool packfilesOnly = false; + /** + * Whether to avoid finding deltas when writing packfiles. It's an + * expensive operation, which should be avoided if no benefit is + * expected from possible deduplication in the same packfile. + */ + bool dontFindDeltas = false; }; static ref openRepo(const std::filesystem::path & path, Options options); From 36bdf2ab83431794a8740e4bb61d1d8c45b91843 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 5 Sep 2026 12:44:38 +0300 Subject: [PATCH 4/4] libfetchers: fix libgit2 pack window/bigfilethreshold config --- packaging/dependencies.nix | 1 + .../0003-packbuilder-correct-config.patch | 45 +++++++++++++++++++ src/libfetchers/git-utils.cc | 2 +- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/0003-packbuilder-correct-config.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 2fc7c6cdcbf..c8a8e76c250 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -113,6 +113,7 @@ scope: { patches = prevAttrs.patches or [ ] ++ [ ./patches/0001-zlib-ng-support.patch ./patches/0002-memory-config.patch + ./patches/0003-packbuilder-correct-config.patch ]; cmakeFlags = prevAttrs.cmakeFlags ++ [ "-DUSE_COMPRESSION=zlib-ng" ]; buildInputs = prevAttrs.buildInputs ++ [ pkgs.zlib-ng ]; diff --git a/packaging/patches/0003-packbuilder-correct-config.patch b/packaging/patches/0003-packbuilder-correct-config.patch new file mode 100644 index 00000000000..8d1ab47920f --- /dev/null +++ b/packaging/patches/0003-packbuilder-correct-config.patch @@ -0,0 +1,45 @@ +diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c +index 0da84b657..d540795f9 100644 +--- a/src/libgit2/pack-objects.c ++++ b/src/libgit2/pack-objects.c +@@ -115,8 +115,9 @@ static int packbuilder_config(git_packbuilder *pb) + GIT_PACK_DELTA_CACHE_SIZE); + config_get("pack.deltaCacheLimit", pb->cache_max_small_delta_size, + GIT_PACK_DELTA_CACHE_LIMIT); +- config_get("pack.deltaCacheSize", pb->big_file_threshold, ++ config_get("core.bigFileThreshold", pb->big_file_threshold, + GIT_PACK_BIG_FILE_THRESHOLD); ++ config_get("pack.window", pb->window_size, GIT_PACK_WINDOW); + config_get("pack.windowMemory", pb->window_memory_limit, 0); + + #undef config_get +@@ -1341,7 +1342,7 @@ int git_packbuilder__prepare(git_packbuilder *pb) + size_t i, n = 0; + int error; + +- if (pb->nr_objects == 0 || pb->done) ++ if (pb->nr_objects == 0 || pb->done || pb->window_size == 0) + return 0; /* nothing to do */ + + /* +@@ -1369,7 +1370,7 @@ int git_packbuilder__prepare(git_packbuilder *pb) + if (n > 1) { + git__tsort((void **)delta_list, n, type_size_sort); + if ((error = ll_find_deltas(pb, delta_list, n, +- GIT_PACK_WINDOW + 1, ++ pb->window_size + 1, + GIT_PACK_DEPTH)) < 0) { + git__free(delta_list); + return error; +diff --git a/src/libgit2/pack-objects.h b/src/libgit2/pack-objects.h +index ad04fb0ab..d9e03df60 100644 +--- a/src/libgit2/pack-objects.h ++++ b/src/libgit2/pack-objects.h +@@ -94,6 +94,7 @@ struct git_packbuilder { + size_t cache_max_small_delta_size; + size_t big_file_threshold; + size_t window_memory_limit; ++ size_t window_size; + + unsigned int nr_threads; /* nr of threads to use */ + diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 9a37cb6aa8d..52331613cc5 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -380,7 +380,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::vector configValues; if (options.dontFindDeltas) - configValues.push_back("pack.deltacachesize=1"); + configValues.push_back("pack.window=0"); GitConfigBackend memBackend; if (git_config_backend_from_values(Setter(memBackend), configValues.data(), configValues.size(), &configOpts))