Preserve non-UTF-8 paths in NAR streams

.gitignore

@@ -12,4 +12,5 @@ TAGS
 tags
 result
 result-*
+benchmark-results/
 *.actual

hnix-store-nar/CHANGELOG.md

@@ -1,6 +1,7 @@
 # Unreleased
 
 * Switch from `cryptonite` to `crypton`
+* Preserve non-UTF-8 filename and symlink target bytes when streaming NARs
 
 # [0.1.1.0](https://github.com/haskell-nix/hnix-store/compare/nar-0.1.0.0...nar-0.1.1.0)  2024-10-09
 

hnix-store-nar/README.md

@@ -4,5 +4,8 @@
 
 For a description of the NAR format, see [`Eelco's thesis`](https://nixos.org/~eelco/pubs/phd-thesis.pdf).
 
+The [NAR streaming benchmark](./benchmarks/README.md) provides a reproducible
+Hyperfine comparison between the working tree and a baseline Git revision.
+
 [System.Nix.Nar]: ./src/System/Nix/Nar.hs
 [System.Nix.Nar.Effects]: ./src/System/Nix/Nar/Effects.hs

hnix-store-nar/benchmarks/NarStream.hs

@@ -0,0 +1,66 @@
+{-# LANGUAGE ImportQualifiedPost #-}
+
+module Main (main) where
+
+import Control.Monad (forM_, replicateM_, unless)
+import Data.ByteString qualified as Bytes
+import Data.IORef (modifyIORef', newIORef, readIORef)
+import System.Directory qualified as Directory
+import System.Environment (getArgs)
+import System.Exit (die)
+import System.FilePath ((</>))
+import Text.Printf (printf)
+import Text.Read (readMaybe)
+
+import System.Nix.Nar.Streamer (dumpPath)
+
+main :: IO ()
+main = do
+  args <- getArgs
+  case args of
+    ["prepare", fixturePath, fileCountText] -> do
+      fileCount <- readPositive "file count" fileCountText
+      prepareFixture fixturePath fileCount
+    ["run", fixturePath, iterationsText] -> do
+      iterations <- readPositive "iteration count" iterationsText
+      runBenchmark fixturePath iterations
+    [] -> putStrLn usage
+    _ -> die usage
+
+usage :: String
+usage = unlines
+  [ "Usage: nar-stream prepare FIXTURE_PATH FILE_COUNT"
+  , "       nar-stream run FIXTURE_PATH ITERATIONS"
+  ]
+
+readPositive :: String -> String -> IO Int
+readPositive label input =
+  case readMaybe input of
+    Just value | value > 0 -> pure value
+    _ -> die $ label <> " must be a positive integer"
+
+prepareFixture :: FilePath -> Int -> IO ()
+prepareFixture fixturePath fileCount = do
+  exists <- Directory.doesPathExist fixturePath
+  if exists
+    then die $ "fixture path already exists: " <> fixturePath
+    else Directory.createDirectory fixturePath
+
+  forM_ [1 .. fileCount] $ \index ->
+    Bytes.writeFile (fixturePath </> fileName index) Bytes.empty
+ where
+  fileName :: Int -> FilePath
+  fileName = printf "file-%08d"
+
+runBenchmark :: FilePath -> Int -> IO ()
+runBenchmark fixturePath iterations = do
+  exists <- Directory.doesDirectoryExist fixturePath
+  unless exists $ die $ "fixture directory does not exist: " <> fixturePath
+
+  byteCount <- newIORef (0 :: Int)
+  replicateM_ iterations $
+    dumpPath fixturePath $ \chunk ->
+      modifyIORef' byteCount (+ Bytes.length chunk)
+
+  streamedBytes <- readIORef byteCount
+  unless (streamedBytes > 0) $ die "NAR streamer produced no output"

hnix-store-nar/benchmarks/README.md

@@ -0,0 +1,57 @@
+# NAR streaming wall-time benchmark
+
+This benchmark compares the working tree's NAR streamer with a baseline Git
+revision. Both binaries archive the same pre-created wide directory, so fixture
+creation, compilation, and cleanup are excluded from Hyperfine's measurements.
+
+## 1. Prepare
+
+From the repository root:
+
+```console
+nix-shell hnix-store-nar/benchmarks/shell.nix --run \
+  'hnix-store-nar/benchmarks/prepare-hyperfine.sh'
+```
+
+The default baseline is `HEAD`. This is appropriate while the fix is still an
+uncommitted working-tree change. After committing the fix, select its parent or
+another known revision explicitly:
+
+```console
+HNIX_NAR_BASELINE_REF=HEAD^ \
+  nix-shell hnix-store-nar/benchmarks/shell.nix --run \
+    'hnix-store-nar/benchmarks/prepare-hyperfine.sh'
+```
+
+Preparation creates ignored state under `benchmark-results/nar-stream-state`:
+the two prebuilt runners, a shared 5,000-file fixture, build metadata,
+and the working-tree patch. It does not run Hyperfine.
+
+## 2. Measure when the machine is idle
+
+```console
+nix-shell hnix-store-nar/benchmarks/shell.nix --run \
+  'hnix-store-nar/benchmarks/run-hyperfine.sh'
+```
+
+The runner refuses to start when one-minute load divided by the CPU count is
+above `0.5`. It pins both binaries to the same CPU, performs five warmups and 30
+measurements, then repeats the comparison in reverse order to expose drift.
+JSON and Markdown reports are written below the prepared state's `results/`
+directory.
+
+Useful overrides:
+
+```console
+HNIX_NAR_FILES=10000             # preparation only
+HNIX_NAR_ITERATIONS=10           # dumps per measured process
+HNIX_NAR_WARMUPS=5
+HNIX_NAR_RUNS=30
+HNIX_NAR_CPU=14
+HNIX_NAR_MAX_LOAD_PER_CPU=0.5
+HNIX_NAR_STATE_DIR=/some/path
+```
+
+This is a warm-cache benchmark of NAR traversal and encoding. Do not clear the
+machine's page cache between runs; doing so needs elevated privileges, disrupts
+other work, and measures a different workload.

hnix-store-nar/benchmarks/prepare-hyperfine.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+benchmark_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+repo_root=$(git -C "$benchmark_dir" rev-parse --show-toplevel)
+state_dir=${HNIX_NAR_STATE_DIR:-"$repo_root/benchmark-results/nar-stream-state"}
+baseline_ref=${HNIX_NAR_BASELINE_REF:-HEAD}
+file_count=${HNIX_NAR_FILES:-5000}
+
+if [[ -e "$state_dir" ]]; then
+  echo "Benchmark state already exists: $state_dir" >&2
+  echo "Set HNIX_NAR_STATE_DIR to a new path or remove the old state first." >&2
+  exit 1
+fi
+
+scratch_dir=$(mktemp -d "${TMPDIR:-/tmp}/hnix-nar-prepare.XXXXXX")
+baseline_tree="$scratch_dir/baseline"
+baseline_added=false
+
+cleanup() {
+  if [[ "$baseline_added" == true ]]; then
+    git -C "$repo_root" worktree remove --force "$baseline_tree" >/dev/null 2>&1 || true
+  fi
+  rm -rf -- "$scratch_dir"
+}
+trap cleanup EXIT
+
+mkdir -p "$state_dir/bin"
+
+build_runner() {
+  local source_tree=$1
+  local label=$2
+  local build_dir="$scratch_dir/dist-$label"
+  local object_dir="$scratch_dir/objects-$label"
+
+  mkdir -p "$object_dir"
+  (
+    cd "$source_tree"
+    cabal build hnix-store-nar \
+      --builddir="$build_dir" \
+      --disable-tests \
+      --disable-benchmarks \
+      -j1
+    cabal exec --builddir="$build_dir" -- \
+      ghc -O2 -threaded -rtsopts "-with-rtsopts=-N1" \
+        -odir "$object_dir" \
+        -hidir "$object_dir" \
+        -package bytestring \
+        -package directory \
+        -package filepath \
+        -package hnix-store-nar \
+        "$benchmark_dir/NarStream.hs" \
+        -o "$state_dir/bin/$label"
+  )
+}
+
+echo "Building fixed benchmark runner..."
+build_runner "$repo_root" fixed
+
+echo "Creating baseline worktree at $baseline_ref..."
+git -C "$repo_root" worktree add --detach "$baseline_tree" "$baseline_ref"
+baseline_added=true
+
+echo "Building baseline benchmark runner..."
+build_runner "$baseline_tree" baseline
+
+echo "Creating the shared $file_count-file fixture..."
+"$state_dir/bin/fixed" prepare "$state_dir/fixture" "$file_count"
+
+baseline_commit=$(git -C "$baseline_tree" rev-parse HEAD)
+fixed_commit=$(git -C "$repo_root" rev-parse HEAD)
+{
+  printf 'baseline_ref=%s\n' "$baseline_ref"
+  printf 'baseline_commit=%s\n' "$baseline_commit"
+  printf 'fixed_commit=%s\n' "$fixed_commit"
+  printf 'file_count=%s\n' "$file_count"
+  printf 'ghc=%s\n' "$(ghc --numeric-version)"
+  printf 'cabal=%s\n' "$(cabal --numeric-version)"
+} > "$state_dir/metadata.txt"
+git -C "$repo_root" diff --binary > "$state_dir/fixed.patch"
+
+echo
+echo "Benchmark state is ready: $state_dir"
+echo "When the machine is idle, run:"
+echo "  nix-shell $benchmark_dir/shell.nix --run '$benchmark_dir/run-hyperfine.sh'"

hnix-store-nar/benchmarks/run-hyperfine.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+benchmark_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+repo_root=$(git -C "$benchmark_dir" rev-parse --show-toplevel)
+state_dir=${HNIX_NAR_STATE_DIR:-"$repo_root/benchmark-results/nar-stream-state"}
+iterations=${HNIX_NAR_ITERATIONS:-7}
+warmups=${HNIX_NAR_WARMUPS:-5}
+runs=${HNIX_NAR_RUNS:-30}
+max_load_per_cpu=${HNIX_NAR_MAX_LOAD_PER_CPU:-0.5}
+cpu_count=$(nproc)
+default_cpu=$((cpu_count > 1 ? cpu_count - 2 : 0))
+benchmark_cpu=${HNIX_NAR_CPU:-$default_cpu}
+
+baseline_bin="$state_dir/bin/baseline"
+fixed_bin="$state_dir/bin/fixed"
+fixture="$state_dir/fixture"
+
+for required_path in "$baseline_bin" "$fixed_bin" "$fixture" "$state_dir/metadata.txt"; do
+  if [[ ! -e "$required_path" ]]; then
+    echo "Missing benchmark state: $required_path" >&2
+    echo "Run prepare-hyperfine.sh first." >&2
+    exit 1
+  fi
+done
+
+check_load() {
+  local load_one
+  load_one=$(awk '{ print $1 }' /proc/loadavg)
+  if ! awk \
+    -v current_load="$load_one" \
+    -v cpus="$cpu_count" \
+    -v maximum="$max_load_per_cpu" \
+    'BEGIN { exit !((current_load / cpus) <= maximum) }'
+  then
+    echo "Refusing to benchmark: load is $load_one across $cpu_count CPUs." >&2
+    echo "Wait for load/CPU <= $max_load_per_cpu, or override HNIX_NAR_MAX_LOAD_PER_CPU." >&2
+    exit 75
+  fi
+}
+
+if ! taskset -c "$benchmark_cpu" true; then
+  echo "CPU $benchmark_cpu is not available to taskset." >&2
+  exit 1
+fi
+
+check_load
+
+timestamp=$(date -u +%Y%m%dT%H%M%SZ)
+results_dir="$state_dir/results/$timestamp"
+mkdir -p "$results_dir"
+
+baseline_command="taskset -c $benchmark_cpu $baseline_bin run $fixture $iterations +RTS -N1 -A64m -RTS"
+fixed_command="taskset -c $benchmark_cpu $fixed_bin run $fixture $iterations +RTS -N1 -A64m -RTS"
+
+common_options=(
+  --warmup "$warmups"
+  --runs "$runs"
+  --shell=none
+)
+
+echo "Benchmarking baseline first on CPU $benchmark_cpu..."
+hyperfine \
+  "${common_options[@]}" \
+  --export-json "$results_dir/baseline-first.json" \
+  --export-markdown "$results_dir/baseline-first.md" \
+  --command-name baseline "$baseline_command" \
+  --command-name fixed "$fixed_command"
+
+check_load
+
+echo "Benchmarking fixed first on CPU $benchmark_cpu..."
+hyperfine \
+  "${common_options[@]}" \
+  --export-json "$results_dir/fixed-first.json" \
+  --export-markdown "$results_dir/fixed-first.md" \
+  --command-name fixed "$fixed_command" \
+  --command-name baseline "$baseline_command"
+
+cp "$state_dir/metadata.txt" "$results_dir/metadata.txt"
+
+echo
+echo "Benchmark results: $results_dir"

hnix-store-nar/benchmarks/shell.nix

@@ -0,0 +1,13 @@
+{ pkgs ? (import ../../default.nix {}).pkgs }:
+
+let
+  narShell = (import ../../shell.nix { inherit pkgs; }).hnix-store-nar;
+in
+pkgs.mkShell {
+  inputsFrom = [ narShell ];
+  packages = [
+    pkgs.cabal-install
+    pkgs.hyperfine
+    pkgs.util-linux
+  ];
+}

hnix-store-nar/hnix-store-nar.cabal

@@ -16,6 +16,10 @@ extra-doc-files:
     CHANGELOG.md
 extra-source-files:
     README.md
+  , benchmarks/README.md
+  , benchmarks/prepare-hyperfine.sh
+  , benchmarks/run-hyperfine.sh
+  , benchmarks/shell.nix
   , tests/fixtures/case-conflict.nar
 
 flag bounded_memory
@@ -116,3 +120,16 @@ test-suite nar
     , tasty-quickcheck
     , text
     , unix
+
+benchmark nar-stream
+  import: commons
+  type: exitcode-stdio-1.0
+  main-is: NarStream.hs
+  hs-source-dirs: benchmarks
+  ghc-options: -O2 -threaded -rtsopts "-with-rtsopts -N1"
+  build-depends:
+      base
+    , bytestring
+    , directory
+    , filepath
+    , hnix-store-nar