Testing improvements, break up repl/plugins tests, add TAP output to repl characterisation tests

packaging/hydra.nix

@@ -440,12 +440,8 @@ rec {
     in
     pkgs.runCommand "install-tests" {
       againstSelf = testNixVersions pkgs pkgs.nix;
-      againstCurrentLatest =
-        # FIXME: temporarily disable this on macOS because of #3605.
-        if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null;
-      # Disabled because the latest stable version doesn't handle
-      # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-      # againstLatestStable = testNixVersions pkgs pkgs.nixStable;
+      againstCurrentLatest = testNixVersions pkgs pkgs.nixVersions.latest;
+      againstLatestStable = testNixVersions pkgs pkgs.nixVersions.stable;
     } "touch $out"
   );
 

tests/functional/common/characterisation/framework.sh

@@ -2,6 +2,19 @@
 
 badTestNames=()
 
+function reportDiffAcceptFailure() {
+    local -r testName=$1
+    # We receive the diff unified stdout/stderr output as the argument
+    printf '%s' "$2" >&2
+    diffExitCode=$?
+    echo >&2 "subtest '$testName' failed: characterised output of $testName not as expected"
+}
+
+function reportDiffAcceptSuccess() {
+    # Do nothing by default
+    :
+}
+
 # Golden test support
 #
 # Test that the output of the given test matches what is expected. If
@@ -16,15 +29,22 @@ function diffAndAcceptInner() {
     if test -e "$expected"; then
         local -r expectedOrEmpty="$expected"
     else
-        local -r expectedOrEmpty=common/characterisation/empty
+        local -r expectedOrEmpty=$_NIX_TEST_SOURCE_DIR/common/characterisation/empty
     fi
 
     # Diff so we get a nice message
-    if ! diff >&2 --color=always --unified "$expectedOrEmpty" "$got"; then
-        echo >&2 "FAIL: evaluation result of $testName not as expected"
+    set +e
+    diffOutput=$(diff 2>&1 --color=always --unified "$expectedOrEmpty" "$got")
+    diffExitCode=$?
+    set -e
+
+    if ((diffExitCode != 0)); then
+        reportDiffAcceptFailure "$diffOutput" "$testName"
         # shellcheck disable=SC2034
         badDiff=1
         badTestNames+=("$testName")
+    else
+        reportDiffAcceptSuccess
     fi
 
     # Update expected if `_NIX_TEST_ACCEPT` is `1`. (Comparing against

tests/functional/common/functions.sh

@@ -374,7 +374,7 @@ initGitRepo() {
     local extraArgs="${2-}"
 
     # shellcheck disable=SC2086 # word splitting of extraArgs is intended
-    git -C "$repo" init $extraArgs
+    git -C "$repo" init --initial-branch=master $extraArgs
     git -C "$repo" config user.email "foobar@example.com"
     git -C "$repo" config user.name "Foobar"
 }

tests/functional/git/packed-refs-no-cache.sh

@@ -21,11 +21,7 @@ repo=$TEST_ROOT/./git
 
 export _NIX_FORCE_HTTP=1
 
-rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix"
-
-git init --initial-branch="master" "$repo"
-git -C "$repo" config user.email "nix-tests@example.com"
-git -C "$repo" config user.name "Nix Tests"
+createGitRepo "$repo"
 
 echo "hello world" > "$repo/hello_world"
 git -C "$repo" add hello_world

tests/functional/meson.build

@@ -7,7 +7,7 @@ project(
     'warning_level=1',
     'errorlogs=true', # Please print logs for tests that fail
   ],
-  meson_version : '>= 1.3',
+  meson_version : '>= 1.8',
   license : 'LGPL-2.1-or-later',
 )
 
@@ -206,27 +206,8 @@ if nix_store.found()
 
 endif
 
-# Plugin tests require shared libraries support.
-nix_expr = dependency('nix-expr', required : false)
-if nix_expr.found() and get_option('default_library') != 'static'
-  add_languages('cpp')
-  subdir('plugins')
-  plugin_test_deps = [ libplugintest ]
-  if get_option('plugin-c-api') and not host_machine.system().startswith(
-    'windows',
-  )
-    plugin_test_deps += [ libplugintest_dlsym ]
-  endif
-  suites += {
-    'name' : 'plugins',
-    'deps' : plugin_test_deps,
-    'tests' : [
-      'plugins.sh',
-    ],
-    'workdir' : meson.current_source_dir(),
-  }
-endif
-
+subdir('suites')
+# TODO: Lay out the tests suites in a less flat manner.
 subdir('ca')
 subdir('dyn-drv')
 subdir('flakes')
@@ -260,6 +241,9 @@ foreach suite : suites
       meson.current_build_dir() / 'asan-log',
     )
 
+    overrides = suite.get('overrides', {}).get(script, {})
+    protocol = overrides.get('protocol', 'exitcode')
+
     test(
       name,
       bash,
@@ -280,14 +264,15 @@ foreach suite : suites
         'TEST_NAME' : name,
         'NIX_REMOTE' : '',
         'PS4' : '+(${BASH_SOURCE[0]-$0}:$LINENO) ',
-      },
+      } + suite.get('env', {}),
       # Some tests take 15+ seconds even on an otherwise idle machine;
       # on a loaded machine this can easily drive them to failure. Give
       # them more time than the default of 30 seconds.
       timeout : 300,
       # Used for target dependency/ordering tracking, not adding compiler flags or anything.
       depends : deps,
       workdir : workdir,
+      protocol : protocol,
     )
   endforeach
 endforeach