perf!: replace merged data model YAML with JSON for faster loading

CHANGELOG.md

@@ -2,8 +2,9 @@
 
 ## Features
 
-- robot rendering: added support for dicts as parent_key in `iterate_list_chunked` 
+- robot rendering: added support for dicts as parent_key in `iterate_list_chunked`
 - add support for SDWAN token authentication for pyATS test cases via SDWAN_USERNAME & SDWAN_API_TOKEN
+- add `NAC_TEST_DUMP_YAML_DATA_MODEL` environment variable to also write the merged data model as YAML (alongside the JSON file) for post-run inspection/debugging. The YAML file is not auto-cleaned up and may contain sensitive values (passwords, tokens, credentials), so review and remove it manually.
 
 ## Bug Fixes
 
@@ -13,6 +14,10 @@
 
 - SSHTestBase.parse_output() is now async — test cases must use await self.parse_output(...)
 
+## Breaking Changes
+
+- **Internal merged data model file format changed to JSON**: For performance reasons, the internal temporary file used to pass the merged data model to test subprocesses is now written as JSON (`merged_data_model_test_variables.json`, previously `.yaml`). This has no effect on your YAML data files or data model structure. The standard `self.data_model` API is unaffected. This is only breaking if your tests or scripts read the file directly via the `MERGED_DATA_MODEL_TEST_VARIABLES_FILEPATH` environment variable — in that case, switch from YAML parsing to `json.load()`. Note two JSON serialization differences versus the previous YAML format: non-string mapping keys (e.g. integer keys such as VLAN IDs used as keys) are coerced to strings, and values that JSON cannot represent natively (e.g. unquoted YAML dates parsed as `datetime.date`) are written as their string form. Quote such keys/values in your data files if you need them preserved as strings anyway.
+
 # 2.0.0
 
 ## Major Features

README.md

@@ -381,7 +381,7 @@ Before test execution, `nac-test` merges all YAML data files into a single data
 
 1. All files from `--data` paths are recursively loaded
 2. YAML structures are deep-merged (later files override earlier ones)
-3. The merged result is written to the output directory as `merged_data_model_test_variables.yaml`
+3. The merged result is written to the output directory as `merged_data_model_test_variables.json` and will be read by pyATS test cases
 4. Both Robot and PyATS tests reference this merged data
 
 ### Accessing the Merged Data
@@ -818,6 +818,7 @@ In addition to CLI options, `nac-test` supports several environment variables fo
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `NAC_TEST_VERBOSE` | unset | Enable verbose mode: verbose output and retain intermediate files (see `NAC_TEST_PYATS_KEEP_REPORT_DATA`) |
+| `NAC_TEST_DUMP_YAML_DATA_MODEL` | unset | Write merged data model as YAML for debugging. File is not auto-cleaned up; you must remove it manually. WARNING: May contain sensitive values. |
 
 ## Troubleshooting
 

nac_test/cli/main.py

@@ -22,6 +22,7 @@
 from nac_test.combined_orchestrator import CombinedOrchestrator
 from nac_test.core.constants import (
     DEBUG_MODE,
+    DUMP_YAML_DATA_MODEL,
     EXIT_DATA_ERROR,
     EXIT_ERROR,
     EXIT_INTERRUPTED,
@@ -390,7 +391,9 @@ def main(
     typer.echo("\n\n📄 Merging data model files...")
 
     merged_data = DataMerger.merge_data_files(data)
-    merged_data_path = DataMerger.write_merged_data_model(merged_data, output)
+    merged_data_path = DataMerger.write_merged_data_model(
+        merged_data, output, dump_yaml=DUMP_YAML_DATA_MODEL
+    )
 
     # Register merged data file for cleanup — always delete, even in debug mode,
     # because it may contain credentials resolved from !env references.

nac_test/core/constants.py

@@ -40,6 +40,9 @@
 # Set NAC_TEST_DISABLE_TESTLEVELSPLIT=true to disable test-level parallelization
 DISABLE_TESTLEVELSPLIT: bool = get_bool_env("NAC_TEST_DISABLE_TESTLEVELSPLIT")
 
+# Merged data model YAML dump for debugging
+# Set NAC_TEST_DUMP_YAML_DATA_MODEL=true to write YAML alongside JSON
+DUMP_YAML_DATA_MODEL: bool = get_bool_env("NAC_TEST_DUMP_YAML_DATA_MODEL")
 
 # Report timestamp format - single source of truth for all report generators
 REPORT_TIMESTAMP_FORMAT: str = "%Y-%m-%d %H:%M:%S"
@@ -86,7 +89,7 @@
 REPORT_HTML: str = "report.html"
 XUNIT_XML: str = "xunit.xml"
 ORDERING_FILENAME: str = "ordering.txt"
-MERGED_DATA_FILENAME: str = "merged_data_model_test_variables.yaml"
+MERGED_DATA_FILENAME: str = "merged_data_model_test_variables.json"
 # Owner read/write only — prevents other users from reading sensitive merged data
 MERGED_DATA_FILE_MODE: int = 0o600
 SUMMARY_SEPARATOR_WIDTH: int = 70

nac_test/data_merger.py

@@ -3,6 +3,7 @@
 
 """Shared data merging utilities for both Robot and PyATS test execution."""
 
+import json
 import logging
 import os
 from pathlib import Path
@@ -50,30 +51,57 @@ def merged_data_path(output_directory: Path) -> Path:
             output_directory: Base output directory passed to write_merged_data_model()
 
         Returns:
-            Full path to the merged data model YAML file
+            Full path to the merged data model JSON file
         """
         return output_directory / MERGED_DATA_FILENAME
 
     @staticmethod
     def write_merged_data_model(
         data: dict[str, Any],
         output_directory: Path,
+        dump_yaml: bool = False,
     ) -> Path:
-        """Write merged data model to YAML file.
+        """Write merged data model to JSON file and optionally YAML.
 
         The output filename is always MERGED_DATA_FILENAME — the single fixed
         location used by all consumers (Robot, PyATS subprocesses, cleanup).
 
+        When dump_yaml is True, a companion YAML file is also written with the
+        same data. The YAML file is not auto-cleaned up and may contain sensitive
+        values.
+
         Args:
             data: The merged data dictionary to write
-            output_directory: Directory where the YAML file will be saved
+            output_directory: Directory where the JSON file will be saved
+            dump_yaml: If True, also write a YAML file for debugging purposes.
+                      Defaults to False.
 
         Returns:
-            Path to the written file (use this instead of reconstructing the path)
+            Path to the written JSON file (use this instead of reconstructing the path)
         """
         full_output_path = DataMerger.merged_data_path(output_directory)
         logger.info("Writing merged data model to %s", full_output_path)
-        yaml.write_yaml_file(data, full_output_path)
+        with open(full_output_path, "w", encoding="utf-8") as f:
+            # default=str prevents a hard crash on values ruamel's safe loader
+            # produces that json cannot natively serialize (e.g. datetime.date
+            # from an unquoted YAML date); such values are stringified.
+            json.dump(data, f, default=str)
         if not IS_WINDOWS:
             os.chmod(full_output_path, MERGED_DATA_FILE_MODE)
+
+        # Optionally write YAML for debugging
+        if dump_yaml:
+            yaml_output_path = full_output_path.with_suffix(".yaml")
+            try:
+                yaml.write_yaml_file(data, yaml_output_path)
+                if not IS_WINDOWS:
+                    os.chmod(yaml_output_path, MERGED_DATA_FILE_MODE)
+                logger.warning(
+                    "Created %s for post-run debugging. It may contain sensitive values "
+                    "(passwords, tokens, credentials). Please review and remove the file manually.",
+                    yaml_output_path,
+                )
+            except Exception as e:
+                logger.warning("Failed to write YAML data model: %s", e)
+
         return full_output_path

nac_test/pyats_core/common/base_test.py

@@ -51,7 +51,6 @@
 from nac_test.pyats_core.reporting.types import ResultStatus
 from nac_test.utils import sanitize_hostname
 from nac_test.utils.formatting import format_file_timestamp_ms
-from nac_test.utils.yaml import safe_load
 
 T = TypeVar("T")
 
@@ -827,7 +826,7 @@ def load_data_model(self) -> dict[str, Any]:
             )
 
         with open(data_file, encoding="utf-8") as f:
-            data = safe_load(f)
+            data = json.load(f)
             return data if isinstance(data, dict) else {}
 
     def get_default_value(

nac_test/pyats_core/discovery/device_inventory.py

@@ -8,13 +8,12 @@
 """
 
 import importlib.util
+import json
 import logging
 import sys
 from pathlib import Path
 from typing import Any
 
-from nac_test.utils.yaml import safe_load
-
 logger = logging.getLogger(__name__)
 
 
@@ -80,7 +79,7 @@ def get_device_inventory(self, test_files: list[Path]) -> list[dict[str, Any]]:
             return []
 
         with open(self.merged_data_filepath, encoding="utf-8") as f:
-            data_model = safe_load(f)
+            data_model = json.load(f)
 
         # Import the first D2D test file - all D2D tests in an architecture share the same SSH base class
         # For SD-WAN: all tests under /d2d/ inherit from SDWANTestBase

nac_test/pyats_core/execution/device/device_executor.py

@@ -41,7 +41,7 @@ def __init__(
             test_status: Dictionary for tracking test status
             test_dir: Directory containing PyATS test files (user-specified)
             base_output_dir: Base output directory for test results
-            merged_data_path: Path to the merged data model YAML file
+            merged_data_path: Path to the merged data model file
             custom_testbed_path: Optional path to custom PyATS testbed YAML
         """
         self.job_generator = job_generator

tests/conftest.py

@@ -14,6 +14,7 @@
 import tempfile
 from collections.abc import Generator
 from pathlib import Path
+from typing import NamedTuple
 
 import pytest
 
@@ -29,6 +30,14 @@
 )
 
 
+class PyATSTestDirs(NamedTuple):
+    """Directory structure for PyATS orchestrator tests."""
+
+    test_dir: Path
+    output_dir: Path
+    merged_file: Path
+
+
 def assert_is_link_to(link: Path, source: Path) -> None:
     """Assert that link points to source as either a hard link or symlink."""
     if link.is_symlink():
@@ -193,3 +202,21 @@ def cc_context() -> ControllerContext:
 def iosxe_context() -> ControllerContext:
     """Pre-built ControllerContext for IOS-XE with session auth."""
     return ControllerContext(controller_type="IOSXE", auth_method=AuthMethod.SESSION)
+
+
+@pytest.fixture()
+def pyats_test_dirs(tmp_path: Path) -> PyATSTestDirs:
+    """Create standard directory structure for PyATS orchestrator tests.
+
+    Returns:
+        PyATSTestDirs with test_dir, output_dir, and merged_file paths.
+    """
+    test_dir = tmp_path / "tests"
+    test_dir.mkdir()
+    output_dir = tmp_path / "output"
+    output_dir.mkdir()
+    merged_file = output_dir / "merged.json"
+    merged_file.write_text('{"test": "data"}')
+    return PyATSTestDirs(
+        test_dir=test_dir, output_dir=output_dir, merged_file=merged_file
+    )

tests/e2e/test_e2e_scenarios.py

@@ -179,7 +179,7 @@ def test_output_root_contains_only_expected_entries(
         )
 
     def test_merged_data_file_removed_after_run(self, results: E2EResults) -> None:
-        """Merged data model YAML must not persist after a successful run.
+        """Merged data model file must not persist after a successful run.
 
         The file contains potentially sensitive variable data and is registered
         with CleanupManager for deletion on exit. Its absence confirms cleanup ran.

tests/integration/fixtures/data_merge/result.json

@@ -0,0 +1 @@
+{"defaults": {"apic": {"version": 6.0}}, "root": {"attr1": "value1", "primitive_list": ["item1", "item1", "item1"], "dict_list": [{"name": "abc", "extra": "def"}], "dict_list_extra": [{"name": "abc", "extra1": "def", "extra2": "ghi"}], "attr2": "value2"}}

tests/integration/test_cli_rendering.py

@@ -314,7 +314,7 @@ def test_merged_data_model_creates_default_filename(tmp_path: Path) -> None:
     templates_path = "tests/integration/fixtures/templates/"
     output_model_path = tmp_path / MERGED_DATA_FILENAME
     data_dir = Path("tests/integration/fixtures/data_merge")
-    expected_model_path = data_dir / "result.yaml"
+    expected_model_path = data_dir / "result.json"
 
     result = runner.invoke(
         nac_test.cli.main.app,
@@ -341,6 +341,11 @@ def test_merged_data_model_creates_default_filename(tmp_path: Path) -> None:
         f"Merged data model content should match expected content from "
         f"{expected_model_path}"
     )
+    # By default (NAC_TEST_DUMP_YAML_DATA_MODEL unset) no YAML companion is written.
+    # The env-var-enabled case is covered in test_yaml_data_model_dump.py.
+    assert not output_model_path.with_suffix(".yaml").exists(), (
+        "YAML data model should not be created unless NAC_TEST_DUMP_YAML_DATA_MODEL is set"
+    )
 
 
 def test_render_only_without_controller_credentials(tmp_path: Path) -> None:

tests/integration/test_controller_detection_integration.py

@@ -34,8 +34,8 @@ def test_end_to_end_controller_detection(
         output_dir.mkdir()
 
         # Create a dummy merged data file
-        merged_file = output_dir / "merged_data.yaml"
-        merged_file.write_text("test: data")
+        merged_file = output_dir / "merged_data.json"
+        merged_file.write_text('{"test": "data"}')
 
         # Create a dummy test file
         test_file = test_dir / "test_dummy.py"

tests/integration/test_yaml_data_model_dump.py

@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (c) 2025 Daniel Schmidt
+
+"""Integration test for the optional YAML merged-data-model dump.
+
+When ``NAC_TEST_DUMP_YAML_DATA_MODEL`` is set, nac-test writes a companion
+YAML file next to the JSON merged data model. Unlike the JSON file, the YAML
+file is intentionally *not* registered with the CleanupManager, so it survives
+after the run for post-run debugging.
+
+This test runs the real ``nac-test`` CLI in a subprocess. A subprocess is
+required because ``DUMP_YAML_DATA_MODEL`` is evaluated from the environment at
+import time — an in-process runner (e.g. CliRunner) cannot toggle it after the
+module is already imported. The default (env var unset) case is covered by
+``test_cli_rendering.py::test_merged_data_model_creates_default_filename``.
+"""
+
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from nac_test.core.constants import IS_WINDOWS, MERGED_DATA_FILENAME
+from nac_test.utils.yaml import safe_load
+
+pytestmark = [pytest.mark.integration, pytest.mark.windows]
+
+
+def test_yaml_dump_created_when_env_set(tmp_path: Path) -> None:
+    """With the env var set, the YAML dump is written and persists after exit.
+
+    We assert on the YAML file (which is *not* cleanup-registered and therefore
+    survives) rather than the JSON file. Note that we do not assert here that the
+    JSON was removed: JSON cleanup-on-exit is already covered end-to-end by
+    ``tests/e2e/test_e2e_scenarios.py::test_merged_data_file_removed_after_run``.
+    """
+    templates_path = "tests/integration/fixtures/templates/"
+    data_dir = Path("tests/integration/fixtures/data_merge")
+    yaml_output_path = (tmp_path / MERGED_DATA_FILENAME).with_suffix(".yaml")
+
+    env = {**os.environ, "NAC_TEST_DUMP_YAML_DATA_MODEL": "true"}
+    result = subprocess.run(
+        [
+            "nac-test",
+            "-d",
+            str(data_dir / "file1.yaml"),
+            "-d",
+            str(data_dir / "file2.yaml"),
+            "-t",
+            templates_path,
+            "-o",
+            str(tmp_path),
+            "--render-only",
+        ],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        env=env,
+    )
+    assert result.returncode == 0, (
+        f"nac-test should succeed, got exit code {result.returncode}\n"
+        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+    )
+
+    assert yaml_output_path.exists(), (
+        f"YAML dump should persist at {yaml_output_path} when "
+        "NAC_TEST_DUMP_YAML_DATA_MODEL=true (it is not cleanup-registered)"
+    )
+
+    # Sanity check: the YAML deserializes to the expected merged content.
+    yaml_data = safe_load(yaml_output_path.read_text(encoding="utf-8"))
+    assert isinstance(yaml_data, dict) and yaml_data, "YAML dump should be non-empty"
+
+    # YAML dump must carry the same restrictive permissions as the JSON file.
+    if not IS_WINDOWS:
+        yaml_mode = stat.S_IMODE(yaml_output_path.stat().st_mode)
+        assert yaml_mode == 0o600, (
+            f"YAML dump should have 0o600 permissions, got {oct(yaml_mode)}"
+        )

tests/pyats_core/common/test_base_test_controller_detection.py

@@ -21,8 +21,8 @@ def setup_test_data_file_env(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> Generator[Path, None, None]:
     """Create temp data file and set MERGED_DATA_MODEL_TEST_VARIABLES_FILEPATH."""
-    temp_file = tmp_path / "test.yaml"
-    temp_file.write_text("test: data")
+    temp_file = tmp_path / "test.json"
+    temp_file.write_text('{"test": "data"}')
     monkeypatch.setenv("MERGED_DATA_MODEL_TEST_VARIABLES_FILEPATH", str(temp_file))
     yield temp_file