Fix: Wait for ptp4l/phc2sys death before PF rebind in validation conftest

.github/instructions/mtl-validation-tests.instructions.md

@@ -104,6 +104,8 @@ sudo grep -E "EAL|hugepage|VF|RxTxApp|RemoteProcess|Traceback|err:" \
 | RxTxApp `Segmentation fault` inside `iavf_tm_node_add` (after `dev_if_init_pacing(0), try rl as drv support TM`) | **(setup)** Stock kernel ice loaded instead of the MTL out-of-tree patched ice (`versions.env::ICE_VER`). Re-run `setup_validation.sh` — the ice stage version-checks and reloads automatically. |
 | RxTxApp `Segmentation fault` anywhere else | **NOT setup.** Capture `gdb -batch -ex 'bt full' tests/tools/RxTxApp/build/RxTxApp /tmp/core.*` (or `coredumpctl gdb RxTxApp`) and report upstream as a real MTL/DPDK bug. Do **not** add a workaround. |
 | `Permission denied (publickey)` to `root@127.0.0.1` | **(setup)** Pubkey not in `/root/.ssh/authorized_keys`. |
+| `preflight: MtlManager or RxTxApp missing` despite binaries present under `.local_install/mtl/bin/` | **(setup)** Fixed: preflight now checks both the legacy in-tree `build/` path and the `.local_install` prefix that `mtl_engine/const.py::PREFIX` actually invokes. Re-run setup. |
+| `preflight: hugepages free is 64 MiB (<1024 MiB)` on a host with 1GB hugepages configured (`default_hugepagesz=1G` on kernel cmdline) | **(setup)** Fixed: hugepage check now reads `Hugepagesize` from `/proc/meminfo` instead of assuming 2MB pages. Re-run setup. |
 
 **(setup)** = delegate to the `MTL Validation Setup` subagent (it re-runs `setup_validation.sh` idempotently).
 

.github/scripts/setup_validation.sh

@@ -172,16 +172,16 @@ print_summary() {
 			"$CYN" "$CLR" "$s" "${STAGE_RESULT[$s]:-?}" "${STAGE_DURATION[$s]:-?}" >&2
 	done
 	log ""
-	log " RxTxApp        : $([[ -x tests/tools/RxTxApp/build/RxTxApp ]] && echo OK || echo MISSING)"
-	log " MtlManager     : $([[ -x build/manager/MtlManager ]] && echo OK || echo MISSING)"
+	log " RxTxApp        : $(mtl_rxtxapp_present && echo OK || echo MISSING)"
+	log " MtlManager     : $(mtl_manager_present && echo OK || echo MISSING)"
 	if ldconfig -p 2>/dev/null | grep -Eq 'libmtl\.so(\s|$)' || [[ -f /usr/local/lib/x86_64-linux-gnu/libmtl.so || -f /usr/local/lib64/libmtl.so || -f /usr/local/lib/libmtl.so ]]; then
 		log " libmtl.so      : OK"
 	else
 		log " libmtl.so      : MISSING"
 	fi
 	log " libdpdk        : $(pkg-config --modversion libdpdk 2>/dev/null || echo MISSING)"
 	log " ice driver     : $(modinfo ice 2>/dev/null | awk '/^version:/ {print $2; exit}' || echo MISSING) @ $(modinfo -n ice 2>/dev/null || echo '<none>')"
-	log " hugepages free : $(awk '/HugePages_Free/ {print $2*2 " MiB"}' /proc/meminfo)"
+	log " hugepages free : $(hugepages_free_mb) MiB"
 	if mountpoint -q /mnt/media; then
 		log " /mnt/media     : $(findmnt -no SOURCE /mnt/media) ($(df -h /mnt/media | awk 'NR==2{print $5" used of "$2}'))"
 		log " media files    : $(find /mnt/media -mindepth 1 -maxdepth 1 2>/dev/null | wc -l) entries"
@@ -195,6 +195,23 @@ print_summary() {
 	trap_arm
 }
 
+# RxTxApp/MtlManager may live at the legacy in-tree build path or at the
+# .local_install prefix that tests/validation/mtl_engine/const.py::PREFIX
+# actually invokes — accept either so preflight matches what pytest runs.
+mtl_rxtxapp_present() {
+	[[ -x tests/tools/RxTxApp/build/RxTxApp || -x .local_install/mtl/bin/RxTxApp ]]
+}
+
+mtl_manager_present() {
+	[[ -x build/manager/MtlManager || -x .local_install/mtl/bin/MtlManager ]]
+}
+
+# Hugepage size varies by host (2MB vs 1GB default_hugepagesz=1G on the
+# kernel cmdline); HugePages_Free*2 silently under-reports on 1GB-page hosts.
+hugepages_free_mb() {
+	awk '/Hugepagesize:/ {sz=$2} /HugePages_Free:/ {free=$2} END {printf "%d", free*sz/1024}' /proc/meminfo
+}
+
 # ============================================================================
 # STAGE FUNCTIONS
 # ============================================================================
@@ -260,7 +277,7 @@ stage_preflight() {
 		warn "preflight: libmtl.so missing in ld cache"
 		missing=1
 	fi
-	if [[ ! -x build/manager/MtlManager || ! -x tests/tools/RxTxApp/build/RxTxApp ]]; then
+	if ! mtl_manager_present || ! mtl_rxtxapp_present; then
 		warn "preflight: MtlManager or RxTxApp missing"
 		missing=1
 	fi
@@ -269,7 +286,7 @@ stage_preflight() {
 		warn "preflight: out-of-tree ice driver not loaded (path=$ice_path)"
 		missing=1
 	fi
-	free_mb=$(awk '/HugePages_Free/ {print $2*2}' /proc/meminfo)
+	free_mb=$(hugepages_free_mb)
 	if ((free_mb < 1024)); then
 		warn "preflight: hugepages free is ${free_mb} MiB (<1024 MiB)"
 		missing=1

tests/validation/common/nicctl.py

@@ -382,6 +382,45 @@ def get_mixed_interfaces_list_single(
                 f"Found {len(host.network_interfaces)} interface(s)."
             )
 
+        tx_pci = host.network_interfaces[tx_index].pci_address.lspci
+        rx_pci = host.network_interfaces[rx_index].pci_address.lspci
+
+        # Check IOMMU group of TX and RX PFs.
+        # One PF is bound to PMD (vfio-pci), while the other PF remains bound to the kernel (host for run/VFs).
+        # This is impossible if they share the same IOMMU group because VFIO group viability is violated.
+        try:
+            tx_group = (
+                self.nicctl_objs[host.name]
+                .connection.execute_command(
+                    f"basename $(readlink /sys/bus/pci/devices/{tx_pci}/iommu_group 2>/dev/null) 2>/dev/null"
+                )
+                .stdout
+                or ""
+            ).strip()
+            rx_group = (
+                self.nicctl_objs[host.name]
+                .connection.execute_command(
+                    f"basename $(readlink /sys/bus/pci/devices/{rx_pci}/iommu_group 2>/dev/null) 2>/dev/null"
+                )
+                .stdout
+                or ""
+            ).strip()
+            if tx_group and rx_group and tx_group == rx_group:
+                if (
+                    tx_interface_type.lower() == "pf"
+                    and rx_interface_type.lower() == "vf"
+                ) or (
+                    tx_interface_type.lower() == "vf"
+                    and rx_interface_type.lower() == "pf"
+                ):
+                    pytest.skip(
+                        f"Skipping mixed PF/VF test: PF {tx_pci} and PF {rx_pci} share the same IOMMU group "
+                        f"({tx_group}) "
+                        f"and cannot be bound to different drivers (vfio-pci vs ice) simultaneously."
+                    )
+        except Exception as e:
+            logger.warning(f"Failed to check IOMMU group conflict: {e}")
+
         tx_interface = self._get_single_interface_by_type(
             host, tx_interface_type, tx_index
         )

tests/validation/conftest.py

@@ -55,6 +55,7 @@
 )
 from pytest_mfd_config.models.topology import TopologyModel
 from pytest_mfd_logging.amber_log_formatter import AmberLogFormatter
+from tests.single.ptp.ptp_helpers import wait_for_ptp4l_foreign_master
 
 logger = logging.getLogger(__name__)
 
@@ -376,13 +377,41 @@ def _select_capture_host(hosts: dict):
     return hosts["client"] if "client" in hosts else list(hosts.values())[0]
 
 
-_REAP_GRACE_SEC = 0.3  # Grace period between SIGTERM and SIGKILL for ptp daemons
+_REAP_POLL_TIMEOUT_SEC = 3  # Max time to wait for a killed daemon to actually exit
 _PHC_SYNC_THRESHOLD_NS = 2000  # Capture PHC must track TAI this tightly
 _PHC_SYNC_TIMEOUT_SEC = 30  # Max wait for phc2sys to converge before capturing
 
 
+def _wait_daemon_dead(host, name: str, timeout_s: float) -> bool:
+    """Poll (best-effort) until no process named *name* remains, up to *timeout_s*.
+
+    A bare ``sleep()`` after ``pkill`` does not guarantee the kernel has
+    finished tearing down the process -- and releasing any PF netdev/PHC fd
+    it held (e.g. ptp4l on the PF interface) -- by the time the caller
+    proceeds. That gap has been observed to race nicctl's VF/PF rebind
+    checks (``_wait_vfio_idle`` / ``bind_kernel`` / ``disable_vf``): if the
+    fd is still open, those calls time out and fall back to a PCI
+    remove+rescan, which force-reprobes the PF (``ice_probe``) and can hit
+    an ``ice`` driver GPF (RSS flow-profile UAF in ``ice_add_prof``).
+    Polling for actual daemon death closes that race. Returns True once
+    confirmed dead (or if liveness can't be probed); False on timeout.
+    """
+    deadline = time.monotonic() + timeout_s
+    while time.monotonic() < deadline:
+        try:
+            res = host.connection.execute_command(
+                f"pgrep -x {name}", expected_return_codes=None
+            )
+        except Exception:
+            return True  # cannot probe; don't block the caller forever
+        if res.return_code != 0:
+            return True
+        time.sleep(0.1)
+    return False
+
+
 def _reap_ptp_daemons(host, *, patterns=("phc2sys", "ptp4l")) -> None:
-    """Forcefully kill any ptp4l/phc2sys daemons.
+    """Forcefully kill any ptp4l/phc2sys daemons and wait for them to exit.
 
     Required because ``host.connection.start_process('sudo <tool> ...')`` wraps
     the daemon in ``bash -c 'sudo ...'``; ``process.kill(SIGTERM)`` only signals
@@ -391,6 +420,9 @@ def _reap_ptp_daemons(host, *, patterns=("phc2sys", "ptp4l")) -> None:
     stale ``struct ptp_clock *`` across SR-IOV VF cycling and have been seen to
     trigger ``ice``-driver use-after-free in ``ptp_clock_index()``, hanging the
     host. Always cleanup via ``pkill`` on the argv, not via the process handle.
+    We also wait for the kill to actually take effect (see
+    :func:`_wait_daemon_dead`) instead of a blind sleep, since callers rebind
+    the PF right after this returns.
     """
     for name in patterns:
         try:
@@ -399,14 +431,23 @@ def _reap_ptp_daemons(host, *, patterns=("phc2sys", "ptp4l")) -> None:
             )
         except Exception as e:
             logger.debug("pkill -TERM %s: %s", name, e)
-    time.sleep(_REAP_GRACE_SEC)
+    for name in patterns:
+        _wait_daemon_dead(host, name, _REAP_POLL_TIMEOUT_SEC)
     for name in patterns:
         try:
             host.connection.execute_command(
                 f"sudo pkill -KILL -x {name} || true", expected_return_codes=None
             )
         except Exception as e:
             logger.debug("pkill -KILL %s: %s", name, e)
+    for name in patterns:
+        if not _wait_daemon_dead(host, name, _REAP_POLL_TIMEOUT_SEC):
+            logger.warning(
+                "%s on %s still alive %ss after SIGKILL; PF rebind may race it",
+                name,
+                host.name,
+                _REAP_POLL_TIMEOUT_SEC,
+            )
 
 
 def _host_tai_utc_offset(host) -> int:
@@ -571,30 +612,115 @@ def ptp_sync(request, test_config: dict, hosts):
 
     host = _select_capture_host(hosts)
     is_single_host = len(hosts) == 1
-    capture_iface = _select_sniff_interface_name(
-        host, capture_cfg, single_host=is_single_host
-    )
 
     # Belt-and-braces: ensure no leftover daemon from a previous test/session
     # is holding a stale PHC handle before we start a new one.
     _reap_ptp_daemons(host)
 
-    logger.info(f"Starting ptp4l for PTP synchronization (iface={capture_iface})")
-    log_path = f"/tmp/ptp4l-{capture_iface}.log"
-    ptp4l_cmd = f"sudo ptp4l -i '{capture_iface}' -s -m -2"
-    ptp4l_process = host.connection.start_process(
-        ptp4l_cmd,
-        stderr_to_stdout=True,
-        output_file=log_path,
-    )
+    # Determine interfaces to sync. If we are running a single-host PTP loopback test,
+    # we need to sync BOTH the active ports (TX on primary, RX on redundant) so that
+    # the hardware clocks on both ports are disciplined to the same GM.
+    # Note: Only sync interfaces that are currently in kernel mode (i.e. having active netdevs in /sys/class/net),
+    # otherwise ptp4l will fail to start.
+    #
+    # Also: some tests (e.g. the "mixed" pf_tx_vf_rx interface_profile) bind an
+    # interface fully to DPDK (as a PF, no VF) from inside the test BODY, which
+    # runs *after* this fixture. At fixture-setup time that interface is still
+    # kernel-bound, so the /sys/class/net check below would pass and we'd start
+    # ptp4l on it -- only for the test body's own bind_pmd() to yank the netdev
+    # away moments later, mid-convergence. ptp4l then faults (link down) and
+    # self-declares grandmaster instead of tracking the real one. Skip syncing
+    # whichever index this test's own interface_profile will bind fully to DPDK.
+    # (Single-host only: the index resolution below only makes sense for the
+    # loopback topology; a mixed profile has never been combined with a
+    # multi-host `hosts` config, and both tx_type/rx_type being "PF" is not a
+    # combination any current test uses -- only one side checked below.)
+    interfaces_to_sync = []
+    if is_single_host:
+        full_pf_dpdk_index = None
+        callspec = getattr(request.node, "callspec", None)
+        profile = callspec.params.get("interface_profile") if callspec else None
+        if profile and profile.get("mode") == "mixed":
+            tx_index = test_config.get("tx_interface_index", 0)
+            rx_index = test_config.get("rx_interface_index", 1)
+            if profile.get("tx_type", "").upper() == "PF":
+                full_pf_dpdk_index = tx_index
+            elif profile.get("rx_type", "").upper() == "PF":
+                full_pf_dpdk_index = rx_index
+
+        # In single-host mode, use both the primary and redundant physical interfaces if available
+        for idx, nic in enumerate(host.network_interfaces[:2]):
+            if idx == full_pf_dpdk_index:
+                logger.debug(
+                    f"PTP skip: interface {nic.name} (index {idx}) will be bound fully to DPDK by this test"
+                )
+                continue
+            try:
+                # Check if the interface is currently in kernel mode
+                check_cmd = f"[ -d /sys/class/net/{nic.name} ]"
+                res = host.connection.execute_command(
+                    check_cmd, expected_return_codes=None
+                )
+                if res.return_code == 0:
+                    interfaces_to_sync.append(nic.name)
+                else:
+                    logger.debug(
+                        f"PTP skip: Interface {nic.name} is not present in /sys/class/net/ (likely bound to DPDK PMD)"
+                    )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to check netdev directories for {nic.name}: {e}"
+                )
+    else:
+        # Multi-host mode: only run on the resolved capture interface
+        capture_iface = _select_sniff_interface_name(
+            host, capture_cfg, single_host=is_single_host
+        )
+        try:
+            # Check if the interface is currently in kernel mode
+            check_cmd = f"[ -d /sys/class/net/{capture_iface} ]"
+            res = host.connection.execute_command(check_cmd, expected_return_codes=None)
+            if res.return_code == 0:
+                interfaces_to_sync.append(capture_iface)
+            else:
+                logger.warning(
+                    f"PTP skip: Capture interface {capture_iface} is not present in /sys/class/net/"
+                )
+        except Exception as e:
+            logger.warning(
+                f"Failed to check netdev directories for capture interface {capture_iface}: {e}"
+            )
 
-    # Give ptp4l a moment to fail fast (e.g., missing interface).
-    time.sleep(0.2)
-    if not ptp4l_process.running:
-        _reap_ptp_daemons(host)
-        raise RuntimeError(
-            f"Failed to start ptp4l (iface={capture_iface}). log={log_path}"
+    ptp4l_processes = []
+    for iface in interfaces_to_sync:
+        logger.info(f"Starting ptp4l for PTP synchronization (iface={iface})")
+        log_path = f"/tmp/ptp4l-{iface}.log"
+        ptp4l_cmd = f"sudo ptp4l -i '{iface}' -s -m -2"
+        ptp4l_process = host.connection.start_process(
+            ptp4l_cmd,
+            stderr_to_stdout=True,
+            output_file=log_path,
         )
+        time.sleep(0.2)
+        if not ptp4l_process.running:
+            _reap_ptp_daemons(host)
+            raise RuntimeError(f"Failed to start ptp4l (iface={iface}). log={log_path}")
+        ptp4l_processes.append(ptp4l_process)
+
+    # A hardware PTP clock that never actually locks to a grandmaster (or
+    # never even hears one) free-runs -- exactly the condition that produced
+    # a ~-55.8s packet_ts_vs_rtp_ts offset and VRX compliance failure in the
+    # past (see /memories/repo/ptp_sync_fixture_root_cause.md). Block here,
+    # before the test body/pcap_capture ever touch this clock, so a dead/
+    # unreachable grandmaster surfaces as a fast, clear ERROR. Must reap on
+    # timeout too, else the just-started ptp4l processes are orphaned (same
+    # leak class _reap_ptp_daemons/reap_leaked_phc_daemons exist to prevent).
+    try:
+        for iface in interfaces_to_sync:
+            wait_for_ptp4l_foreign_master(host, f"/tmp/ptp4l-{iface}.log")
+    except Exception:
+        _reap_ptp_daemons(host)
+        raise
 
     try:
         yield
@@ -1246,12 +1372,12 @@ def pcap_capture(
                         streams = (report or {}).get("streams") or []
                         if not streams:
                             # Empty capture — interface may not see VF-to-VF
-                            # loopback traffic.  Not a real failure.
-                            update_compliance_result(request.node.nodeid, "N/A")
-                            logger.warning(
-                                "PCAP compliance check skipped: capture "
-                                "contains no streams (capture interface may "
-                                "not see VF-to-VF loopback traffic)"
+                            # loopback traffic. Reject with a test failure so we catch false passes.
+                            update_compliance_result(request.node.nodeid, "Fail")
+                            log_fail(
+                                "PCAP compliance check failed: capture contains no streams. "
+                                "Normally interface may not see VF-to-VF loopback traffic, "
+                                "but empty capture is disallowed for strict validation/compliance."
                             )
                         else:
                             update_compliance_result(request.node.nodeid, "Fail")

tests/validation/tests/single/ptp/grandmaster/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 Intel Corporation

tests/validation/tests/single/ptp/grandmaster/conftest.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 Intel Corporation
+"""Fixtures local to tests/single/ptp/grandmaster/ (per-category, not framework)."""
+import logging
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def require_grandmaster(hosts, ptp_sync):
+    """Return the ptp4l log path, already confirmed synced by ``ptp_sync``.
+
+    ``ptp_sync`` itself now blocks in fixture setup until every ptp4l it
+    starts has heard a foreign master (see conftest.py), so this fixture no
+    longer needs to re-run that wait -- it only resolves and returns the log
+    path for callers that want to inspect it further.
+    """
+    host = list(hosts.values())[0]
+    log_path = (
+        host.connection.execute_command(
+            "ls -t /tmp/ptp4l-*.log 2>/dev/null | head -1", expected_return_codes=None
+        ).stdout
+        or ""
+    ).strip()
+    if not log_path:
+        raise RuntimeError(
+            "ENVIRONMENT NOT READY: ptp_sync did not start ptp4l (check "
+            "capture_cfg.enable in test_config.yaml) -- cannot verify a "
+            "grandmaster is reachable"
+        )
+    return log_path