diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..bb611ef334 --- /dev/null +++ b/conftest.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Repository-wide pytest bootstrap.""" + +from __future__ import annotations + +import os +from pathlib import Path + + +def _pin_child_processes_to_this_tree() -> None: + """Put this checkout's ``src`` first on the child ``PYTHONPATH``. + + ``pythonpath`` in ``pyproject.toml`` only reaches the pytest process. Tests + that shell out inherit the ambient one, so a Hyperloom installed elsewhere + on the host silently answers their imports instead of the tree under test. + """ + src = str(Path(__file__).resolve().parent / "src") + existing = [entry for entry in os.environ.get("PYTHONPATH", "").split(os.pathsep) if entry] + os.environ["PYTHONPATH"] = os.pathsep.join([src, *(entry for entry in existing if entry != src)]) + + +_pin_child_processes_to_this_tree() diff --git a/docs/compatibility.rst b/docs/compatibility.rst index 0d8a05dfd9..5eaee88761 100644 --- a/docs/compatibility.rst +++ b/docs/compatibility.rst @@ -123,8 +123,12 @@ The following inference frameworks are supported: - 10.0 (docker) / 7.2.4 (bare-metal wheel) - Default framework. Validated ``docker`` images use ROCm 10.0 (``rocm10`` tags below). On a ROCm 7.2.x bare-metal host the installer still derives ``SGLANG_ROCM_EXTRA=rocm724`` from ``torch.version.hip`` (see below). * - vLLM - - 7.2.3 - - Do not mix frameworks within one session + - 10.0 (docker, bare-metal source) / 7.2.3 (bare-metal wheel) + - Validated ``docker`` image is the ROCm 10.0 build below. On bare metal the + installer routes by the detected stack: a ROCm 7.2.x host installs the + ``rocm723`` wheel, a ROCm 10 host (``torch.version.hip`` 7.15) builds vLLM + from source, and any other combination is rejected (see below). Do not mix + frameworks within one session * - Atom - 7.2.4 - AMD out-of-tree engine, launched as ``python3 -m atom.entrypoints.openai_server``. Container image only: ``install_baremetal.sh`` verifies ``atom`` but cannot install it, because ``--install-framework`` accepts only ``none``, ``sglang`` and ``vllm``. The kernel phase defaults to the KernelForge backend here (``KERNEL_OPT_BACKEND_ORDER`` is set to ``forge`` when you leave it unset). On a quantized non-vLLM backend GEAK must resolve a live rewrite seam rather than guess one, which forge does not require. @@ -137,7 +141,7 @@ Container images Pick the image that matches your environment. Public Docker Hub refs are used on your own GPU machine: the official upstream ``lmsysorg/sglang-rocm:`` -for SGLang, ``vllm/vllm-openai-rocm:`` for vLLM and +for SGLang, ``rocm/vllm:`` for vLLM and ``rocm/atom-dev:`` for Atom. If your deployment uses a private registry mirror, set the registry prefix accordingly. @@ -151,7 +155,7 @@ mirror, set the registry prefix accordingly. - MI300X / MI325X * - ``lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260920`` - MI355X - * - ``vllm/vllm-openai-rocm:v0.29.0`` + * - ``rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0`` - MI300X / MI325X / MI355X * - ``rocm/atom-dev:v0.1.7-rc0`` - MI355X (verified); MI300X / MI325X untested @@ -159,12 +163,19 @@ mirror, set the registry prefix accordingly. The vLLM image entrypoint is ``vllm serve``, so override it (for example ``--entrypoint tail``) when starting a long-running Hyperloom container. +The ``rocm/vllm`` image serves an installed wheel, so ``install.sh`` turns its +``/app/vllm`` checkout into the patchable source tree: it pins the tree to vLLM +commit ``f46a9dfe2c5f`` (``VLLM_IMAGE_SOURCE_COMMIT``) under a synthetic baseline +commit and puts it ahead of site-packages on the server's ``PYTHONPATH``. This +activates only when the installed wheel's version names that commit; any other +vLLM image is left on its wheel. + ``rocm/atom-dev`` also publishes a ``latest`` tag, which tracks the newest nightly build and moves. Pin the versioned tag so a session stays reproducible. Browse all available tags at `hub.docker.com/r/lmsysorg/sglang-rocm/tags `_, -`hub.docker.com/r/vllm/vllm-openai-rocm/tags `_ +`hub.docker.com/r/rocm/vllm/tags `_ and `hub.docker.com/r/rocm/atom-dev/tags `_. @@ -185,11 +196,11 @@ Hyperloom does not install ROCm or torch itself. - Ubuntu 24.04 - Recommended bare-metal baseline. vLLM 0.28.0+ ROCm wheels require glibc >= 2.39, so Ubuntu 22.04 hosts must downgrade vLLM (for example ``VLLM_VERSION=0.27.1``) or use ``docker`` mode instead. * - ROCm - - 7.2.x (bare metal) / 10.0 (SGLang docker) - - Patch levels differ by framework and setup mode. vLLM uses ROCm 7.2.3 (``rocm723``). On a ROCm 7.2.x bare-metal host, SGLang resolves ``SGLANG_ROCM_EXTRA=rocm724`` (ROCm 7.2.4 layer). The validated SGLang ``docker`` stack uses ROCm 10.0 user space (``rocm10`` images below), not the 7.2.4 bare-metal wheel path. + - 7.2.x or 10.0 (bare metal) / 10.0 (docker) + - Patch levels differ by framework and setup mode. On ROCm 7.2.x, vLLM installs the ROCm 7.2.3 wheel (``rocm723``); on ROCm 10.0, vLLM is built from source. On a ROCm 7.2.x bare-metal host, SGLang resolves ``SGLANG_ROCM_EXTRA=rocm724`` (ROCm 7.2.4 layer). The validated ``docker`` stacks use ROCm 10.0 user space (``rocm10`` images above), not the 7.2.x bare-metal wheel paths. * - Python - 3.12 - - Required by the vLLM ROCm wheel. + - Required by the vLLM ROCm wheel. The vLLM source build accepts Python >= 3.10, < 3.15. * - ROCm torch - ROCm build matching the host ROCm - Preinstalled by the operator; not managed by Hyperloom. @@ -197,12 +208,13 @@ Hyperloom does not install ROCm or torch itself. - 0.5.20 (rocm10), pinned to commit ``94602c9c2b7c`` - Recommended ``docker`` stack uses the ``lmsysorg/sglang-rocm:v0.5.20-rocm10-*`` images above (ROCm 10.0 user space). ``SGLANG_REF`` is the ``v0.5.20`` release commit (peeled from the tag object). The pin sits at 0.5.20 because through 0.5.18 the HIP extra pinned ``compressed-tensors==0.15.0``, which caps torch below 2.11 and therefore cannot resolve at all against a ROCm 10 stack; 0.5.19 moved that dependency into ``runtime_common`` unpinned, leaving the installer's ROCm torch constraint as the version pip solves for. Bare-metal installs on ROCm 10 take the source-install path; on ROCm 7.2.x hosts, ``SGLANG_ROCM_EXTRA=rocm724`` still selects the AMD wheel index. Kernel-shape profiling for SGLang >= 0.5.18 uses TraceLens ``kernel_shape_tool`` rather than git-applying SGLang roofline patches. * - vLLM - - v0.29.0 (rocm723), isolated venv - - Installs ``vllm==0.29.0+rocm723`` from the wheels.vllm.ai pip index on Ubuntu 24.04+. vLLM's ROCm wheel pins its own torch, so it installs into a dedicated venv (``--framework-env isolated``, the default for vLLM) and never touches the host torch. + - v0.29.0 (rocm723) wheel on ROCm 7.2.x; v0.29.0 source build, pinned to commit ``98dff2a81d74``, on ROCm 10; isolated venv + - On ROCm 7.2.x, installs ``vllm==0.29.0+rocm723`` from the wheels.vllm.ai pip index on Ubuntu 24.04+. vLLM's ROCm wheel pins its own torch, so it installs into a dedicated venv (``--framework-env isolated``, the default for vLLM) and never touches the host torch. On ROCm 10 (``torch.version.hip`` 7.15) no wheel is published, so the installer checks out ``VLLM_SOURCE_REF`` (the ``v0.29.0`` release commit) into ``VLLM_ROOT`` (default ``/opt/hyperloom/vllm``) and builds it into ``VLLM_VENV_ROOT`` as a system-site-packages venv over the host ROCm torch. The source route requires ``--framework-env isolated``, ``git``, ``gcc``/``g++`` >= 11.3, ``cmake`` >= 3.26.1, ``ninja``, ``hipcc`` and the ROCm devel headers, installs AITER into the same venv when it is not already importable, and writes ``VLLM_ROCM_USE_AITER=1`` to ``.env`` when ``aiter`` imports. ``VLLM_INSTALL_METHOD`` (``auto``, ``wheel``, ``source``) can only confirm the detected route; a ROCm stack other than 7.2.x or 10 is rejected. -Bare-metal ROCm patch levels differ per framework, and each one matches its -container image. The vLLM stack installs the ``rocm723`` variant (ROCm -7.2.3), matching ``vllm/vllm-openai-rocm:v0.29.0``; the SGLang stack +Bare-metal ROCm patch levels differ per framework. On ROCm 7.2.x the vLLM stack +installs the ``rocm723`` variant (ROCm 7.2.3); on ROCm 10 it builds from source +against the host ROCm, matching the ROCm 10 user space of the ``rocm/vllm`` image +the ``docker`` route uses; the SGLang stack installs from the ROCm 7.2.4 AMD wheel index when a host stays on ROCm 7.2.x; the recommended SGLang ``docker`` stack uses the two ``lmsysorg/sglang-rocm:v0.5.20-rocm10-*`` images (ROCm 10.0). ``docker`` mode is still @@ -211,7 +223,8 @@ surrounding torch, Triton, and AITER builds. These are recommended defaults, not hard pins. Framework and ROCm versions are overridable via env (``SGLANG_REF``, ``SGLANG_ROCM_EXTRA``, ``VLLM_VERSION``, -``VLLM_ROCM_VARIANT``) for hosts that need a different pinned stack. +``VLLM_ROCM_VARIANT``, ``VLLM_INSTALL_METHOD``, ``VLLM_REPO``, ``VLLM_SOURCE_REF``, +``VLLM_ROOT``) for hosts that need a different pinned stack. The table above is the validated combination, and ROCm 7.2.x under a single ``/opt/rocm`` prefix is the layout to prefer. A host where ROCm arrives as diff --git a/docs/install/install.md b/docs/install/install.md index 867fef0c6a..80e6a0fe64 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -427,7 +427,7 @@ It is recommended that you use a ROCm image that already ships the serving framework, so nothing needs to be installed inside the container beyond Hyperloom's runtime deps. The following images are recommended: -- `vllm`: `docker.io/vllm/vllm-openai-rocm:v0.29.0` +- `vllm`: `docker.io/rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0` - `sglang` MI300X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi30x-20260920` - `sglang` MI355X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260920` @@ -435,7 +435,7 @@ Start a long-running container from the repo root, mounting it at the same path so `.env`, logs, and session artifacts stay valid: ```bash -export HYPERLOOM_IMAGE=docker.io/vllm/vllm-openai-rocm:v0.29.0 +export HYPERLOOM_IMAGE=docker.io/rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0 export REPO_ROOT="$(pwd -P)" docker run -d \ --name "${HYPERLOOM_CONTAINER_NAME:-hyperloom-local}" \ diff --git a/examples/hyperloom-custom-advanced/SKILL.md b/examples/hyperloom-custom-advanced/SKILL.md index e64ed018e1..c4706b8792 100644 --- a/examples/hyperloom-custom-advanced/SKILL.md +++ b/examples/hyperloom-custom-advanced/SKILL.md @@ -44,7 +44,7 @@ skip the user-approval step (#1314). Suggested Docker images: -- `vllm`: `docker.io/vllm/vllm-openai-rocm:v0.29.0` +- `vllm`: `docker.io/rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0` - `sglang` MI300X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi30x-20260920` - `sglang` MI355X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260920` diff --git a/examples/hyperloom-qwen3-14b-fp8-12h-forge/SKILL.md b/examples/hyperloom-qwen3-14b-fp8-12h-forge/SKILL.md index d66a8b7903..712d9e6827 100644 --- a/examples/hyperloom-qwen3-14b-fp8-12h-forge/SKILL.md +++ b/examples/hyperloom-qwen3-14b-fp8-12h-forge/SKILL.md @@ -78,7 +78,7 @@ skip the user-approval step (#1314). Suggested Docker images: -- `vllm`: `docker.io/vllm/vllm-openai-rocm:v0.29.0` +- `vllm`: `docker.io/rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0` - `sglang` MI300X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi30x-20260920` - `sglang` MI355X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260920` diff --git a/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md b/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md index 0d255ef7e4..722a274c6c 100644 --- a/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md +++ b/examples/hyperloom-qwen3-14b-fp8-12h/SKILL.md @@ -34,7 +34,7 @@ skip the user-approval step (#1314). Suggested Docker images: -- `vllm`: `docker.io/vllm/vllm-openai-rocm:v0.29.0` +- `vllm`: `docker.io/rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0` - `sglang` MI300X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi30x-20260920` - `sglang` MI355X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260920` diff --git a/examples/hyperloom-qwen3-8b-3h/SKILL.md b/examples/hyperloom-qwen3-8b-3h/SKILL.md index 1d92e7fbde..5ed9fc58da 100644 --- a/examples/hyperloom-qwen3-8b-3h/SKILL.md +++ b/examples/hyperloom-qwen3-8b-3h/SKILL.md @@ -34,7 +34,7 @@ skip the user-approval step (#1314). Suggested Docker images: -- `vllm`: `docker.io/vllm/vllm-openai-rocm:v0.29.0` +- `vllm`: `docker.io/rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0` - `sglang` MI300X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi30x-20260920` - `sglang` MI355X: `docker.io/lmsysorg/sglang-rocm:v0.5.20-rocm10-mi35x-20260920` diff --git a/src/hyperloom/agents/kernel/tests/test_install_env_credentials.py b/src/hyperloom/agents/kernel/tests/test_install_env_credentials.py index 8ff39b165f..2fc91ff14e 100644 --- a/src/hyperloom/agents/kernel/tests/test_install_env_credentials.py +++ b/src/hyperloom/agents/kernel/tests/test_install_env_credentials.py @@ -6,6 +6,7 @@ from __future__ import annotations +import os import re import subprocess import tempfile @@ -16,6 +17,20 @@ ROOT = Path(__file__).resolve().parents[1] INSTALL_SCRIPT = ROOT / "scripts" / "install.sh" +# The env file's whole subject is "was this name already exported at launch", so every name these tests reason about +# has to start out unset regardless of what the operator's shell carries. +_CONTROLLED_ENV_NAMES = ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_CUSTOM_HEADERS", + "CLAUDE_CODE_OAUTH_TOKEN", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_CUSTOM_HEADERS", + "MAGPIE_PATH", +) + _INSTALL_KEY = "ak-install-time-key" _INSTALL_URL = "https://gateway.install-time.example/v1" _INSTALL_TOKEN = "sk-ant-oat01-install-time-token" @@ -24,6 +39,11 @@ _ROTATED_TOKEN = "sk-ant-oat01-rotated-token" +def _controlled_env() -> dict[str, str]: + """A copy of the ambient environment with every name these tests own removed.""" + return {k: v for k, v in os.environ.items() if k not in _CONTROLLED_ENV_NAMES} + + def _sourceable_installer(dest_dir: Path) -> Path: """Write a copy of install.sh with the top-level ``main \"$@\"`` call removed.""" text = INSTALL_SCRIPT.read_text(encoding="utf-8") @@ -63,6 +83,7 @@ def _write_env_file(self, workdir: Path, repo_root: Path) -> Path: text=True, capture_output=True, timeout=60, + env=_controlled_env(), ) self.assertEqual( proc.returncode, @@ -86,6 +107,7 @@ def _source_and_report(self, env_file: Path, exported: dict[str, str]) -> tuple[ text=True, capture_output=True, timeout=60, + env=_controlled_env(), ) self.assertEqual( proc.returncode, diff --git a/src/hyperloom/agents/kernel/tests/test_ray_version_mismatch.py b/src/hyperloom/agents/kernel/tests/test_ray_version_mismatch.py index bbd54a4bfc..64889b9cc0 100644 --- a/src/hyperloom/agents/kernel/tests/test_ray_version_mismatch.py +++ b/src/hyperloom/agents/kernel/tests/test_ray_version_mismatch.py @@ -53,6 +53,22 @@ def _shutdown(*args, **kwargs): return fake +def _settle_ray_init_state(timeout: float = 30.0) -> None: + """Wait out any abandoned ``ray-init`` runner, then clear the process-wide flag. + + Clearing alone is not isolation: a runner abandoned by an earlier test sets + the flag whenever it finally returns, which under load is after the next + test has already cleared it. The runner releases ``_INIT_GATE`` as its last + action, so holding the gate proves no connect can still set the flag. + """ + held = ray_runtime._INIT_GATE.acquire(timeout=timeout) + try: + ray_runtime._STALE_CONNECT_POSSIBLE.clear() + finally: + if held: + ray_runtime._INIT_GATE.release() + + @pytest.fixture(autouse=True) def _clear_stale_connect_flag(): """``_STALE_CONNECT_POSSIBLE`` is process-wide by design, so tests must isolate it. @@ -61,9 +77,9 @@ def _clear_stale_connect_flag(): clear a session it never left behind -- the coupling is real in production too, where it is exactly the intended behaviour across legs. """ - ray_runtime._STALE_CONNECT_POSSIBLE.clear() + _settle_ray_init_state() yield - ray_runtime._STALE_CONNECT_POSSIBLE.clear() + _settle_ray_init_state() def test_is_version_mismatch_detects_banner(): diff --git a/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py b/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py index 4c13c8c1be..ad05c88013 100644 --- a/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py +++ b/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py @@ -150,12 +150,17 @@ def resolve_arch_json_path(platform: str) -> Path | None: def build_hyperloom_arch_spec(platform: str) -> dict | None: - """Build a TraceLens arch spec from hyperloom's own achievable-TFLOPS table.""" + """Build a TraceLens arch spec from hyperloom's own hardware tables.""" try: - from hyperloom.orchestrator.kernel.roofline_ceiling import HW_SPECS_ACHIEVABLE + from hyperloom.orchestrator.kernel.roofline_ceiling import HW_SPECS, HW_SPECS_ACHIEVABLE except Exception: return None - spec = HW_SPECS_ACHIEVABLE.get((platform or "").strip().lower()) + key = (platform or "").strip().lower() + # Achievable first, vendor theoretical when a GPU has no measured entry: the same order the + # model-level ceiling and the bypass roofline already resolve peaks in. + spec = HW_SPECS_ACHIEVABLE.get(key) + if not isinstance(spec, dict): + spec = HW_SPECS.get(key) if not isinstance(spec, dict): return None table = spec.get("peak_tflops") diff --git a/src/hyperloom/common/provenance.py b/src/hyperloom/common/provenance.py index cb4c428a47..8c38673cb7 100644 --- a/src/hyperloom/common/provenance.py +++ b/src/hyperloom/common/provenance.py @@ -7,6 +7,7 @@ import hashlib import os +import gzip import re import subprocess # nosec B404 - best-effort, guarded provenance probes only. import sys @@ -29,6 +30,9 @@ "aiter": ("AITER_COMMIT", "AITER_VERSION", "AITER_REF"), "sglang": ("SGLANG_VERSION", "SGL_VERSION"), "vllm": ("VLLM_VERSION",), + # Probe-only: no env var pins torch, and inventing one nothing writes would be a pin that never fires. It is + # recorded because the torch build selects Kineto's GPU backend, which decides whether a trace has any kernel. + "torch": (), } # The interpreter preflight resolved for the serving framework. ``--framework-env @@ -164,7 +168,11 @@ def detect_stack_fingerprint(env: Mapping[str, str], *, probe: bool = True) -> d def _framework_site_packages(env: Mapping[str, str], component: str) -> list[str] | None: """``site-packages`` dirs of the interpreter preflight resolved, if any.""" resolved_for = _env_first(env, RESOLVED_FRAMEWORK_ENV) - if not resolved_for or resolved_for.strip().lower() != component: + resolved_for = resolved_for.strip().lower() if resolved_for else "" + # torch rides whichever interpreter serves: the isolated vLLM overlay pins its own, and the orchestrator's answer + # would name a build the run never executed on. + rides_framework_venv = component == "torch" or (resolved_for == "vllm" and component == "aiter") + if component != resolved_for and not rides_framework_venv: return None python_exe = _env_first(env, RESOLVED_FRAMEWORK_PYTHON_ENV) if not python_exe or python_exe == sys.executable: @@ -185,13 +193,48 @@ def _framework_site_packages(env: Mapping[str, str], component: str) -> list[str return hits or None +#: Kineto names its GPU backend in the trace's top-level metadata, which PyTorch writes ahead of ``traceEvents``, so +#: only the head of the file is read rather than the whole (often multi-hundred-MB) event array. +_KINETO_BACKEND_RE = re.compile(r"\"(rocprofiler-sdk|roctracer)_version\"\s*:\s*\"?([0-9][0-9.]*)\"?") +_TRACE_HEAD_BYTES = 65536 + + +def detect_kineto_backend(trace_path: str | Path) -> str: + """The GPU backend Kineto recorded a trace with, e.g. ``rocprofiler-sdk 1.3``, or ``""`` when unreadable. + + The torch build selects this backend and the trace is the only place that states it outright; an unreadable trace + yields empty rather than a guess, because this is recorded as provenance. + """ + path = Path(trace_path) + if path.is_dir(): + # Profile executors record the workspace, not the file; the backend is named inside the trace. + traces = sorted(path.glob("*.trace.json.gz")) or sorted(path.glob("*.trace.json")) + if not traces: + return "" + path = traces[-1] + try: + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rb") as handle: # type: ignore[operator] + head = handle.read(_TRACE_HEAD_BYTES) + except (OSError, EOFError, gzip.BadGzipFile): + return "" + found = _KINETO_BACKEND_RE.search(head.decode("utf-8", errors="replace")) + return f"{found.group(1)} {found.group(2)}" if found else "" + + def _probe_pkg_version(component: str, venv_path: list[str] | None = None) -> str: """Best-effort installed-package version for a stack component.""" # AITER renamed its distribution from ``aiter`` to ``amd-aiter`` at v0.1.8, so ``aiter`` was simply the wrong # name. It is not kept as a fallback: on PyPI that name belongs to an unrelated async-iterator library, and # recording its ``0.13.20191203`` as the AITER version would be worse than recording nothing, because it looks # like an answer. Installs older than v0.1.8 are covered by ``AITER_REF`` instead, which is exact. - dist = {"sglang": "sglang", "vllm": "vllm", "aiter": "amd-aiter"}.get(component) + dist = { + "rocm": "rocm-sdk-core", + "sglang": "sglang", + "vllm": "vllm", + "aiter": "amd-aiter", + "torch": "torch", + }.get(component) if not dist: return "" if venv_path is not None: diff --git a/src/hyperloom/inference_optimizer/SKILL.md b/src/hyperloom/inference_optimizer/SKILL.md index 95c6aed9d3..6db29390d4 100644 --- a/src/hyperloom/inference_optimizer/SKILL.md +++ b/src/hyperloom/inference_optimizer/SKILL.md @@ -1132,6 +1132,9 @@ if [ -f "$REPO_ROOT/.env" ]; then set -a; . "$REPO_ROOT/.env"; set +a; fi eval "$_dotenv_prev" unset _dotenv_prev . "${KERNEL_AGENT_ENV:-${USER_DATA_PATH:-/workspace/hyperloom}/runtime/kernel-agent.env.sh}" +# Launch is its own shell and inherits no PYTHON from the smoke test, and +# kernel-agent.env.sh does not set it; resolve it before it leads PATH. +export PYTHON="${PYTHON:-$(command -v python3)}" export PATH="$(dirname "$PYTHON"):/usr/local/bin:$PATH" export RUN_TAG="$(basename "$MODEL_PATH")-$(date +%Y%m%d_%H%M%S)" # RUN_LOG/PID/launch-info live under the workspace until the session_dir diff --git a/src/hyperloom/inference_optimizer/assets/install.sh b/src/hyperloom/inference_optimizer/assets/install.sh index 271843761c..587c6e638c 100755 --- a/src/hyperloom/inference_optimizer/assets/install.sh +++ b/src/hyperloom/inference_optimizer/assets/install.sh @@ -151,6 +151,10 @@ if [ -z "${_user_data_was_set}" ]; then fi HYPERLOOM_RUNTIME_DIR="${HYPERLOOM_RUNTIME_DIR:-${USER_DATA_PATH}/runtime}" KERNEL_AGENT_ENV="${KERNEL_AGENT_ENV:-${HYPERLOOM_RUNTIME_DIR}/kernel-agent.env.sh}" +VLLM_IMAGE_SOURCE_ROOT="/app/vllm" +VLLM_IMAGE_SOURCE_COMMIT="f46a9dfe2c5f57bebbd29556cbbb25eabd874226" +VLLM_IMAGE_REPO="${VLLM_IMAGE_REPO:-https://github.com/vllm-project/vllm.git}" +VLLM_IMAGE_SOURCE_ACTIVE=0 # Legacy variable kept for compatibility; open-source checkouts use _open_source_root. HYPERLOOM_ROOT="${HYPERLOOM_ROOT:-${HYPERLOOM_RUNTIME_DIR}/source-mirrors}" # Writable, repo-local base for auto-cloned deps: $HYPERLOOM_CACHE_DIR else @@ -383,6 +387,210 @@ git_fetch_pinned() { return 0 } +probe_vllm_image_wheel() { + "$PYTHON" - <<'PY' +import re +from importlib import metadata +from pathlib import Path + +try: + dist = metadata.distribution("vllm") +except metadata.PackageNotFoundError: + raise SystemExit(1) +version = dist.version.lower() +match = re.search(r"(?:^|[.+])g([0-9a-f]{7,40})(?=$|[.+])", version) +if match is None: + raise SystemExit(2) +print(f"{version}\t{match.group(1)}\t{Path(dist.locate_file('vllm')).resolve()}") +PY +} + +prepare_vllm_image_git_tree() { + local root="$1" origin="" head="" parent="" subject="" baseline_ref="" upstream_ref="" + local index_tmp git_dir tree baseline commit_date + [[ "$VLLM_IMAGE_SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || + { die "VLLM image source commit must be a full SHA"; return 1; } + if git -C "$root" rev-parse --git-dir >/dev/null 2>&1; then + head="$(git -C "$root" rev-parse --verify HEAD 2>/dev/null || true)" + if [ -n "$head" ] && + { ! git -C "$root" diff --quiet --ignore-submodules=all || + ! git -C "$root" diff --cached --quiet --ignore-submodules=all; }; then + die "vLLM image source has tracked or staged user changes: ${root}"; return 1 + fi + else + [ ! -e "$root/.git" ] || + { die "vLLM image source has invalid Git metadata: ${root}"; return 1; } + git init -q "$root" || { die "failed to initialize Git metadata in ${root}"; return 1; } + fi + origin="$(git -C "$root" remote get-url origin 2>/dev/null || true)" + if [ -z "$origin" ]; then + git -C "$root" remote add origin "$VLLM_IMAGE_REPO" || + { die "failed to add vLLM image source remote"; return 1; } + elif [ "$origin" != "$VLLM_IMAGE_REPO" ]; then + die "vLLM image source origin mismatch: ${origin}"; return 1 + fi + if [ -n "$head" ]; then + parent="$(git -C "$root" rev-parse "${head}^" 2>/dev/null || true)" + subject="$(git -C "$root" show -s --format=%s "$head" 2>/dev/null || true)" + baseline_ref="$(git -C "$root" rev-parse refs/hyperloom/image-baseline 2>/dev/null || true)" + upstream_ref="$(git -C "$root" rev-parse refs/hyperloom/upstream 2>/dev/null || true)" + if [ "$parent" != "$VLLM_IMAGE_SOURCE_COMMIT" ] || + [ "$subject" != "Hyperloom prebuilt vLLM image baseline" ] || + [ "$baseline_ref" != "$head" ] || + [ "$upstream_ref" != "$VLLM_IMAGE_SOURCE_COMMIT" ]; then + die "vLLM image source HEAD is not the managed synthetic baseline: ${root}"; return 1 + fi + export VLLM_IMAGE_UPSTREAM_SHA="$VLLM_IMAGE_SOURCE_COMMIT" + export VLLM_IMAGE_BASELINE_SHA="$head" + return 0 + fi + git -C "$root" fetch --quiet origin "$VLLM_IMAGE_SOURCE_COMMIT" || + { die "failed to fetch vLLM image source commit ${VLLM_IMAGE_SOURCE_COMMIT}"; return 1; } + index_tmp="$(mktemp)" + if ! GIT_INDEX_FILE="$index_tmp" git -C "$root" read-tree "$VLLM_IMAGE_SOURCE_COMMIT" || + ! GIT_INDEX_FILE="$index_tmp" git -C "$root" add -u -- .; then + rm -f "$index_tmp"; die "failed to capture vLLM image tracked deltas"; return 1 + fi + tree="$(GIT_INDEX_FILE="$index_tmp" git -C "$root" write-tree)" || + { rm -f "$index_tmp"; die "failed to write vLLM image baseline tree"; return 1; } + commit_date="$(git -C "$root" show -s --format=%cI "$VLLM_IMAGE_SOURCE_COMMIT")" + baseline="$( + printf '%s\n' "Hyperloom prebuilt vLLM image baseline" | + GIT_AUTHOR_NAME=Hyperloom GIT_AUTHOR_EMAIL=hyperloom@amd.com GIT_AUTHOR_DATE="$commit_date" \ + GIT_COMMITTER_NAME=Hyperloom GIT_COMMITTER_EMAIL=hyperloom@amd.com GIT_COMMITTER_DATE="$commit_date" \ + git -C "$root" commit-tree "$tree" -p "$VLLM_IMAGE_SOURCE_COMMIT" + )" || { rm -f "$index_tmp"; die "failed to commit vLLM image baseline tree"; return 1; } + git_dir="$(git -C "$root" rev-parse --absolute-git-dir)" + mv "$index_tmp" "$git_dir/index" + git -C "$root" update-ref refs/hyperloom/upstream "$VLLM_IMAGE_SOURCE_COMMIT" && + git -C "$root" update-ref refs/hyperloom/image-baseline "$baseline" && + git -C "$root" update-ref --no-deref HEAD "$baseline" || + { die "failed to pin vLLM image source HEAD"; return 1; } + git -C "$root" diff-index --quiet "$baseline" -- || + { die "vLLM image source verification changed after pinning"; return 1; } + export VLLM_IMAGE_UPSTREAM_SHA="$VLLM_IMAGE_SOURCE_COMMIT" + export VLLM_IMAGE_BASELINE_SHA="$baseline" +} + +copy_missing_vllm_wheel_artifacts() { + local root="$1" wheel_package="$2" + "$PYTHON" - "$root" "$wheel_package" <<'PY' +import os +import shutil +import subprocess +import sys +from pathlib import Path + +root, wheel = map(Path, sys.argv[1:]) +source = root / "vllm" +tracked = set(subprocess.check_output(["git", "-C", str(root), "ls-files"], text=True).splitlines()) +overlay = [] +for path in wheel.rglob("*"): + rel = path.relative_to(wheel) + native = path.name.endswith((".so", ".pyd", ".dll", ".dylib")) or ".so." in path.name + if path.name != "_version.py" and not native: + continue + destination = source / rel + relative = Path("vllm") / rel + if os.path.lexists(destination): + if relative.as_posix() not in tracked: + overlay.append(relative) + continue + destination.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + destination.symlink_to(os.readlink(path)) + elif path.is_file(): + shutil.copy2(path, destination) + else: + continue + overlay.append(relative) + +info = root / ".git" / "info" +info.mkdir(parents=True, exist_ok=True) +exclude = info / "exclude" +existing = set(exclude.read_text(encoding="utf-8").splitlines()) if exclude.exists() else set() +entries = [f"/{path.as_posix()}" for path in overlay] +if entries: + with exclude.open("a", encoding="utf-8") as stream: + for entry in entries: + if entry not in existing: + stream.write(entry + "\n") + (info / "hyperloom-wheel-overlay").write_text("".join(f"{path.as_posix()}\n" for path in overlay), encoding="utf-8") +print(len(overlay)) +PY +} + +verify_vllm_image_source_import() { + local root="$1" wheel_package="$2" + PYTHONPATH="${root}${PYTHONPATH:+:${PYTHONPATH}}" "$PYTHON" - "$root" "$wheel_package" <<'PY' +import importlib +import sys +from pathlib import Path + +root, wheel = map(Path, sys.argv[1:]) +import vllm + +loaded = Path(vllm.__file__).resolve() +expected = (root / "vllm").resolve() +if expected not in loaded.parents: + raise SystemExit(f"vLLM loaded from {loaded}, expected {expected}") +modules = [name for name in ("_C", "_rocm_C") if (wheel / f"{name}.so").exists() or list(wheel.glob(f"{name}*.so"))] +if not modules: + raise SystemExit("wheel exposes neither vllm._C nor vllm._rocm_C") +for name in modules: + importlib.import_module(f"vllm.{name}") +PY +} + +activate_vllm_image_source() { + local info runtime_version runtime_commit wheel_package + [ -d "$VLLM_IMAGE_SOURCE_ROOT/vllm" ] || return 0 + if ! info="$(probe_vllm_image_wheel 2>/dev/null)"; then + log "vLLM image source skipped: installed wheel has no commit-qualified version" + return 0 + fi + IFS=$'\t' read -r runtime_version runtime_commit wheel_package <<<"$info" + case "$VLLM_IMAGE_SOURCE_COMMIT" in + "$runtime_commit"*) ;; + *) log "vLLM image source skipped: runtime ${runtime_version} is not ${VLLM_IMAGE_SOURCE_COMMIT}"; return 0 ;; + esac + if [ "${DRY_RUN:-0}" -eq 1 ] || [ "${CHECK_ONLY:-0}" -eq 1 ]; then + log "would activate ${VLLM_IMAGE_SOURCE_ROOT} for runtime ${runtime_version}" + return 0 + fi + prepare_vllm_image_git_tree "$VLLM_IMAGE_SOURCE_ROOT" || return 1 + copy_missing_vllm_wheel_artifacts "$VLLM_IMAGE_SOURCE_ROOT" "$wheel_package" >/dev/null || + { die "failed to overlay vLLM wheel artifacts"; return 1; } + verify_vllm_image_source_import "$VLLM_IMAGE_SOURCE_ROOT" "$wheel_package" || + { die "vLLM image source import verification failed"; return 1; } + export FRAMEWORK_REPO_PATH="$VLLM_IMAGE_SOURCE_ROOT" + export VLLM_REPO_PATH="$VLLM_IMAGE_SOURCE_ROOT" + export VLLM_DIR="$VLLM_IMAGE_SOURCE_ROOT" + export HYPERLOOM_VLLM_IMAGE_SOURCE=1 + VLLM_IMAGE_SOURCE_ACTIVE=1 + log "activated exact vLLM image source at ${VLLM_IMAGE_SOURCE_ROOT}" +} + +persist_vllm_image_source_env() { + [ "$VLLM_IMAGE_SOURCE_ACTIVE" -eq 1 ] || return 0 + local pair name value + for pair in \ + "FRAMEWORK_REPO_PATH=${VLLM_IMAGE_SOURCE_ROOT}" \ + "VLLM_REPO_PATH=${VLLM_IMAGE_SOURCE_ROOT}" \ + "VLLM_DIR=${VLLM_IMAGE_SOURCE_ROOT}" \ + "HYPERLOOM_VLLM_IMAGE_SOURCE=1" \ + "VLLM_IMAGE_UPSTREAM_SHA=${VLLM_IMAGE_UPSTREAM_SHA}" \ + "VLLM_IMAGE_BASELINE_SHA=${VLLM_IMAGE_BASELINE_SHA}"; do + name="${pair%%=*}" + value="${pair#*=}" + if grep -q "^export ${name}=" "$KERNEL_AGENT_ENV" 2>/dev/null; then + sed -i "s|^export ${name}=.*|export ${name}='${value}'|" "$KERNEL_AGENT_ENV" + else + printf "export %s='%s'\n" "$name" "$value" >> "$KERNEL_AGENT_ENV" + fi + done +} + # Serialize concurrent installs that share one open-source checkout root # (Magpie / InferenceX, plus GEAK / TraceLens via the chained # kernel-agent installer). With no lock, two installs race and corrupt each @@ -2031,8 +2239,10 @@ if [ "$HYPERLOOM_BENCHMARK_BACKEND_LC" != "bypass" ]; then fi ensure_bench_serving_deps ensure_scriptable_quality_deps +activate_vllm_image_source ensure_framework_deps chain_kernel_agent +persist_vllm_image_source_env # rocprof-compute + pandas<3 pin runs LAST — strictly AFTER every pip-installing # step (chain_kernel_agent included; nothing below installs packages). This makes # the pandas<3 pin the final word (no later `pip install` can re-pull pandas>=3) diff --git a/src/hyperloom/inference_optimizer/assets/install_baremetal.sh b/src/hyperloom/inference_optimizer/assets/install_baremetal.sh index b4b997795b..9c39cacba3 100755 --- a/src/hyperloom/inference_optimizer/assets/install_baremetal.sh +++ b/src/hyperloom/inference_optimizer/assets/install_baremetal.sh @@ -55,9 +55,9 @@ SGLANG_REPO="${SGLANG_REPO:-https://github.com/sgl-project/sglang.git}" # below 2.11 and so cannot resolve against a ROCm 10 stack at all; 0.5.19 moved # that dependency into runtime_common unpinned, which leaves the installer's # ROCm torch constraint as the version pip solves for. -# vLLM installs 0.29.0+rocm723 from the wheels.vllm.ai pip index, matching the -# vllm/vllm-openai-rocm:v0.29.0 Docker image. The rocm723 variant puts the -# vLLM ROCm layer at 7.2.3, one patch level above the SGLang stack. AITER_REF +# vLLM installs 0.29.0+rocm723 from the wheels.vllm.ai pip index. The rocm723 +# variant puts the vLLM ROCm layer at 7.2.3; the docker route instead uses the +# ROCm 10.0 rocm/vllm image, so the two paths no longer share a patch level. AITER_REF # can pin ROCm/aiter to a released tag; when unset, the installer selects the # newest tag compatible with the already-installed ROCm torch/triton stack. SGLANG_REF="${SGLANG_REF:-94602c9c2b7cbdb8efd5c52802dac6a1c180089e}" @@ -75,6 +75,13 @@ AITER_REF="${AITER_REF:-}" VLLM_VERSION="${VLLM_VERSION:-0.29.0}" VLLM_ROCM_VARIANT="${VLLM_ROCM_VARIANT:-rocm723}" VLLM_ROCM_INDEX="${VLLM_ROCM_INDEX:-https://wheels.vllm.ai/rocm/${VLLM_VERSION}/${VLLM_ROCM_VARIANT}}" +VLLM_INSTALL_METHOD="${VLLM_INSTALL_METHOD:-auto}" +VLLM_REPO="${VLLM_REPO:-https://github.com/vllm-project/vllm.git}" +VLLM_SOURCE_REF="${VLLM_SOURCE_REF:-98dff2a81d747d1dba01a47f939f48c3526d4206}" +# The source checkout is a depth-1 fetch of a commit SHA and carries no tags, so +# setuptools_scm would otherwise stamp the build 0.1.dev1 instead of the release. +VLLM_PRETEND_VERSION="${VLLM_PRETEND_VERSION:-${VLLM_VERSION}}" +VLLM_ROOT="${VLLM_ROOT:-/opt/hyperloom/vllm}" # Index publishing the TheRock ROCm SDK wheels these images are built from; # rocm-sdk-devel is pulled from here to supply source-build headers. ROCM_SDK_INDEX_URL="${ROCM_SDK_INDEX_URL:-https://stable.repo.amd.com/rocm/whl-next}" @@ -133,7 +140,8 @@ PYTHON, INFERENCE_OPTIMIZER_FORCE_PYTHON, SGLANG_REPO, SGLANG_REF, SGLANG_ROOT, SGLANG_ROCM_PYPI_VERSION, SGLANG_ROCM_EXTRA, SGLANG_BUILD_RUST_EXTS, AITER_REPO, AITER_REF, AITER_ROOT, ROCM_PATH, HIP_PATH, LD_LIBRARY_PATH, ROCM_SDK_INDEX_URL, VLLM_VERSION, VLLM_ROCM_VARIANT, VLLM_ROCM_INDEX, -VLLM_VENV_ROOT, HYPERLOOM_WHEEL_REPO, HYPERLOOM_WHEEL_TAG. +VLLM_INSTALL_METHOD, VLLM_REPO, VLLM_SOURCE_REF, VLLM_ROOT, VLLM_VENV_ROOT, +HYPERLOOM_WHEEL_REPO, HYPERLOOM_WHEEL_TAG. EOF } @@ -185,7 +193,7 @@ die() { echo "[install-baremetal ERROR] $*" >&2; exit 1; } IMAGE_HINT="Provision the ROCm framework base first (SGLang: lmsysorg/sglang-rocm:v0.5.20-rocm10-mi30x|mi35x-*; \ vLLM bare-metal: Ubuntu 24.04+ host with ROCm torch, or use docker mode with \ -vllm/vllm-openai-rocm:v0.29.0), then re-run." +rocm/vllm:rocm10.0.0_ubuntu24.04_py3.14_pytorch_2.12.0_vllm_0.27.0), then re-run." is_interactive() { [ "$ASSUME_YES" -eq 0 ] && [ -t 0 ] && [ -t 1 ]; } @@ -742,6 +750,18 @@ install_aiter_ref_with_constraints() { check_torch_triton_alignment "$py" || return 1 } +# Both engines need AITER: SGLang routes GEMM through it, and kernelforge's +# fusion validation runs vLLM with VLLM_ROCM_USE_AITER=1. +ensure_aiter_for_python() { + local py="$1" aiter_root="$2" + if "$py" -c "import aiter" >/dev/null 2>&1; then + log "aiter already importable; skipping AITER source install" + return 0 + fi + ensure_rocm_devel_headers "$py" + install_compatible_aiter "$py" "$aiter_root" +} + install_compatible_aiter() { local py="$1" aiter_root="$2" constraint_file ref tried=0 constraint_file="$(mktemp)" @@ -906,12 +926,7 @@ PY log "sglang + sgl_kernel already importable; skipping amd-sglang install" fi - if ! "$py" -c "import aiter" >/dev/null 2>&1; then - ensure_rocm_devel_headers "$py" - install_compatible_aiter "$py" "$aiter_root" - else - log "aiter already importable; skipping AITER source install" - fi + ensure_aiter_for_python "$py" "$aiter_root" "$py" -c "import sglang" >/dev/null || die "sglang not importable after install" "$py" -c "import sgl_kernel" >/dev/null || die "sgl_kernel not importable after install" @@ -1030,9 +1045,253 @@ ensure_openmpi_runtime() { warn "could not install an OpenMPI runtime package (tried libopenmpi3t64, libopenmpi3); vLLM's torch import may fail on libmpi.so.40" } +vllm_install_method_for_stack() { + local release="$1" hip="$2" detected="" + case "${release}:${hip}" in + 10:7.15|10:7.15.*|10.*:7.15|10.*:7.15.*) detected=source ;; + 7.2:7.2|7.2:7.2.*|7.2.*:7.2|7.2.*:7.2.*) detected=wheel ;; + :7.2|:7.2.*) detected=wheel ;; + esac + if [ -z "$detected" ]; then + echo "unsupported or conflicting ROCm stack (release=${release:-unknown}, torch HIP=${hip:-unknown})" >&2 + return 1 + fi + case "$VLLM_INSTALL_METHOD" in + auto) ;; + source|wheel) + if [ "$VLLM_INSTALL_METHOD" != "$detected" ]; then + echo "VLLM_INSTALL_METHOD=${VLLM_INSTALL_METHOD} conflicts with detected ${detected} route" >&2 + return 1 + fi + ;; + *) echo "VLLM_INSTALL_METHOD must be one of: auto, source, wheel" >&2; return 1 ;; + esac + printf '%s\n' "$detected" +} + +detect_vllm_stack() { + local py="$1" + "$py" - <<'PY' +import glob, os, re, sys +from importlib import metadata + +releases = set() +try: + value = metadata.version("rocm-sdk-core") + match = re.search(r"(\d+)(?:\.(\d+))?", value) + if match: + releases.add(match.group(1) + (f".{match.group(2)}" if match.group(2) else "")) +except metadata.PackageNotFoundError: + pass +for root in filter(None, (os.environ.get("ROCM_PATH"), os.environ.get("ROCM_HOME"))): + for path in glob.glob(os.path.join(root, ".info", "version*")): + try: + match = re.search(r"(\d+)(?:\.(\d+))?", open(path).read()) + if match: + releases.add(match.group(1) + (f".{match.group(2)}" if match.group(2) else "")) + except OSError: + pass +if len(releases) > 1: + print(f"conflicting ROCm release evidence: {sorted(releases)}", file=sys.stderr) + raise SystemExit(1) +try: + import torch + hip = getattr(torch.version, "hip", None) or "" +except Exception: + hip = "" +print(f"{next(iter(releases), '')}|{hip}") +PY +} + +route_vllm_install_method() { + local stack release hip selected + stack="$(detect_vllm_stack "$1")" || die "failed to detect a consistent ROCm stack for vLLM" + release="${stack%%|*}"; hip="${stack#*|}" + selected="$(vllm_install_method_for_stack "$release" "$hip")" || die "cannot route vLLM install safely" + VLLM_INSTALL_METHOD="$selected" + export VLLM_INSTALL_METHOD + log "vLLM install route: ${selected} (release=${release:-none}, torch HIP=${hip:-unknown})" +} + +check_vllm_source_python() { + "$1" - <<'PY' || die "vLLM source install requires Python >=3.10,<3.15" +import sys +raise SystemExit(0 if (3, 10) <= sys.version_info < (3, 15) else 1) +PY +} + +check_vllm_source_prereqs() { + local tool current required arch + for tool in git gcc g++ cmake ninja hipcc; do + command -v "$tool" >/dev/null 2>&1 || die "${tool} is required for vLLM source install" + done + for tool in gcc g++ cmake; do + required=11.3 + [ "$tool" = cmake ] && required=3.26.1 + current="$("$tool" --version | grep -oE '[0-9]+(\.[0-9]+)+' | head -1)" + _version_ge "$current" "$required" || die "${tool} >=${required} is required (found ${current:-unknown})" + done + arch="${PYTORCH_ROCM_ARCH:-$(detect_rocm_gfx_arch)}" + [ -n "$arch" ] || die "cannot detect a gfx architecture for vLLM source install" +} + +vllm_overlay_is_valid() { + [ -x "$1/bin/python" ] && + grep -Eqi '^[[:space:]]*include-system-site-packages[[:space:]]*=[[:space:]]*true' "$1/pyvenv.cfg" +} + +inherit_vllm_base_site_packages() { + local base_py="$1" py="$2" base_site overlay_site amdsmi_site + base_site="$("$base_py" - <<'PY' +from pathlib import Path +import torch +print(Path(torch.__file__).resolve().parent.parent) +PY +)" || die "cannot resolve the base ROCm torch site-packages" + overlay_site="$("$py" - <<'PY' +import site +print(site.getsitepackages()[0]) +PY +)" || die "cannot resolve the vLLM overlay site-packages" + [ -d "$base_site" ] && [ -d "$overlay_site" ] || + die "base or overlay site-packages directory is missing" + amdsmi_site="${base_site}/_rocm_sdk_core/share/amd_smi" + { + printf '%s\n' "$base_site" + [ ! -d "$amdsmi_site" ] || printf '%s\n' "$amdsmi_site" + } > "${overlay_site}/hyperloom-base-venv.pth" +} + +vllm_checkout_state() { + local root="$1" origin head + [ -e "$root" ] || { echo new; return 0; } + [ -d "$root/.git" ] || die "vLLM source root is an existing non-git directory: ${root}" + origin="$(git -C "$root" remote get-url origin 2>/dev/null || true)" + [ "$origin" = "$VLLM_REPO" ] || die "vLLM checkout origin mismatch: ${origin:-missing}" + git -C "$root" diff --quiet && git -C "$root" diff --cached --quiet || + die "vLLM checkout has dirty tracked or index changes: ${root}" + head="$(git -C "$root" rev-parse HEAD 2>/dev/null || true)" + [ "$head" = "$VLLM_SOURCE_REF" ] && echo exact || echo update +} + +ensure_vllm_checkout() { + local root="$1" state + state="$(vllm_checkout_state "$root")" || return $? + [ "$state" = exact ] && { echo exact; return 0; } + check_vllm_source_prereqs + if [ "$state" = new ]; then + mkdir -p "$(dirname "$root")" + git init -q "$root" + git -C "$root" remote add origin "$VLLM_REPO" + fi + git -C "$root" fetch --quiet --depth 1 origin "$VLLM_SOURCE_REF" || + die "failed to fetch vLLM source ref ${VLLM_SOURCE_REF}" + git -C "$root" checkout --quiet --detach FETCH_HEAD + [ "$(git -C "$root" rev-parse HEAD)" = "$VLLM_SOURCE_REF" ] || + die "vLLM checkout did not reach ${VLLM_SOURCE_REF}" + echo updated +} + +verify_vllm_source() { + local base_py="$1" py="$2" base_info + base_info="$("$base_py" - <<'PY' +import os, torch +print("|".join((os.path.realpath(torch.__file__), torch.__version__, torch.version.hip or ""))) +PY +)" || return 1 + "$py" - "$base_info" <<'PY' || return 1 +import os, sys, torch +actual = "|".join((os.path.realpath(torch.__file__), torch.__version__, torch.version.hip or "")) +if actual != sys.argv[1]: + raise SystemExit(f"overlay torch differs from host torch: {actual} != {sys.argv[1]}") +import vllm._rocm_C # noqa: F401 +from vllm.platforms import current_platform +checker = getattr(current_platform, "is_rocm", None) +if not ((callable(checker) and checker()) or "rocm" in repr(current_platform).lower()): + raise SystemExit("vLLM source overlay did not select ROCm") +torch.empty(1, device="cuda") +PY + "$py" -m pip check +} + +install_vllm_from_source() { + local base_py="$1" py="${VLLM_VENV_ROOT}/bin/python" state arch constraint_file aiter_root + export VLLM_TARGET_DEVICE=rocm + [[ "$VLLM_SOURCE_REF" =~ ^[0-9a-fA-F]{40}$ ]] || die "VLLM_SOURCE_REF must be a full 40-character commit SHA" + check_vllm_source_python "$base_py" + if [ "$CHECK_ONLY" -eq 1 ]; then + if [ -x "$py" ] && "$py" -c "import vllm" >/dev/null 2>&1; then + verify_vllm_source "$base_py" "$py" || die "installed vLLM source overlay failed runtime verification" + else + warn "vLLM source overlay is not installed (check-only)" + fi + return 0 + fi + if [ "$DRY_RUN" -eq 1 ]; then + rocm_devel_headers_present "$base_py" || + log "would ensure ROCm devel headers before the vLLM source build" + log "would prepare ${VLLM_ROOT} at ${VLLM_SOURCE_REF} and build into ${VLLM_VENV_ROOT}" + return 0 + fi + if [ -e "$VLLM_VENV_ROOT" ] && ! vllm_overlay_is_valid "$VLLM_VENV_ROOT"; then + die "existing VLLM_VENV_ROOT is not a system-site-packages venv: ${VLLM_VENV_ROOT}" + fi + aiter_root="${AITER_ROOT:-$(framework_deps_root)/aiter}" + state="$(ensure_vllm_checkout "$VLLM_ROOT")" || return $? + if vllm_overlay_is_valid "$VLLM_VENV_ROOT"; then + inherit_vllm_base_site_packages "$base_py" "$py" + fi + if [ "$state" = exact ] && vllm_overlay_is_valid "$VLLM_VENV_ROOT" && + verify_vllm_source "$base_py" "$py"; then + export VLLM_ROOT FRAMEWORK_REPO_PATH="$VLLM_ROOT" + ensure_aiter_for_python "$py" "$aiter_root" + link_vllm_into_shared_bin "$base_py" "$py" + log "reusing verified vLLM source overlay" + return 0 + fi + [ "$state" != exact ] || check_vllm_source_prereqs + ensure_rocm_devel_headers "$base_py" + rocm_devel_headers_present "$base_py" || die "ROCm devel headers are required for vLLM source install" + arch="${PYTORCH_ROCM_ARCH:-$(detect_rocm_gfx_arch)}" + [ -n "$arch" ] || die "cannot detect a gfx architecture for vLLM source install" + if [ ! -e "$VLLM_VENV_ROOT" ]; then + "$base_py" -m venv --system-site-packages "$VLLM_VENV_ROOT" + fi + inherit_vllm_base_site_packages "$base_py" "$py" + [ -f "$VLLM_ROOT/requirements/rocm.txt" ] || die "vLLM requirements/rocm.txt is missing" + constraint_file="$(mktemp)" + write_rocm_torch_constraints "$base_py" "$constraint_file" + "$py" -m pip install --no-build-isolation --constraint "$constraint_file" \ + --extra-index-url "$ROCM_SDK_INDEX_URL" \ + -r "$VLLM_ROOT/requirements/rocm.txt" || + { rm -f "$constraint_file"; die "failed to install vLLM ROCm source requirements"; } + # setup.py develop leaves the dist name unknown, and vcs-versioning only consults the + # per-distribution variable when it knows that name, so the generic one is the one that lands. + (cd "$VLLM_ROOT" && VLLM_TARGET_DEVICE=rocm PYTORCH_ROCM_ARCH="$arch" \ + SETUPTOOLS_SCM_PRETEND_VERSION="$VLLM_PRETEND_VERSION" \ + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_VLLM="$VLLM_PRETEND_VERSION" \ + PIP_CONSTRAINT="$constraint_file" "$py" setup.py develop --no-deps) || + { rm -f "$constraint_file"; die "vLLM source build failed"; } + rm -f "$constraint_file" + verify_vllm_source "$base_py" "$py" || die "vLLM source install failed runtime verification" + export VLLM_ROOT FRAMEWORK_REPO_PATH="$VLLM_ROOT" + ensure_aiter_for_python "$py" "$aiter_root" + link_vllm_into_shared_bin "$base_py" "$py" + log "vLLM source install complete (${VLLM_ROOT})" +} + install_vllm_framework() { local py base_py py_mm constraint_file package_spec rocm_torch_ver base_py="$(resolve_python)" || die "no usable Python found for vLLM install" + route_vllm_install_method "$base_py" || return $? + if [ "$VLLM_INSTALL_METHOD" = source ]; then + if [ "$FRAMEWORK_ENV" != isolated ]; then + die "vLLM source install requires --framework-env isolated" + return 1 + fi + install_vllm_from_source "$base_py" + return $? + fi if [ "$CHECK_ONLY" -eq 0 ] && [ "$DRY_RUN" -eq 0 ]; then ensure_openmpi_runtime fi @@ -1131,7 +1390,7 @@ PY "$py" -m pip install --upgrade --extra-index-url "$VLLM_ROCM_INDEX" "torch==${rocm_torch_ver}" \ || die "failed to install ROCm torch==${rocm_torch_ver} into ${VLLM_VENV_ROOT}" else - warn "could not resolve a ROCm torch version from ${VLLM_ROCM_INDEX}/torch/; vLLM install will rely on its own torch pin" + die "could not resolve a ROCm torch version from ${VLLM_ROCM_INDEX}/torch/" fi if ! "$py" -m pip install --upgrade \ --extra-index-url "$VLLM_ROCM_INDEX" \ @@ -2108,6 +2367,8 @@ resolve_credentials() { unset LLM_GATEWAY_KEY if [ "$DUAL_PROTOCOL_GATEWAY" -eq 0 ]; then unset OPENAI_API_KEY OPENAI_BASE_URL OPENAI_CUSTOM_HEADERS + # Clear the resolved copies too, so the codex acceptance below reads the scrubbed state and not a stale local. + dual_url=""; dual_secret="" fi fi @@ -2146,11 +2407,19 @@ resolve_credentials() { { [ -n "$anthropic_url" ] || [ -n "$oauth_token" ]; } && has_url=1 { [ -n "$anthropic_key" ] || [ -n "$anthropic_token" ] || [ -n "$oauth_token" ]; } && has_key=1 - if [ "$has_url" -eq 0 ] || [ "$has_key" -eq 0 ]; then + # The codex backend drives the OpenAI side on its own, which ``llm_config.has_openai_side`` and the CLI's + # ``_provider_only_mode`` both already accept, so that pair is a complete credential here and not only a rider. + local has_openai_side=0 + if [ -n "$dual_url" ] && [ -n "$dual_secret" ]; then + has_openai_side=1 + export OPENAI_BASE_URL="$dual_url" + export OPENAI_API_KEY="$dual_secret" + fi + if { [ "$has_url" -eq 0 ] || [ "$has_key" -eq 0 ]; } && [ "$has_openai_side" -eq 0 ]; then if [ "$CHECK_ONLY" -eq 1 ] || [ "$DRY_RUN" -eq 1 ]; then warn "LLM credentials not fully resolved (continuing: --check-only / --dry-run)" else - die "no usable LLM endpoint: configure ANTHROPIC_BASE_URL + ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN, or a Claude subscription token (CLAUDE_CODE_OAUTH_TOKEN). A dual-protocol gateway such as DeepSeek also sets OPENAI_BASE_URL + OPENAI_API_KEY." + die "no usable LLM endpoint: for the claude backend configure ANTHROPIC_BASE_URL + ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN, or a Claude subscription token (CLAUDE_CODE_OAUTH_TOKEN); for the codex backend configure OPENAI_BASE_URL + OPENAI_API_KEY. A dual-protocol gateway serves both sides from one host and sets all four." fi fi @@ -2182,7 +2451,7 @@ resolve_credentials() { else remove_dotenv_var CLAUDE_CODE_OAUTH_TOKEN fi - if [ "$DUAL_PROTOCOL_GATEWAY" -eq 1 ]; then + if [ "$DUAL_PROTOCOL_GATEWAY" -eq 1 ] || [ "$has_openai_side" -eq 1 ]; then [ -n "${OPENAI_BASE_URL:-}" ] && upsert_dotenv_var OPENAI_BASE_URL "$OPENAI_BASE_URL" [ -n "${OPENAI_API_KEY:-}" ] && upsert_dotenv_var OPENAI_API_KEY "$OPENAI_API_KEY" [ -n "${CLAUDE_MODEL:-}" ] && upsert_dotenv_var CLAUDE_MODEL "$CLAUDE_MODEL" @@ -2211,6 +2480,25 @@ resolve_credentials() { fi } +# vLLM gates every aiter kernel behind VLLM_ROCM_USE_AITER and defaults it off, +# unlike the SGLang images that ship SGLANG_USE_AITER pre-set. +enable_vllm_aiter_when_available() { + local framework="$1" py + [ "$framework" = vllm ] || return 0 + if [ "$FRAMEWORK_ENV" = "isolated" ] && [ -x "${VLLM_VENV_ROOT}/bin/python" ]; then + py="${VLLM_VENV_ROOT}/bin/python" + else + py="$(resolve_python 2>/dev/null)" || return 0 + fi + # Turning the gate on without a matching aiter fails at serve time, so a + # missing module leaves vLLM on its built-in kernels instead. + if [ -z "$py" ] || ! "$py" -c "import aiter" >/dev/null 2>&1; then + warn "aiter is not importable for vLLM; leaving VLLM_ROCM_USE_AITER unset" + return 0 + fi + upsert_dotenv_var VLLM_ROCM_USE_AITER "${VLLM_ROCM_USE_AITER:-1}" +} + # Persist bare-metal runtime env to .env (single source of truth). PATH-class # values are NOT written here; preflight derives them from ROCM_PATH / # VIRTUAL_ENV / VLLM_VENV_ROOT at launch (_derive_runtime_paths). @@ -2242,10 +2530,16 @@ write_runtime_dotenv() { [ -n "${HYPERLOOM_WHEEL_TAG:-}" ] && upsert_dotenv_var HYPERLOOM_WHEEL_TAG "$HYPERLOOM_WHEEL_TAG" [ -n "${HYPERLOOM_SKILL_PATH:-}" ] && upsert_dotenv_var HYPERLOOM_SKILL_PATH "$HYPERLOOM_SKILL_PATH" [ -n "${SGLANG_USE_AITER:-}" ] && upsert_dotenv_var SGLANG_USE_AITER "$SGLANG_USE_AITER" + enable_vllm_aiter_when_available "$detected_framework" upsert_dotenv_var HYPERLOOM_FRAMEWORK_ENV "$FRAMEWORK_ENV" if [ "$FRAMEWORK_ENV" = "isolated" ] && [ "$INSTALL_FRAMEWORK" = "vllm" ]; then upsert_dotenv_var VLLM_VENV_ROOT "$VLLM_VENV_ROOT" upsert_dotenv_var VLLM_PYTHON "${VLLM_VENV_ROOT}/bin/python" + if [ "$VLLM_INSTALL_METHOD" = "source" ]; then + upsert_dotenv_var VLLM_ROOT "$VLLM_ROOT" + upsert_dotenv_var FRAMEWORK_REPO_PATH "$FRAMEWORK_REPO_PATH" + upsert_dotenv_var VLLM_TARGET_DEVICE "$VLLM_TARGET_DEVICE" + fi fi log "updated ${DOTENV} with bare-metal runtime env" } diff --git a/src/hyperloom/inference_optimizer/breakdown/recorder/recorder.py b/src/hyperloom/inference_optimizer/breakdown/recorder/recorder.py index 7f8dd57d0b..58bd2eec6b 100644 --- a/src/hyperloom/inference_optimizer/breakdown/recorder/recorder.py +++ b/src/hyperloom/inference_optimizer/breakdown/recorder/recorder.py @@ -306,6 +306,9 @@ def _stable_item_filename(self, section: str, key: str) -> str: prefix = f"{_slug(section)}__{self._producer}__{slug}" digest = hashlib.sha256(key.encode("utf-8", errors="replace")).hexdigest()[:8] filename = f"{prefix}-{digest}.json" + if len(filename.encode("utf-8")) > 180: + short_digest = hashlib.sha256(key.encode("utf-8", errors="replace")).hexdigest()[:16] + return f"{_slug(section)}__{self._producer}__id-{short_digest}.json" legacy = self._dir / f"{prefix}.json" if slug != key: if legacy.exists(): diff --git a/src/hyperloom/inference_optimizer/references/operations.md b/src/hyperloom/inference_optimizer/references/operations.md index 9294d8594b..17efabd521 100644 --- a/src/hyperloom/inference_optimizer/references/operations.md +++ b/src/hyperloom/inference_optimizer/references/operations.md @@ -96,6 +96,9 @@ if [ -f "$REPO_ROOT/.env" ]; then set -a; . "$REPO_ROOT/.env"; set +a; fi eval "$_dotenv_prev" unset _dotenv_prev . "${KERNEL_AGENT_ENV:-${USER_DATA_PATH:-/workspace/hyperloom}/runtime/kernel-agent.env.sh}" +# Launch is its own shell and inherits no PYTHON from the smoke test, and +# kernel-agent.env.sh does not set it; resolve it before it leads PATH. +export PYTHON="${PYTHON:-$(command -v python3)}" export PATH="$(dirname "$PYTHON"):/usr/local/bin:$PATH" export RUN_TAG="$(basename "$MODEL_PATH")-$(date +%Y%m%d_%H%M%S)" export RUN_DIR="${USER_DATA_PATH:-/workspace/hyperloom}/optimizer_runs" diff --git a/src/hyperloom/inference_optimizer/tests/test_baremetal_doc_version_consistency.py b/src/hyperloom/inference_optimizer/tests/test_baremetal_doc_version_consistency.py index 5125c2b322..63ed80054b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baremetal_doc_version_consistency.py +++ b/src/hyperloom/inference_optimizer/tests/test_baremetal_doc_version_consistency.py @@ -45,6 +45,11 @@ def test_baremetal_defaults_match_compat_doc(): "docs/compatibility.rst pip spec must be 'vllm==%s+%s'" % (vllm_version, vllm_variant) ) + vllm_source_ref = _default("VLLM_SOURCE_REF", sh) + assert vllm_source_ref[:12] in doc, ( + "docs/compatibility.rst must name the pinned vLLM source commit %s" % vllm_source_ref[:12] + ) + assert not sglang_ref.startswith("v"), "SGLANG_REF is expected to pin a commit SHA (see docs/compatibility.rst)" assert sglang_ref[:12] in doc, "docs/compatibility.rst must name the pinned SGLang commit %s" % sglang_ref[:12] diff --git a/src/hyperloom/inference_optimizer/tests/test_breakdown_event_base.py b/src/hyperloom/inference_optimizer/tests/test_breakdown_event_base.py index 18fc8a174f..a266c82b01 100644 --- a/src/hyperloom/inference_optimizer/tests/test_breakdown_event_base.py +++ b/src/hyperloom/inference_optimizer/tests/test_breakdown_event_base.py @@ -5,6 +5,7 @@ from __future__ import annotations +import hashlib import json import logging from pathlib import Path @@ -109,6 +110,24 @@ def test_a_sink_writes_the_event_id_into_both_the_key_and_the_payload(tmp_path): assert path.name.startswith("kernel_lane_run__orchestrator__kernel_agent-3-kernel-lane-att-7-") +def test_a_long_kernel_identity_uses_a_short_stable_filename(tmp_path): + kernel_id = "kernel-" + ("x" * 500) + key = rec.fragment_key(_EID, "lane", kernel_id) + + with session_scope(tmp_path): + path = rec.make_sink(_EID, producer="orchestrator").record( + "kernel_lane_run", + {"kernel_id": kernel_id}, + row_type="lane", + natural_ids=kernel_id, + ) + + assert path is not None + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] + assert path.name == f"kernel_lane_run__orchestrator__id-{digest}.json" + assert json.loads(path.read_text(encoding="utf-8"))["payload"]["kernel_id"] == kernel_id + + def test_a_sink_drops_a_payload_that_claims_a_different_event_and_says_so_loudly(tmp_path, caplog): # The core is meant to be ignorant of its event id, so a payload that names one is a leak rather than a value to # be trusted or silently overridden. diff --git a/src/hyperloom/inference_optimizer/tests/test_claude_backend_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_claude_backend_helpers_coverage_unit.py index 872e2933e1..5f9a680a8e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_claude_backend_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_claude_backend_helpers_coverage_unit.py @@ -11,6 +11,7 @@ from typing import Any +from hyperloom.common.llm_config import CLAUDE_GATEWAY_SIGNAL_KEYS, LEGACY_DEEPSEEK_ENV_KEYS from hyperloom.orchestrator.roles import ( ClaudeBackend, EMIT_INTENT_TOOL_NAME, @@ -102,13 +103,8 @@ def test_build_options_pins_gateway_env_and_ignores_global_settings(monkeypatch) def test_build_options_leaves_settings_sources_unset_without_gateway_env(monkeypatch) -> None: - for key in ( - "ANTHROPIC_BASE_URL", - "ANTHROPIC_API_KEY", - "ANTHROPIC_AUTH_TOKEN", - "ANTHROPIC_CUSTOM_HEADERS", - "LLM_GATEWAY_KEY", - ): + # Taken from the product's signal set so a newly recognised gateway variable cannot leak in from the shell. + for key in (*CLAUDE_GATEWAY_SIGNAL_KEYS, *LEGACY_DEEPSEEK_ENV_KEYS, "LLM_GATEWAY_KEY"): monkeypatch.delenv(key, raising=False) b = _backend(model="claude-opus-4-6") diff --git a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py index 3e4f4f5ccc..13d1a7c234 100644 --- a/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py +++ b/src/hyperloom/inference_optimizer/tests/test_close_phase_sequencer.py @@ -762,6 +762,35 @@ async def test_close_sequencer_preserves_failed_conc_sweep_reason(coord): assert coord.shared_state.stop_reason == "sweep_failed" +@pytest.mark.asyncio +async def test_close_sequencer_does_not_mark_budgeted_sweep_without_pairs_done(coord): + """A sweep that spent its budget without a comparable pair did not validate the sweep objective.""" + coord.shared_state.last_conc_sweep = { + "status": "skipped", + "was_skipped": True, + "budget_exhausted": True, + "skip_reason": "budget_exhausted_no_successful_pairs", + "summary": {"successful_pairs": 0}, + } + coord.shared_state.phase_history = [ + { + "to_phase": "CLOSE", + "reason": "sweep_done", + "evidence": { + "sweep_status": "skipped", + "sweep_was_skipped": True, + "sweep_skip_budget_exhausted": True, + "sweep_skip_reason": "budget_exhausted_no_successful_pairs", + }, + }, + ] + assert coord.shared_state.stop_reason == "" + + await coord._on_enter_close(from_phase="SWEEP") + + assert coord.shared_state.stop_reason == "sweep_failed" + + @pytest.mark.asyncio async def test_close_sequencer_does_not_overwrite_caller_set_stop_reason( coord, diff --git a/src/hyperloom/inference_optimizer/tests/test_common_provenance.py b/src/hyperloom/inference_optimizer/tests/test_common_provenance.py index 58c65da401..d175e6524a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_common_provenance.py +++ b/src/hyperloom/inference_optimizer/tests/test_common_provenance.py @@ -9,6 +9,8 @@ import json import sys +import pytest + from hyperloom.common.provenance import ( PROVENANCE_SOURCE, build_provenance, @@ -96,7 +98,13 @@ def test_missing_degrades_to_none_never_raises(): "max_model_len", ): assert p[key] is None - assert p["stack_fingerprint"] == {"rocm": "unknown", "aiter": "unknown", "sglang": "unknown", "vllm": "unknown"} + assert p["stack_fingerprint"] == { + "rocm": "unknown", + "aiter": "unknown", + "sglang": "unknown", + "vllm": "unknown", + "torch": "unknown", + } assert p["server_args"] == [] assert p["server_args_hash"] == "" assert p["code_revision"] == "" @@ -302,6 +310,20 @@ def test_a_framework_in_its_own_venv_is_still_versioned(tmp_path): assert fp["vllm"] == "0.27.1+rocm723" +def test_resolved_vllm_venv_versions_vllm_and_aiter(tmp_path): + venv_root = tmp_path / "vllm-venv" + python_exe = _installed(venv_root, "vllm", "0.29.0+rocm723") + info = venv_root / "lib" / "python3.12" / "site-packages" / "amd_aiter-0.1.19.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.4\nName: amd-aiter\nVersion: 0.1.19\n") + + fp = _prov.detect_stack_fingerprint( + {"HYPERLOOM_RESOLVED_FRAMEWORK": "vllm", "HYPERLOOM_RESOLVED_FRAMEWORK_PYTHON": python_exe}, probe=True + ) + assert fp["vllm"] == "0.29.0+rocm723" + assert fp["aiter"] == "0.1.19" + + def test_an_operator_pin_still_wins_over_the_resolved_interpreter(tmp_path): python_exe = _installed(tmp_path / "vllm-venv", "vllm", "0.27.1+rocm723") fp = _prov.detect_stack_fingerprint( @@ -361,6 +383,105 @@ def _version(dist: str) -> str: return _version +def test_therock_rocm_sdk_core_distribution_is_versioned(monkeypatch): + monkeypatch.setattr(_prov, "_read_first_line", lambda path: "") + monkeypatch.setattr(_prov._im, "version", _only_installed(**{"rocm-sdk-core": "10.0.0"})) + + assert _prov.detect_stack_fingerprint({}, probe=True)["rocm"] == "10.0.0" + + +def test_torch_is_part_of_the_stack_fingerprint(monkeypatch): + """The ROCm 10 legs differ in torch alone, and that difference decides whether the profiler records any GPU + kernel, so a fingerprint without it cannot tell the legs apart. + """ + monkeypatch.setattr(_prov, "_read_first_line", lambda path: "") + monkeypatch.setattr(_prov._im, "version", _only_installed(torch="2.13.0+rocm10.0.0")) + + assert _prov.detect_stack_fingerprint({}, probe=True)["torch"] == "2.13.0+rocm10.0.0" + + +def test_the_framework_venvs_torch_is_the_one_recorded(tmp_path): + """The isolated vLLM overlay pins its own torch and that is the build actually serving; recording the + orchestrator's names a build the run never executed on. + """ + venv_root = tmp_path / "vllm-venv" + python_exe = _installed(venv_root, "vllm", "0.29.0+rocm723") + info = venv_root / "lib" / "python3.12" / "site-packages" / "torch-2.12.0+git6bbd260.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.4\nName: torch\nVersion: 2.12.0+git6bbd260\n") + + fp = _prov.detect_stack_fingerprint( + {"HYPERLOOM_RESOLVED_FRAMEWORK": "vllm", "HYPERLOOM_RESOLVED_FRAMEWORK_PYTHON": python_exe}, probe=True + ) + assert fp["torch"] == "2.12.0+git6bbd260" + + +def _trace_with_head(path, head: str, *, events: int = 3) -> str: + """A gzipped chrome trace whose top-level metadata precedes ``traceEvents``, as PyTorch writes them.""" + import gzip as _gzip + + body = '{\n "schemaVersion": 1,\n' + head + ' "traceEvents": [\n' + body += ",\n".join(' {"cat": "kernel", "name": "k%d"}' % i for i in range(events)) + body += "\n ]\n}\n" + with _gzip.open(path, "wb") as fh: + fh.write(body.encode("utf-8")) + return str(path) + + +@pytest.mark.parametrize( + ("head", "expected"), + [ + pytest.param(' "rocprofiler-sdk_version": 1.3,\n', "rocprofiler-sdk 1.3", id="rocprofiler-sdk"), + pytest.param(' "roctracer_version": 4.1,\n', "roctracer 4.1", id="roctracer"), + pytest.param(' "hip_runtime_version": 71526333,\n', "", id="neither-backend"), + ], +) +def test_the_kineto_backend_is_read_from_the_trace_head(tmp_path, head, expected): + """The torch build selects this backend and the trace is the only place that states it outright.""" + trace = _trace_with_head(tmp_path / "r.trace.json.gz", head) + assert _prov.detect_kineto_backend(trace) == expected + + +def test_an_unreadable_trace_yields_no_backend(tmp_path): + """A backend we could not read must come back empty rather than as a guess, since this is recorded as fact.""" + broken = tmp_path / "broken.trace.json.gz" + broken.write_bytes(b"not gzip at all") + assert _prov.detect_kineto_backend(broken) == "" + assert _prov.detect_kineto_backend(tmp_path / "absent.trace.json.gz") == "" + + +def test_a_workspace_directory_resolves_to_the_trace_inside_it(tmp_path): + """Profile executors record the workspace directory, so reading the file directly finds nothing.""" + workspace = tmp_path / "torch_trace" + workspace.mkdir() + _trace_with_head(workspace / "r.trace.json.gz", ' "roctracer_version": 4.1,\n') + assert _prov.detect_kineto_backend(workspace) == "roctracer 4.1" + assert _prov.detect_kineto_backend(tmp_path / "empty") == "" + + +def test_explicit_versions_outrank_distribution_probes(monkeypatch, tmp_path): + venv_root = tmp_path / "vllm-venv" + python_exe = _installed(venv_root, "vllm", "0.29.0+rocm723") + info = venv_root / "lib" / "python3.12" / "site-packages" / "amd_aiter-0.1.19.dist-info" + info.mkdir() + (info / "METADATA").write_text("Metadata-Version: 2.4\nName: amd-aiter\nVersion: 0.1.19\n") + monkeypatch.setattr(_prov._im, "version", _only_installed(**{"rocm-sdk-core": "10.0.0"})) + + fp = _prov.detect_stack_fingerprint( + { + "HYPERLOOM_RESOLVED_FRAMEWORK": "vllm", + "HYPERLOOM_RESOLVED_FRAMEWORK_PYTHON": python_exe, + "ROCM_VERSION": "9.9.9", + "AITER_VERSION": "0.1.20", + "VLLM_VERSION": "0.30.0", + }, + probe=True, + ) + assert fp["rocm"] == "9.9.9" + assert fp["aiter"] == "0.1.20" + assert fp["vllm"] == "0.30.0" + + def test_the_aiter_ref_the_installer_resolved_reaches_the_fingerprint(monkeypatch): """``install_baremetal.sh`` exports ``AITER_REF`` and persists it to ``.env``; nothing wrote the other two.""" monkeypatch.setattr(_prov._im, "version", _only_installed()) diff --git a/src/hyperloom/inference_optimizer/tests/test_common_utils.py b/src/hyperloom/inference_optimizer/tests/test_common_utils.py index cf0ed3dfe3..101eef9426 100644 --- a/src/hyperloom/inference_optimizer/tests/test_common_utils.py +++ b/src/hyperloom/inference_optimizer/tests/test_common_utils.py @@ -226,6 +226,8 @@ def test_reset_claude_config_leaves_file_alone_for_oauth_only(tmp_path: Path, mo oauth_env = "_".join(("CLAUDE", "CODE", "OAUTH", "TOKEN")) monkeypatch.setenv(oauth_env, "sk-ant-oat01-fake") + monkeypatch.delenv("_".join(("ANTHROPIC", "API", "KEY")), raising=False) + monkeypatch.delenv("_".join(("ANTHROPIC", "AUTH", "TOKEN")), raising=False) monkeypatch.setattr(Path, "home", classmethod(lambda _cls: tmp_path)) # Pre-seeded so "left alone" is observable rather than indistinguishable from "was never going to be written". cfg_path = tmp_path / ".claude" / "config.json" @@ -264,6 +266,8 @@ def test_reset_claude_config_preserves_existing_file_for_oauth_only( oauth_env = "_".join(("CLAUDE", "CODE", "OAUTH", "TOKEN")) monkeypatch.setenv(oauth_env, "sk-ant-oat01-fake") + monkeypatch.delenv("_".join(("ANTHROPIC", "API", "KEY")), raising=False) + monkeypatch.delenv("_".join(("ANTHROPIC", "AUTH", "TOKEN")), raising=False) monkeypatch.setattr(Path, "home", classmethod(lambda _cls: tmp_path)) cfg_path = tmp_path / ".claude" / "config.json" cfg_path.parent.mkdir(parents=True) diff --git a/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py b/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py index 98b8f4a736..897db0ad6d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py +++ b/src/hyperloom/inference_optimizer/tests/test_conc_sweep.py @@ -1215,12 +1215,25 @@ def test_the_sweep_exit_evidence_separates_a_skip_from_a_spent_budget(): "was_skipped": True, "budget_exhausted": True, "skip_reason": "budget_exhausted_no_successful_pairs", + "summary": {"successful_pairs": 0}, } ) - _, spent = exit_normal_sweep(state) + spent_reason, spent = exit_normal_sweep(state) + assert spent_reason == "sweep_failed" assert spent["sweep_was_skipped"] is True assert spent["sweep_skip_budget_exhausted"] is True + state.record_conc_sweep( + { + "status": "skipped", + "was_skipped": True, + "summary": {"successful_pairs": 0}, + } + ) + no_pair_reason, no_pair = exit_normal_sweep(state) + assert no_pair_reason == "sweep_failed" + assert no_pair["sweep_status"] == "skipped" + def test_on_enter_sweep_drains_pending_keep_integrates(monkeypatch): """Bug #7: KERNEL→SWEEP must drain pending KEEP integrates before closeout.""" @@ -2169,6 +2182,58 @@ async def _fake_run_grid(*, grid: list[GridVariant], **kw): assert last["server_lifecycle"]["cleanup"] is True +def test_single_server_boot_only_is_measured_by_reuse_round( + session_dir: Path, + baseline_yaml: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Booting the persistent server is not a data point; the same CONC must still run as a client round.""" + state = _make_state(baseline_config_path=str(baseline_yaml)) + teardown_log: list[tuple] = [] + _patch_lifecycle_eligible(monkeypatch, teardown_log) + monkeypatch.setattr( + "hyperloom.orchestrator.actions.executors._ray_serving.maybe_serving_lease", lambda **_kwargs: None + ) + calls: list[dict[str, Any]] = [] + + async def _fake_run_grid(*, grid: list[GridVariant], **kw): + variant = grid[0] + calls.append( + { + "name": variant.name, + "server_lifecycle": kw.get("server_lifecycle"), + "server_already_ready": kw.get("server_already_ready"), + "base_extra_envs": kw.get("base_extra_envs") or {}, + } + ) + if kw.get("server_lifecycle") and kw.get("server_already_ready") is False: + booted = _fake_variant( + variant.name, + throughput=None, + envs=variant.extra_envs, + status="succeeded", + error=None, + ) + booted.note = "server_lifecycle_boot_only" + return [booted] + return [_fake_variant(variant.name, throughput=100.0, envs=variant.extra_envs)] + + monkeypatch.setattr("hyperloom.orchestrator.kernel.conc_sweep.run_grid", _fake_run_grid) + + payload = asyncio.run(run_conc_sweep(state, session_dir, concs=[32, 16])) + + assert [call["name"] for call in calls[:3]] == [ + "optimized_conc32", + "optimized_conc32", + "optimized_conc16", + ] + boot, measured_boot = calls[0], calls[1] + assert boot["server_already_ready"] is False + assert boot["base_extra_envs"] == {"MAGPIE_RUN_PHASE": "server"} + assert measured_boot["server_already_ready"] is True + assert payload["summary"]["successful_pairs"] == 2 + + def test_single_server_boot_retry_descend( session_dir: Path, baseline_yaml: Path, diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index f8be33ae89..f84393fabc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -1445,7 +1445,7 @@ async def test_advance_phase_escalation_transition(coord: Coordinator, monkeypat lambda *a, **k: ("FRAMEWORK_AGENT", "robustness_escalated", {"evidence": "llm_escalation"}), ) - async def _entered(*, from_phase, to_phase): + async def _entered(*, from_phase, to_phase, reason="", evidence=None): return None monkeypatch.setattr(coord.phase_machine, "_on_phase_entered", _entered) @@ -1463,7 +1463,7 @@ async def test_advance_phase_terminal_sets_stop_reason(coord: Coordinator, monke ps, "compute_next_phase", lambda *a, **k: (ps.PHASE_CLOSE, "target_reached", {"terminal": True}) ) - async def _entered(*, from_phase, to_phase): + async def _entered(*, from_phase, to_phase, reason="", evidence=None): return None monkeypatch.setattr(coord.phase_machine, "_on_phase_entered", _entered) @@ -1480,7 +1480,7 @@ async def test_advance_phase_hint_survives_arrival_at_its_consumer(coord: Coordi coord.shared_state.pending_escalate_hint = "skip_to_kernel" monkeypatch.setattr(ps, "compute_next_phase", lambda *a, **k: ("FRAMEWORK_AGENT", "prelude_done", {})) - async def _entered(*, from_phase, to_phase): + async def _entered(*, from_phase, to_phase, reason="", evidence=None): return None monkeypatch.setattr(coord.phase_machine, "_on_phase_entered", _entered) @@ -1498,7 +1498,7 @@ async def test_advance_phase_hint_discarded_when_not_headed_to_its_consumer(coor coord.shared_state.pending_escalate_hint = "skip_to_kernel" monkeypatch.setattr(ps, "compute_next_phase", lambda *a, **k: ("SWEEP", "some_other_reason", {})) - async def _entered(*, from_phase, to_phase): + async def _entered(*, from_phase, to_phase, reason="", evidence=None): return None monkeypatch.setattr(coord.phase_machine, "_on_phase_entered", _entered) @@ -1525,7 +1525,7 @@ async def test_advance_phase_hint_consumed_when_it_drove_the_transition(coord: C lambda *a, **k: ("KERNEL_AGENT", "skip_to_kernel", {"hint": "skip_to_kernel"}), ) - async def _entered(*, from_phase, to_phase): + async def _entered(*, from_phase, to_phase, reason="", evidence=None): return None monkeypatch.setattr(coord.phase_machine, "_on_phase_entered", _entered) diff --git a/src/hyperloom/inference_optimizer/tests/test_coverage_gap_units.py b/src/hyperloom/inference_optimizer/tests/test_coverage_gap_units.py index f579e52982..122117f831 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coverage_gap_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_coverage_gap_units.py @@ -88,9 +88,18 @@ def _boom(_tmp, _dst): def test_credentials_endpoint_resolution_and_geak_sync(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: from hyperloom.inference_optimizer.cli import credentials + # An inherited key on either side resolves that side's official URL, so both must be absent for "unconfigured". + for name in ( + "ANTHROPIC_BASE_URL", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", + "OPENAI_API_KEY", + ): + monkeypatch.delenv(name, raising=False) + # Each side resolves on its own; an unconfigured side stays empty. monkeypatch.setenv("OPENAI_BASE_URL", "https://open.example/v1") - monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) assert credentials._resolve_llm_endpoints() == ("", "https://open.example/v1") monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://anthropic.example") diff --git a/src/hyperloom/inference_optimizer/tests/test_delegate_denial_loop.py b/src/hyperloom/inference_optimizer/tests/test_delegate_denial_loop.py index 17d5917221..9a68b8d9c8 100644 --- a/src/hyperloom/inference_optimizer/tests/test_delegate_denial_loop.py +++ b/src/hyperloom/inference_optimizer/tests/test_delegate_denial_loop.py @@ -88,6 +88,28 @@ async def test_delegate_running_collision_denies_without_new_task(session_dir): await c.stop() +@pytest.mark.asyncio +async def test_delegate_source_patch_without_git_root_prunes_without_retry(session_dir): + c = _silent_coordinator(session_dir) + try: + c.shared_state.framework_repo_path = "" + await c._handle_delegate( + "orchestration", + _delegate( + action="specialist", + key="patch-no-root", + params={"domain": "serving_specialist"}, + ), + ) + + assert await c.tasks.by_state("queued") == [] + assert c.shared_state.pruned_families == ["source_patch"] + failures = [(row["action"], row["task_id"], row["error_class"]) for row in c.shared_state.last_action_failures] + assert failures == [("specialist", "patch-no-root", "no_git_framework_source_root")] + finally: + await c.stop() + + @pytest.mark.asyncio async def test_delegate_fallback_key_uses_tick_and_content_fingerprint(session_dir): c = _silent_coordinator(session_dir) diff --git a/src/hyperloom/inference_optimizer/tests/test_dispatcher_execution_shutdown.py b/src/hyperloom/inference_optimizer/tests/test_dispatcher_execution_shutdown.py index 19dd221ea0..e8ad6d10e4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_dispatcher_execution_shutdown.py +++ b/src/hyperloom/inference_optimizer/tests/test_dispatcher_execution_shutdown.py @@ -9,6 +9,7 @@ import inspect import json import sqlite3 +import sys import threading from contextlib import closing from concurrent.futures import CancelledError as FuturesCancelledError @@ -20,6 +21,7 @@ import pytest from hyperloom.orchestrator.actions.cancel_channel import current_cancel_scope +from hyperloom.orchestrator.actions.executors import _ray_serving as ray_serving from hyperloom.orchestrator.bus.message_bus import MessageBus from hyperloom.orchestrator.bus.resource_lock import ResourceLockManager, SqliteLeaseBackend from hyperloom.orchestrator.bus.storage.connection import SqliteConnection @@ -276,6 +278,51 @@ async def run(): dispatcher.db.close() +def test_forced_specialist_actor_kill_releases_capacity_and_lane(tmp_path, monkeypatch): + class RayError(Exception): + pass + + killed = [] + fake_ray = SimpleNamespace( + get=lambda ref, **_kwargs: ref, + kill=killed.append, + exceptions=SimpleNamespace(RayError=RayError), + ) + monkeypatch.setitem(sys.modules, "ray", fake_ray) + dispatcher = _dispatcher(tmp_path) + dispatcher.gpu_specialist_pool = SimpleNamespace(release=AsyncMock()) + actor = SimpleNamespace(stop=SimpleNamespace(remote=lambda: False)) + specialist_lease = ray_serving.GpuSpecialistLease(num_gpus=1) + specialist_lease._actor = actor + specialist_lease._start_ref = object() + gpu_lease = object() + + async def run(): + dispatcher.sub.register_executor("shutdown_test", AsyncMock(return_value={"status": "ok"})) + task = await dispatcher.tasks.create( + kind="shutdown_test", params={}, idempotency_key="forced-actor-kill", requires_lanes=["research_lane"] + ) + result = await dispatcher.run_task_registered( + task, + gpu_specialist_lease=specialist_lease, + gpu_lease=gpu_lease, + ) + assert result.state == "succeeded" + assert killed == [actor] + assert specialist_lease._actor is None + assert specialist_lease._start_ref is None + dispatcher.gpu_specialist_pool.release.assert_awaited_once_with(gpu_lease) + assert await dispatcher.locks.lane_holders() == {} + assert (await dispatcher.tasks.get(task.task_id)).state == "succeeded" + assert dispatcher._executions == set() + assert dispatcher._inflight_actions == {} + + try: + asyncio.run(run()) + finally: + dispatcher.db.close() + + @pytest.mark.parametrize("cleanup", ["false", "raises"]) @pytest.mark.parametrize("outcome", ["succeeded", "failed", "cancelled"]) def test_cleanup_unconfirmed_preserves_outcome_without_completion(tmp_path, cleanup, outcome): diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index ffe4288f1e..f47d23712a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -21,6 +21,7 @@ canonical_fingerprint, ) from hyperloom.orchestrator.actions.executors._grid_runner import ( + GridVariant, apply_compatibility_filter, ) from hyperloom.orchestrator.actions.executors._subprocess_kill import ( @@ -2197,6 +2198,187 @@ async def test_explore_executor_sglang_empty_grid_still_fails_with_empty_grid( assert res.result["error_class"] == "empty_grid" +@pytest.mark.asyncio +async def test_explore_rejects_unsafe_aiter_unified_attn_before_benchmark( + sub_agent_runner, + tmp_path, +): + sub, tr, _ = sub_agent_runner + model = tmp_path / "Qwen3-14B-FP8" + model.mkdir() + (model / "config.json").write_text( + json.dumps( + { + "architectures": ["Qwen3ForCausalLM"], + "model_type": "qwen3", + "torch_dtype": "bfloat16", + "head_dim": 128, + "num_attention_heads": 40, + "num_key_value_heads": 8, + "quantization_config": {"quant_method": "fp8"}, + } + ), + encoding="utf-8", + ) + base = tmp_path / "base_sglang.yaml" + _write_baseline_yaml(base) + config = yaml.safe_load(base.read_text(encoding="utf-8")) + config["benchmark"]["model"] = str(model) + config["benchmark"]["precision"] = "fp8" + config["benchmark"]["envs"]["EXTRA_SGLANG_ARGS"] = "--page-size 1 --kv-cache-dtype auto" + base.write_text(yaml.safe_dump(config), encoding="utf-8") + + state = SharedState() + state.model_path = str(model) + state.model_name = "Qwen3-14B-FP8" + state.model_type = "qwen3" + state.gpu_type = "mi355x" + state.baseline_double_run = False + state.stack_fingerprint_meta = { + "sglang": "0.5.20.dev20260920+gc610c40399", + "aiter": "4ad99832823dde2315b361cbd3b54b1c5c12acd5", + "rocm": "10.0.0", + } + sub.shared_state = state + + output_dir = tmp_path / "explore-unified-attn-filter" + benchmarked: list[str] = [] + + def _fake_run(cmd, *args, **kwargs): + slot = Path(cmd[cmd.index("--output-dir") + 1]) + benchmarked.append(slot.relative_to(output_dir).as_posix()) + _fake_workspace(slot, tput=800.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(output_dir), + "base_tput": 800.0, + "grid": [ + { + "name": "unified", + "extra_envs": { + "SGLANG_USE_AITER": "1", + "SGLANG_USE_AITER_UNIFIED_ATTN": "1", + }, + }, + { + "name": "unified_combo", + "extra_args": "--chunked-prefill-size 32768", + "extra_envs": { + "SGLANG_USE_AITER": "1", + "SGLANG_USE_AITER_UNIFIED_ATTN": "1", + "SGLANG_USE_AITER_FP8_PER_TOKEN": "1", + }, + }, + {"name": "master", "extra_envs": {"SGLANG_USE_AITER": "1"}}, + { + "name": "per_token", + "extra_envs": { + "SGLANG_USE_AITER": "1", + "SGLANG_USE_AITER_FP8_PER_TOKEN": "1", + }, + }, + { + "name": "unified_page16", + "extra_args": "--page-size 16", + "extra_envs": { + "SGLANG_USE_AITER": "1", + "SGLANG_USE_AITER_UNIFIED_ATTN": "1", + }, + }, + ], + }, + idempotency_key="ex-unified-attn-filter", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + result = await sub.run_task(task) + + assert benchmarked == [ + "v00_master/variant_00_master", + "v01_per_token/variant_00_per_token", + "v02_unified_page16/variant_00_unified_page16", + ] + assert result.result["skipped_dup"] == [ + { + "name": "unified", + "reason": "compatibility_filter", + "detail": "SGLANG_USE_AITER_UNIFIED_ATTN=1 is unsafe on the exact ROCm 10 Qwen3-14B-FP8 stack", + }, + { + "name": "unified_combo", + "reason": "compatibility_filter", + "detail": "SGLANG_USE_AITER_UNIFIED_ATTN=1 is unsafe on the exact ROCm 10 Qwen3-14B-FP8 stack", + }, + ] + other_stack = dict(state.stack_fingerprint_meta) + other_stack["aiter"] = "newer-aiter" + other_variant = GridVariant( + "other_stack_unified", + extra_envs={"SGLANG_USE_AITER_UNIFIED_ATTN": "1"}, + ) + kept, dropped = apply_compatibility_filter( + [other_variant], + framework="sglang", + model_path=str(model), + gpu_type="mi355x", + stack_fingerprint=other_stack, + base_server_args="--page-size 1 --kv-cache-dtype auto", + ) + assert ([variant.name for variant in kept], dropped) == (["other_stack_unified"], []) + + +def test_unified_attn_filter_matches_aiter_dist_version_short_sha(tmp_path): + """The aiter fingerprint degrades to a git-describe dist version when ``AITER_COMMIT`` is unset; the same source + tree must still be recognised, otherwise the filter fails open and the unsafe lever reaches the benchmark. + """ + model = tmp_path / "Qwen3-14B-FP8" + model.mkdir() + (model / "config.json").write_text( + json.dumps( + { + "architectures": ["Qwen3ForCausalLM"], + "model_type": "qwen3", + "torch_dtype": "bfloat16", + "head_dim": 128, + "num_attention_heads": 40, + "num_key_value_heads": 8, + "quantization_config": {"quant_method": "fp8"}, + } + ), + encoding="utf-8", + ) + kept, dropped = apply_compatibility_filter( + [GridVariant("unified", extra_envs={"SGLANG_USE_AITER_UNIFIED_ATTN": "1"})], + framework="sglang", + model_path=str(model), + gpu_type="mi355x", + stack_fingerprint={ + "sglang": "0.5.20.dev20260920+gc610c40399", + # The dist version install_baremetal.sh falls back to; embeds the short sha of the pinned commit. + "aiter": "0.1.21.dev48+g4ad998328.d20260920", + "rocm": "10.0.0", + }, + base_server_args="--page-size 1 --kv-cache-dtype auto", + ) + assert ([variant.name for variant in kept], dropped) == ( + [], + [ + { + "name": "unified", + "source": "compatibility_filter", + "reason": "SGLANG_USE_AITER_UNIFIED_ATTN=1 is unsafe on the exact ROCm 10 Qwen3-14B-FP8 stack", + } + ], + ) + + def test_atom_default_grid_survives_compatibility_filter_without_help_probe( monkeypatch, ): diff --git a/src/hyperloom/inference_optimizer/tests/test_install_baremetal_vllm_target.py b/src/hyperloom/inference_optimizer/tests/test_install_baremetal_vllm_target.py new file mode 100644 index 0000000000..868cb3cce8 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_install_baremetal_vllm_target.py @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Behavior tests for bare-metal vLLM source and wheel routing.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +_INSTALL_SH = Path(__file__).resolve().parents[1] / "assets" / "install_baremetal.sh" +_REF = "98dff2a81d747d1dba01a47f939f48c3526d4206" + + +def _functions(*names: str) -> str: + text = _INSTALL_SH.read_text() + chunks = [] + for name in names: + result = subprocess.run( + ["sed", "-n", f"/^{name}()/,/^}}/p", str(_INSTALL_SH)], + check=True, + capture_output=True, + text=True, + ).stdout + assert result.strip(), f"{name}() not found in install_baremetal.sh" + chunks.append(result) + return "\n".join(chunks) + + +def _bash(functions: tuple[str, ...], body: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + script = f"set -uo pipefail\n{_functions(*functions)}\n{body}\n" + return subprocess.run(["bash", "-c", script], check=False, capture_output=True, text=True, env=env) + + +@pytest.mark.parametrize( + ("release", "hip", "configured", "expected", "ok"), + [ + ("10.0", "7.15.0", "auto", "source", True), + ("7.2", "7.2.4", "auto", "wheel", True), + ("", "7.2", "auto", "wheel", True), + ("", "7.2.4", "wheel", "wheel", True), + ("10.0", "7.15.0", "wheel", "", False), + ("", "7.15.0", "auto", "", False), + ("10.0", "7.2.4", "auto", "", False), + ("", "", "auto", "", False), + ], +) +def test_route_matrix(release: str, hip: str, configured: str, expected: str, ok: bool) -> None: + result = _bash( + ("vllm_install_method_for_stack",), + f'VLLM_INSTALL_METHOD="{configured}"; vllm_install_method_for_stack "{release}" "{hip}"', + ) + assert (result.returncode == 0) is ok + assert result.stdout.strip() == expected + + +def test_unknown_stack_fails_before_mutation(tmp_path: Path) -> None: + fake_py = tmp_path / "python" + fake_py.write_text("#!/bin/sh\ncat >/dev/null; echo 3.12\n") + fake_py.chmod(0o755) + body = f""" +log() {{ :; }}; warn() {{ :; }}; die() {{ echo "$*" >&2; return 1; }} +resolve_python() {{ echo "{fake_py}"; }} +route_vllm_install_method() {{ die "unsupported stack"; }} +ensure_openmpi_runtime() {{ echo MUTATION; }} +FRAMEWORK_ENV=isolated; VLLM_VENV_ROOT="{tmp_path}/venv"; CHECK_ONLY=0; DRY_RUN=0 +VLLM_VERSION=0.29.0; VLLM_ROCM_VARIANT=rocm723; VLLM_ROCM_INDEX=https://example.invalid +install_vllm_framework +""" + result = _bash(("install_vllm_framework",), body) + assert result.returncode != 0 + assert "MUTATION" not in result.stdout + + +@pytest.mark.parametrize("mode", ["dry", "check"]) +def test_source_non_mutating_modes(tmp_path: Path, mode: str) -> None: + dry, check = (1, 0) if mode == "dry" else (0, 1) + body = f""" +log() {{ echo "$*"; }}; warn() {{ echo "$*" >&2; }}; die() {{ echo "$*" >&2; return 1; }} +rocm_devel_headers_present() {{ return 1; }} +ensure_rocm_devel_headers() {{ echo MUTATION; }} +check_vllm_source_prereqs() {{ echo MUTATION; }} +ensure_vllm_checkout() {{ echo MUTATION; }} +verify_vllm_source() {{ return 1; }} +DRY_RUN={dry}; CHECK_ONLY={check}; VLLM_SOURCE_REF={_REF} +VLLM_ROOT="{tmp_path}/src"; VLLM_VENV_ROOT="{tmp_path}/venv" +install_vllm_from_source /usr/bin/python3 +""" + result = _bash(("install_vllm_from_source",), body) + assert result.returncode == 0, result.stderr + assert "MUTATION" not in result.stdout + assert ("would ensure ROCm devel headers" if mode == "dry" else "not installed") in (result.stdout + result.stderr) + + +@pytest.mark.parametrize("case", ["dirty", "origin"]) +def test_checkout_rejects_unsafe_existing_tree(tmp_path: Path, case: str) -> None: + root = tmp_path / "vllm" + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "config", "user.email", "test@example.com"], check=True) + subprocess.run(["git", "-C", str(root), "config", "user.name", "Test"], check=True) + (root / "tracked").write_text("clean") + subprocess.run(["git", "-C", str(root), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(root), "commit", "-qm", "base"], check=True) + repo = "https://example.invalid/vllm.git" + subprocess.run(["git", "-C", str(root), "remote", "add", "origin", repo], check=True) + if case == "dirty": + (root / "tracked").write_text("dirty") + else: + repo = "https://other.invalid/vllm.git" + body = f""" +die() {{ echo "$*" >&2; exit 1; }} +VLLM_REPO="{repo}"; VLLM_SOURCE_REF={_REF} +ensure_vllm_checkout "{root}" +""" + result = _bash(("vllm_checkout_state", "ensure_vllm_checkout"), body) + assert result.returncode != 0 + assert case in result.stderr.lower() + + +def _source_stubs(tmp_path: Path, checkout: str) -> str: + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + fake_py = venv / "bin" / "python" + fake_py.write_text( + '#!/bin/sh\necho "PY:$* CONSTRAINT=${PIP_CONSTRAINT:-}' + " TARGET=${VLLM_TARGET_DEVICE:-}" + " SCM=${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_VLLM:-}" + ' SCMGEN=${SETUPTOOLS_SCM_PRETEND_VERSION:-}" >> "$CALLS"\n' + ) + fake_py.chmod(0o755) + (venv / "pyvenv.cfg").write_text("include-system-site-packages = true\n") + return f""" +log() {{ echo "$*"; }}; warn() {{ echo "$*" >&2; }}; die() {{ echo "$*" >&2; return 1; }} +ensure_vllm_checkout() {{ echo {checkout}; }} +vllm_overlay_is_valid() {{ return 0; }} +verify_vllm_source() {{ return 0; }} +check_vllm_source_prereqs() {{ echo PREREQS >> "$CALLS"; }} +check_vllm_source_python() {{ :; }} +rocm_devel_headers_present() {{ return 0; }} +ensure_rocm_devel_headers() {{ echo HEADERS >> "$CALLS"; }} +detect_rocm_gfx_arch() {{ echo gfx950; }} +write_rocm_torch_constraints() {{ echo torch==2.12.0 > "$2"; echo triton==3.6.0 >> "$2"; }} +inherit_vllm_base_site_packages() {{ echo BASE_SITE >> "$CALLS"; }} +link_vllm_into_shared_bin() {{ echo LINK >> "$CALLS"; }} +framework_deps_root() {{ echo "{tmp_path}/deps"; }} +ensure_aiter_for_python() {{ echo "AITER:$2" >> "$CALLS"; }} +export CALLS="{tmp_path}/calls"; DRY_RUN=0; CHECK_ONLY=0; VLLM_SOURCE_REF={_REF} +VLLM_ROOT="{tmp_path}/src"; VLLM_VENV_ROOT="{venv}"; VLLM_REPO=https://example.invalid/vllm.git +ROCM_SDK_INDEX_URL=https://stable.repo.amd.com/rocm/whl-next +VLLM_PRETEND_VERSION=0.29.0 +mkdir -p "$VLLM_ROOT/requirements"; : > "$VLLM_ROOT/requirements/rocm.txt" +""" + + +def test_source_exact_checkout_fast_path_skips_build(tmp_path: Path) -> None: + result = _bash( + ("install_vllm_from_source",), + _source_stubs(tmp_path, "exact") + f'\ninstall_vllm_from_source "{tmp_path}/venv/bin/python"\n', + env={**os.environ}, + ) + assert result.returncode == 0, result.stderr + assert (tmp_path / "calls").read_text() == f"BASE_SITE\nAITER:{tmp_path}/deps/aiter\nLINK\n" + + +def test_source_build_preserves_constraint_for_develop(tmp_path: Path) -> None: + result = _bash( + ("install_vllm_from_source",), + _source_stubs(tmp_path, "updated") + f'\ninstall_vllm_from_source "{tmp_path}/venv/bin/python"\n', + env={**os.environ}, + ) + assert result.returncode == 0, result.stderr + calls = (tmp_path / "calls").read_text() + assert calls.index("BASE_SITE") < calls.index("pip install") + assert "pip install --no-build-isolation --constraint" in calls + assert "--extra-index-url https://stable.repo.amd.com/rocm/whl-next" in calls + assert "setup.py develop --no-deps CONSTRAINT=/tmp/" in calls + assert "TARGET=rocm" in calls + # A depth-1 checkout has no tags, so the build must be told its version or + # setuptools_scm stamps it 0.1.dev1. setup.py develop leaves the dist name + # unknown, so only the generic variable is consulted. + assert "SCM=0.29.0" in calls + assert "SCMGEN=0.29.0" in calls + assert f"AITER:{tmp_path}/deps/aiter" in calls + + +def test_source_route_requires_isolated_framework_env(tmp_path: Path) -> None: + fake_py = tmp_path / "python" + fake_py.write_text("#!/bin/sh\nexit 0\n") + fake_py.chmod(0o755) + body = f""" +die() {{ echo "$*" >&2; return 1; }} +resolve_python() {{ echo "{fake_py}"; }} +route_vllm_install_method() {{ VLLM_INSTALL_METHOD=source; }} +install_vllm_from_source() {{ echo MUTATION; }} +FRAMEWORK_ENV=shared +install_vllm_framework +""" + result = _bash(("install_vllm_framework",), body) + assert result.returncode != 0 + assert result.stdout == "" + assert "requires --framework-env isolated" in result.stderr + + +def test_overlay_inherits_sdk_amdsmi_path(tmp_path: Path) -> None: + base_site = tmp_path / "base-site" + overlay_site = tmp_path / "overlay-site" + amdsmi = base_site / "_rocm_sdk_core" / "share" / "amd_smi" + amdsmi.mkdir(parents=True) + overlay_site.mkdir() + base_py = tmp_path / "base-python" + overlay_py = tmp_path / "overlay-python" + base_py.write_text(f"#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' '{base_site}'\n") + overlay_py.write_text(f"#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' '{overlay_site}'\n") + base_py.chmod(0o755) + overlay_py.chmod(0o755) + body = f""" +die() {{ echo "$*" >&2; exit 1; }} +inherit_vllm_base_site_packages "{base_py}" "{overlay_py}" +""" + result = _bash(("inherit_vllm_base_site_packages",), body) + assert result.returncode == 0, result.stderr + inherited = (overlay_site / "hyperloom-base-venv.pth").read_text().splitlines() + assert inherited == [str(base_site), str(amdsmi)] + + +def test_source_roots_are_persisted() -> None: + body = """ +resolve_installed_framework() { echo vllm; }; log() { :; }; warn() { :; } +upsert_dotenv_var() { echo "$1=$2"; }; remove_dotenv_var() { :; } +DRY_RUN=0; CHECK_ONLY=0; DOTENV=/tmp/.env; USER_DATA_PATH=/data; FRAMEWORK_ENV=isolated; INSTALL_FRAMEWORK=vllm +VLLM_INSTALL_METHOD=source; VLLM_ROOT=/src/vllm; FRAMEWORK_REPO_PATH=/src/vllm; VLLM_VENV_ROOT=/venv +VLLM_TARGET_DEVICE=rocm +PYTHON=; INFERENCE_OPTIMIZER_FORCE_PYTHON=; VIRTUAL_ENV=; ROCM_PATH=; HIP_PATH= +SGLANG_ROCM_EXTRA=; SGLANG_ROCM_PYPI_VERSION=; AITER_REF=; KERNEL_OPT_BACKEND_ORDER= +HYPERLOOM_WHEEL_REPO=; HYPERLOOM_WHEEL_TAG=; HYPERLOOM_SKILL_PATH=; SGLANG_USE_AITER= +write_runtime_dotenv +""" + result = _bash(("write_runtime_dotenv",), body) + assert "VLLM_ROOT=/src/vllm" in result.stdout + assert "FRAMEWORK_REPO_PATH=/src/vllm" in result.stdout + assert "VLLM_TARGET_DEVICE=rocm" in result.stdout + + +def _aiter_gate_body(tmp_path: Path, framework: str, has_aiter: bool) -> str: + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + py = venv / "bin" / "python" + py.write_text(f"#!/bin/sh\nexit {0 if has_aiter else 1}\n") + py.chmod(0o755) + return f""" +warn() {{ echo "$*" >&2; }} +upsert_dotenv_var() {{ echo "$1=$2"; }} +resolve_python() {{ echo "{py}"; }} +FRAMEWORK_ENV=isolated; VLLM_VENV_ROOT="{venv}" +enable_vllm_aiter_when_available "{framework}" +""" + + +@pytest.mark.parametrize( + ("framework", "has_aiter", "expected"), + [ + ("vllm", True, "VLLM_ROCM_USE_AITER=1"), + ("vllm", False, ""), + ("sglang", True, ""), + ], +) +def test_vllm_aiter_gate(tmp_path: Path, framework: str, has_aiter: bool, expected: str) -> None: + result = _bash( + ("enable_vllm_aiter_when_available",), + _aiter_gate_body(tmp_path, framework, has_aiter), + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + if framework == "vllm" and not has_aiter: + assert "aiter is not importable" in result.stderr + + +def test_wheel_torch_index_resolution_failure_is_fatal(tmp_path: Path) -> None: + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + py = venv / "bin" / "python" + py.write_text('#!/bin/sh\nif [ "$1" = "-" ]; then cat >/dev/null; [ "$#" -eq 1 ] && echo 3.12; fi\n') + py.chmod(0o755) + body = f""" +log() {{ :; }}; warn() {{ :; }}; die() {{ echo "$*" >&2; exit 1; }} +resolve_python() {{ echo "{py}"; }}; route_vllm_install_method() {{ VLLM_INSTALL_METHOD=wheel; }} +ensure_openmpi_runtime() {{ :; }}; assert_vllm_glibc_compatible() {{ :; }} +verify_vllm_rocm() {{ return 0; }}; link_vllm_into_shared_bin() {{ :; }} +FRAMEWORK_ENV=isolated; VLLM_VENV_ROOT="{venv}"; CHECK_ONLY=0; DRY_RUN=0 +VLLM_VERSION=0.29.0; VLLM_ROCM_VARIANT=rocm723; VLLM_ROCM_INDEX=https://example.invalid +install_vllm_framework +""" + result = _bash(("install_vllm_framework",), body) + assert result.returncode != 0 + assert "resolve" in result.stderr.lower() diff --git a/src/hyperloom/inference_optimizer/tests/test_kernel_attempt_summary.py b/src/hyperloom/inference_optimizer/tests/test_kernel_attempt_summary.py index 59d8cdd9f6..f7e5e66d34 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kernel_attempt_summary.py +++ b/src/hyperloom/inference_optimizer/tests/test_kernel_attempt_summary.py @@ -668,3 +668,81 @@ def test_failure_breakdown_classifies_by_error_class(tmp_path: Path) -> None: assert breakdown.get("preprocess_failed") == 1, breakdown assert breakdown.get("timeout") == 1, breakdown assert breakdown.get("other", 0) == 0, f"new buckets should absorb root causes, leaving other empty: {breakdown}" + + +def test_geak_accepted_kernel_is_counted_as_e2e_success(tmp_path: Path) -> None: + state = _make_state(top15=[]) + state.geak_result = { + "status": "ok", + "accepted_kernels": [{"kernel_name": "rms_norm", "e2e_delta_pct": 3.05}], + } + + out = build_kernel_optimization_summary(state, tmp_path) + + assert out["schema_version"] == 2 + assert out["totals"]["attempted"] == 1 + assert out["lane_totals"]["geak"] == { + "attempted": 1, + "success": 1, + "unvalidated": 0, + "failed": 0, + "outcome": "success", + } + + +def test_forge_micro_winners_without_e2e_are_unvalidated(tmp_path: Path) -> None: + state = _make_state(top15=[]) + state.gemm_tuning_attempts = [ + { + "status": "ok", + "engine": "forge", + "requires_e2e_validation": True, + "tuners_run": [ + { + "tuner": f"tuner-{index}", + "kept": index < 22, + "best_micro_speedup": 1.05 if index < 22 else 1.0, + } + for index in range(23) + ], + } + ] + + out = build_kernel_optimization_summary(state, tmp_path) + + assert out["lane_totals"]["gemm_tuning"] == { + "attempted": 23, + "success": 0, + "unvalidated": 22, + "failed": 1, + "outcome": "unvalidated", + } + assert out["kernel_opt_outcome"] == "unvalidated" + + +def test_only_never_run_lanes_are_skipped(tmp_path: Path) -> None: + out = build_kernel_optimization_summary(_make_state(top15=[]), tmp_path) + + assert out["lane_totals"] == { + "source_level": { + "attempted": 0, + "success": 0, + "unvalidated": 0, + "failed": 0, + "outcome": "skip", + }, + "geak": { + "attempted": 0, + "success": 0, + "unvalidated": 0, + "failed": 0, + "outcome": "skip", + }, + "gemm_tuning": { + "attempted": 0, + "success": 0, + "unvalidated": 0, + "failed": 0, + "outcome": "skip", + }, + } diff --git a/src/hyperloom/inference_optimizer/tests/test_kernel_idle_streak.py b/src/hyperloom/inference_optimizer/tests/test_kernel_idle_streak.py index bd685b66a7..c17fa9eb3b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kernel_idle_streak.py +++ b/src/hyperloom/inference_optimizer/tests/test_kernel_idle_streak.py @@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone +import asyncio import pytest from hyperloom.orchestrator.phases import machine_state as ps @@ -102,6 +103,120 @@ async def test_idle_kernel_winds_down_even_while_work_pending(kernel_coordinator assert ps.kernel_work_pending(st) is True +@pytest.mark.asyncio +async def test_phase_entry_hook_cannot_starve_the_next_tick(kernel_coordinator, monkeypatch): + """Phase entry side effects are long-running work; they must not freeze the coordinator tick loop.""" + from hyperloom.orchestrator.phases import machine as phase_machine_mod + + c = kernel_coordinator + st = c.shared_state + st.phase = ps.PHASE_FRAMEWORK_AGENT + st.phase_started_ts = datetime.now(timezone.utc).isoformat() + st.max_minutes = 96 * 60 + entered = asyncio.Event() + + async def _slow_entry(**_kwargs): + entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr(c.phase_machine, "_on_phase_entered", _slow_entry) + monkeypatch.setattr( + phase_machine_mod._phase_state, + "compute_next_phase", + lambda *_args, **_kwargs: (ps.PHASE_KERNEL_AGENT, "test_enter_kernel", {"source": "test"}), + ) + + await asyncio.wait_for(c._advance_phase_if_needed(), timeout=0.2) + await asyncio.sleep(0) + + assert entered.is_set() + assert st.phase == ps.PHASE_KERNEL_AGENT + assert st.phase_history[-1]["reason"] == "test_enter_kernel" + + +def _enter_kernel_once(monkeypatch, phase_machine_mod): + """Make the next scan enter KERNEL, then hand every later scan back to the real exit rules.""" + real = phase_machine_mod._phase_state.compute_next_phase + calls = {"n": 0} + + def _next(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + return (ps.PHASE_KERNEL_AGENT, "test_enter_kernel", {"source": "test"}) + return real(*args, **kwargs) + + monkeypatch.setattr(phase_machine_mod._phase_state, "compute_next_phase", _next) + + +@pytest.mark.asyncio +async def test_kernel_holds_while_its_entry_run_is_in_flight(kernel_coordinator, monkeypatch): + """GEAK owns KERNEL from inside the entry hook; the idle guard must not hand the GPUs to SWEEP mid-run.""" + from hyperloom.orchestrator.phases import machine as phase_machine_mod + + c = kernel_coordinator + st = c.shared_state + _arm_kernel_phase(st) + st.phase = ps.PHASE_FRAMEWORK_AGENT + release = asyncio.Event() + + async def _geak_like_entry(**kwargs): + if kwargs.get("to_phase") == ps.PHASE_KERNEL_AGENT: + await release.wait() + + monkeypatch.setattr(c.phase_machine, "_on_phase_entered", _geak_like_entry) + _enter_kernel_once(monkeypatch, phase_machine_mod) + + await asyncio.wait_for(c._advance_phase_if_needed(), timeout=2.0) + assert st.phase == ps.PHASE_KERNEL_AGENT + + for _ in range(ps.KERNEL_IDLE_MAX_TICKS * 5): + await asyncio.wait_for(c._advance_phase_if_needed(), timeout=2.0) + _backdate_streak(st, ps.KERNEL_IDLE_MIN_SECONDS * 10) + budget_spent = datetime.now(timezone.utc) - timedelta(hours=30) + st.phase_started_ts = budget_spent.isoformat() + st.phase_started_unix = budget_spent.timestamp() + assert ps.exit_normal_kernel(st) is not None + await asyncio.wait_for(c._advance_phase_if_needed(), timeout=2.0) + + assert st.phase == ps.PHASE_KERNEL_AGENT + assert st.kernel_idle_ticks == 0 + assert st.phase_history[-1]["reason"] == "test_enter_kernel" + + release.set() + await asyncio.sleep(0) + _arm_kernel_phase(st) + for _ in range(ps.KERNEL_IDLE_MAX_TICKS + 1): + await c._advance_phase_if_needed() + _backdate_streak(st, ps.KERNEL_IDLE_MIN_SECONDS + 1.0) + await c._advance_phase_if_needed() + + assert st.phase == ps.PHASE_SWEEP + assert st.phase_history[-1]["reason"] == "kernel_no_more_leverage" + + +@pytest.mark.asyncio +async def test_run_waits_for_the_kernel_entry_run_before_returning(kernel_coordinator, monkeypatch): + """A session that stops while GEAK still runs must not return and leave the run behind it.""" + from hyperloom.orchestrator.phases import machine as phase_machine_mod + + c = kernel_coordinator + st = c.shared_state + _arm_kernel_phase(st) + st.phase = ps.PHASE_FRAMEWORK_AGENT + finished: list[bool] = [] + + async def _geak_like_entry(**_kwargs): + await asyncio.sleep(0.2) + finished.append(True) + + monkeypatch.setattr(c.phase_machine, "_on_phase_entered", _geak_like_entry) + _enter_kernel_once(monkeypatch, phase_machine_mod) + + await c.run(max_ticks=1) + + assert finished == [True] + + @pytest.mark.asyncio async def test_running_kernel_task_never_winds_down(kernel_coordinator): c = kernel_coordinator diff --git a/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py b/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py index 64ed8ca0c6..ba049ba560 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_kernel_request_handlers_units.py @@ -190,6 +190,28 @@ def test_resolve_forge_precision_fp8_unreadable_config_keeps_auto(self): state.current_best = {"extra_server_args": "--quantization fp8", "extra_envs": {}} assert krh._resolve_forge_precision_and_quant(state, {}) == ("fp8", "auto") + def test_resolve_aiter_root_from_editable_source_layout(self, tmp_path, monkeypatch): + monkeypatch.delenv("AITER_ROOT_DIR", raising=False) + root = tmp_path / "sgl-workspace" / "aiter" + package = root / "aiter" + package.mkdir(parents=True) + (root / "csrc").mkdir() + origin = package / "__init__.py" + origin.write_text("", encoding="utf-8") + + def _find_spec(name): + if name == "aiter_meta": + return None + assert name == "aiter" + return types.SimpleNamespace( + origin=str(origin), + submodule_search_locations=[str(package)], + ) + + monkeypatch.setattr(krh.importlib.util, "find_spec", _find_spec) + + assert krh._resolve_aiter_root_for_forge() == str(root) + def test_forge_gemm_tune_available_probes_the_command_it_will_run(self, monkeypatch): # The probe must be the same invocation the tool makes, in the same interpreter. seen: list[list[str]] = [] diff --git a/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py b/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py index de16908df9..6c757a513b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py +++ b/src/hyperloom/inference_optimizer/tests/test_longrun_phase1.py @@ -472,7 +472,7 @@ async def test_skip_to_close_is_consumed_when_sweep_already_settled( } st.set_pending_escalate_hint(ps.ESCALATE_HINT_SKIP_TO_CLOSE) - async def _entered(*, from_phase, to_phase): + async def _entered(*, from_phase, to_phase, reason="", evidence=None): return None monkeypatch.setattr(c.phase_machine, "_on_phase_entered", _entered) diff --git a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py index 6eb01742bb..d3ed5b3334 100644 --- a/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py +++ b/src/hyperloom/inference_optimizer/tests/test_pre_release_stall_liveness.py @@ -243,6 +243,16 @@ def test_demo_prompts_forbid_ending_the_turn_early(script: str) -> None: assert "state.json" in text +def test_demo_prompts_forbid_retired_robustness_monitor(script: str) -> None: + prompts = _BOOTSTRAP.parent / "prompts" / "pre-release" # type: ignore[union-attr] + for name in ("demo-3h.md", "demo-12h.md", "demo-12h-forge.md"): + text = (prompts / name).read_text(encoding="utf-8") + lowered = text.lower() + assert "robustness_monitor" not in lowered + assert "do **start**" not in lowered + assert "do **not** start a background watchdog" in lowered + + def test_dockerd_never_runs_on_vfs(script: str) -> None: """vfs copies every layer in full and evicted the host pod twice (200Gi, 1792Gi).""" assert "--storage-driver=vfs" not in script diff --git a/src/hyperloom/inference_optimizer/tests/test_preflight_auth_override.py b/src/hyperloom/inference_optimizer/tests/test_preflight_auth_override.py index 3f93cf8362..0dcc35168b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_preflight_auth_override.py +++ b/src/hyperloom/inference_optimizer/tests/test_preflight_auth_override.py @@ -1780,6 +1780,7 @@ def test_smoke_test_codex_model_warns_on_probe_failure(monkeypatch, capsys): def test_smoke_test_codex_model_skips_for_anthropic_only_fallback(monkeypatch, capsys): monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://api.anthropic.com") monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) def _no_probe(**kw): raise AssertionError("Anthropic-only fallback does not use CodexBackend") @@ -1795,6 +1796,7 @@ def test_parser_anthropic_only_empty_codex_model_uses_claude_model(monkeypatch): """With only Anthropic configured, an empty CODEX_MODEL follows CLAUDE_MODEL.""" monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://llm.example.invalid/anthropic") monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setenv("CLAUDE_MODEL", "claude-opus-4-6") monkeypatch.setenv("CODEX_MODEL", "") @@ -1811,7 +1813,11 @@ def test_parser_dual_protocol_gateway_empty_codex_model_uses_gateway_model(monke monkeypatch.delenv("DEEPSEEK_BASE_URL", raising=False) monkeypatch.delenv("DEEPSEEK_MODEL", raising=False) monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setenv("CLAUDE_MODEL", "deepseek-v4-pro") monkeypatch.setenv("CODEX_MODEL", "") @@ -1832,7 +1838,11 @@ def test_parser_retired_deepseek_key_only_defaults_to_gateway_model(monkeypatch) "DEEPSEEK_BASE_URL", "DEEPSEEK_MODEL", "ANTHROPIC_BASE_URL", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN", "OPENAI_BASE_URL", + "OPENAI_API_KEY", "CLAUDE_MODEL", "CODEX_MODEL", "INFERENCE_OPTIMIZER_CLAUDE_FOLLOWS_CODEX", @@ -1885,6 +1895,7 @@ def test_parser_anthropic_only_generated_codex_default_uses_claude_model(monkeyp """Generated setup env defaults must not force GPT on an Anthropic-only run.""" monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://llm.example.invalid/anthropic") monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setenv("CLAUDE_MODEL", "claude-opus-4-6") monkeypatch.setenv("CODEX_MODEL", "gpt-5.4") @@ -1930,6 +1941,9 @@ def test_preflight_does_not_clear_cached_anthropic_only_codex_follow( def test_parser_openai_only_empty_claude_model_uses_codex_model(monkeypatch): """With only OpenAI configured, an empty CLAUDE_MODEL follows CODEX_MODEL.""" monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) monkeypatch.setenv("OPENAI_BASE_URL", "https://llm.example.invalid/Unified/v1") monkeypatch.setenv("CODEX_MODEL", "GPT-5.4") monkeypatch.delenv("CLAUDE_MODEL", raising=False) @@ -1958,6 +1972,9 @@ def test_parser_marker_forces_claude_model_to_follow_codex(monkeypatch): def test_validate_claude_model_openai_only_accepts_codex_model(monkeypatch): """OpenAI-only runs validate the followed orchestration model against the OpenAI catalog.""" monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False) monkeypatch.setenv("OPENAI_BASE_URL", "https://llm.example.invalid/Unified/v1") monkeypatch.setenv("_".join(("OPENAI", "API", "KEY")), "openai-token") monkeypatch.setenv("CODEX_MODEL", "GPT-5.4") diff --git a/src/hyperloom/inference_optimizer/tests/test_prelude_roofline.py b/src/hyperloom/inference_optimizer/tests/test_prelude_roofline.py index 5e421a3493..87c3e22b4f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_prelude_roofline.py +++ b/src/hyperloom/inference_optimizer/tests/test_prelude_roofline.py @@ -281,6 +281,36 @@ def test_watermark_stops_re_arming_once_retries_are_spent(coord: Coordinator): assert coord._needs_roofline_for_watermark() is False +@pytest.mark.asyncio +async def test_a_condemned_stack_enqueues_no_analysis_at_all(coord: Coordinator): + """``roofline_failure_streak`` only bounds the watermark path, so a stack that can never produce a GPU trace has + to be stopped here instead -- cycle_start and the KERNEL re-profile do not pass through that counter. + """ + coord.shared_state.gpu_trace_unsupported_reason = "no GPU kernels on this stack" + + for reason in ("prelude_initial", "cycle_start", "kernel_entry_g1_abc", "close_post_opt"): + assert await coord._enqueue_internal_analysis_task(reason=reason) is None + + assert coord.tasks._tasks == {} + + +@pytest.mark.asyncio +async def test_watermark_gate_closes_on_a_condemned_stack(coord: Coordinator): + """The watermark gate is consulted before the enqueue, so it must agree rather than arm a task that is dropped.""" + state = coord.shared_state + state.baseline_tput = 100.0 + state.cumulative_gain_validated = 50.0 + state.last_roofline_tput = 0.0 + state.auto_roofline_pending_task_id = "" + state.roofline_failure_streak = 1 # anchors on baseline_tput, which is what arms the gate + assert coord._needs_roofline_for_watermark() is True + + state.gpu_trace_unsupported_reason = "no GPU kernels on this stack" + + assert coord._needs_roofline_for_watermark() is False + assert await coord._maybe_enqueue_watermark_roofline(reason="integrate_keep_watermark") is False + + def test_watermark_roofline_inherits_current_best_args(coord: Coordinator): """Watermark roofline still profiles the optimized current_best config.""" coord.shared_state.current_best = { diff --git a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py index 03329f2481..3d699e85ee 100644 --- a/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py +++ b/src/hyperloom/inference_optimizer/tests/test_profile_and_kernel_handlers.py @@ -447,7 +447,7 @@ def test_materialize_profile_window_vllm_skill_formula_default_R( assert "--profiler-config.delay_iterations 6080" in extra, extra assert "--profiler-config.max_iterations 128" in extra, extra # ``profiler=torch`` and a trace dir have to be asserted here too, not left to - # Magpie's launcher script alone: that script appends its own flags *after* + # Magpie's launcher script alone: that script prepends its own flags *before* # EXTRA_VLLM_ARGS at actual launch, but the argv preflight probe only sees # EXTRA_VLLM_ARGS, and vLLM's ProfilerConfig validator rejects # delay/max_iterations without both present in the checked fragment. @@ -455,6 +455,32 @@ def test_materialize_profile_window_vllm_skill_formula_default_R( assert "--profiler-config.torch_profiler_dir" in extra, extra +def test_candidate_trace_dirs_covers_the_vllm_output_root(tmp_path): + """vLLM writes rank traces into the run output dir, one level above the Magpie workspace.""" + from hyperloom.orchestrator.actions.executors.profile import _candidate_trace_dirs + + workspace = tmp_path / "task" / "benchmark_vllm_20260922" + assert _candidate_trace_dirs(workspace) == [ + workspace / "torch_trace", + workspace, + workspace / "capture_traces", + workspace.parent / "capture_traces", + workspace.parent, + ] + + +def test_materialize_profile_pins_magpie_and_vllm_to_one_trace_dir(tmp_path, monkeypatch): + """Whichever of the two duplicate flags vLLM keeps, the trace has to land where discovery looks.""" + import yaml + + _clear_workload_env(monkeypatch) + src = _profile_yaml(tmp_path, "vllm", {"CONC": 32, "ISL": 256, "OSL": 1024}) + out = _materialize_config_with_envs(src, tmp_path) + envs = yaml.safe_load(out.read_text())["benchmark"]["envs"] + injected = envs["EXTRA_VLLM_ARGS"].split("--profiler-config.torch_profiler_dir ", 1)[1].split()[0] + assert envs["VLLM_TORCH_PROFILER_DIR"] == injected + + def test_materialize_profile_does_not_duplicate_an_explicit_profiler_flag( tmp_path, monkeypatch, @@ -2418,8 +2444,7 @@ def _fake_run_ix(cmd, *args, **kwargs): @pytest.mark.asyncio -async def test_profile_executor_extracts_vllm_capture_traces(tmp_path): - """TraceLens-patched vLLM writes graph-capture traces next to the benchmark workspace, under the profile task's ``capture_traces`` dir.""" +async def test_profile_executor_prefers_workspace_trace_over_capture_sidecar(tmp_path): db = SqliteConnection(tmp_path / "x.db") locks = ResourceLockManager(SqliteLeaseBackend(db)) tr = TaskRegistry(db) @@ -2448,10 +2473,10 @@ def _fake_run(cmd, *args, **kwargs): } ) ) - capture_dir = output_dir / "capture_traces" + _gz_trace(workspace / "rank0.177.pt.trace.json.gz", 128) + capture_dir = workspace / "capture_traces" capture_dir.mkdir(exist_ok=True) - (capture_dir / "graph_capture_rank_0.1.pt.trace.json.gz").write_bytes(b"fake-trace") - (capture_dir / "graph_capture_rank_0.2.pt.trace.json.gz").write_bytes(b"fake-trace") + _gz_trace(capture_dir / "graph_capture_rank_0.1.pt.trace.json.gz", 32) return subprocess.CompletedProcess(args=[], returncode=0, stdout="ok", stderr="") pe = ProfileExecutor(session_dir=tmp_path / "ignored_root") @@ -2464,12 +2489,15 @@ def _fake_run(cmd, *args, **kwargs): with patch("hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", side_effect=_fake_run): res = await sub.run_task(task) - capture_dir = output_dir / "capture_traces" + workspace = output_dir / "benchmark_vllm_20260501_001122" + complete_trace = workspace / "rank0.177.pt.trace.json.gz" assert res.state == "succeeded" assert res.result["framework"] == "vllm" - assert res.result["trace_dir"] == str(capture_dir) - assert len(res.result["trace_files"]) == 2 - assert res.result["main_trace_path"].startswith(str(capture_dir)) + assert res.result["trace_dir"] == str(workspace) + assert res.result["trace_files"] == [str(complete_trace)] + assert res.result["main_trace_path"] == str(workspace) + assert res.result["profile_trace_selection_reason"] == "trace_dir_preferred" + assert res.result["profile_trace_selection_reason"] != "capture_only_fallback" db.close() diff --git a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py index 8b5a4d2270..64e9905f4e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_ray_backend_unit.py @@ -116,7 +116,7 @@ def run(cmd, **kwargs): @pytest.mark.parametrize("ack", [None, False, True]) -def test_specialist_close_requires_positive_worker_cleanup_ack(monkeypatch, ack): +def test_specialist_close_forces_actor_kill_after_stop_attempt(monkeypatch, ack): from types import SimpleNamespace actor = SimpleNamespace(stop=SimpleNamespace(remote=lambda: ack)) @@ -124,10 +124,12 @@ def test_specialist_close_requires_positive_worker_cleanup_ack(monkeypatch, ack) monkeypatch.setitem(sys.modules, "ray", SimpleNamespace(get=lambda ref, **kw: ref, kill=killed.append)) lease = rs.GpuSpecialistLease(num_gpus=1) lease._actor = actor + lease._start_ref = object() closed = lease.close() - assert closed is (ack is True) - assert killed == ([actor] if ack is True else []) - assert (lease._actor is None) is (ack is True) + assert closed is True + assert killed == [actor] + assert lease._actor is None + assert lease._start_ref is None @pytest.mark.parametrize( @@ -152,7 +154,7 @@ def test_managed_exited_root_cannot_confirm_detached_descendants(monkeypatch, gr assert signals == [], "an old PGID does not establish ownership" -def test_pending_actor_is_retained_without_cleanup_ack(monkeypatch): +def test_pending_actor_is_killed_without_cleanup_ack(monkeypatch): from types import SimpleNamespace class RayError(Exception): @@ -174,46 +176,39 @@ def get(ref, **kwargs): lease._actor = actor lease._start_ref = object() closed = lease.close() - assert closed is False - assert killed == [] - assert lease._actor is actor - assert lease._start_ref is not None + assert closed is True + assert killed == [actor] + assert lease._actor is None + assert lease._start_ref is None assert lease.pid() is None -def test_specialist_close_can_confirm_on_same_actor_after_unconfirmed_stop(monkeypatch): +def test_specialist_close_is_idempotent_after_forced_kill(monkeypatch): from types import SimpleNamespace - replies = iter([False, True]) stops = [] def stop(): stops.append(True) - return next(replies) + return False actor = SimpleNamespace(stop=SimpleNamespace(remote=stop)) killed = [] monkeypatch.setitem(sys.modules, "ray", SimpleNamespace(get=lambda ref, **kw: ref, kill=killed.append)) lease = rs.GpuSpecialistLease(num_gpus=1) lease._actor = actor - lease._start_ref = start_ref = object() - - first_close = lease.close() - assert first_close is False - assert lease._actor is actor - assert lease._start_ref is start_ref - assert killed == [] - assert len(stops) == 1 + lease._start_ref = object() confirmed_close = lease.close() assert confirmed_close is True assert lease._actor is None assert lease._start_ref is None assert killed == [actor] - assert len(stops) == 2 + assert len(stops) == 1 repeated_close = lease.close() assert repeated_close is True - assert len(stops) == 2 + assert killed == [actor] + assert len(stops) == 1 @pytest.mark.parametrize("confirmed", [False, True]) @@ -648,6 +643,7 @@ def remote(self, *_a, **kw): class _FakeActor: def __init__(self, ret): self.run_blocking = _FakeMethod(ret) + self.pid = _FakeMethod(None) class _LeaseFakeRay: @@ -678,6 +674,9 @@ def get(self, ref, **_kw): raise ref return ref + def wait(self, refs, num_returns=1, timeout=None): + return list(refs)[:num_returns], list(refs)[num_returns:] + def kill(self, actor): self.killed.append(actor) @@ -1812,8 +1811,8 @@ def test_gpu_specialist_lease_infeasible_raises(monkeypatch: pytest.MonkeyPatch) lease.start_async(["agent"]) -class _NoTimeoutCapturingFakeRay: - """Fake ray that captures whether a timeout kwarg was passed to get().""" +class _PendingAcquireFakeRay: + """Leave the actor pending once, then expose the benchmark result.""" class exceptions: # noqa: N801 class RayTaskError(Exception): @@ -1824,33 +1823,51 @@ class RayActorError(Exception): def __init__(self, result=(0, "ok", "")): self._result = result - self.get_kwargs: list[dict] = [] self.killed: list = [] + self.wait_count = 0 def get(self, ref, **kwargs): - self.get_kwargs.append(dict(kwargs)) - if isinstance(ref, _NoTimeoutCapturingFakeRay.exceptions.RayTaskError): + if isinstance(ref, _PendingAcquireFakeRay.exceptions.RayTaskError): raise ref - if isinstance(ref, _NoTimeoutCapturingFakeRay.exceptions.RayActorError): + if isinstance(ref, _PendingAcquireFakeRay.exceptions.RayActorError): raise ref return ref + def cluster_resources(self): + return {"GPU": 1.0, "serving_slot": 1.0} + + def wait(self, refs, num_returns=1, timeout=None): + self.wait_count += 1 + if self.wait_count == 1: + return [], list(refs) + return [refs[0]], list(refs)[1:] + def kill(self, actor): self.killed.append(actor) -def test_serving_lease_coordinator_no_timeout(monkeypatch: pytest.MonkeyPatch): - """ServingLease.run_session_kill calls ray.get(ref) with NO timeout kwarg.""" - fake = _NoTimeoutCapturingFakeRay(result=(0, "ok", "")) +def test_serving_lease_pending_actor_has_resource_acquire_timeout(monkeypatch: pytest.MonkeyPatch): + fake = _PendingAcquireFakeRay(result=(rs._ACTOR_TIMEOUT_RC, "", "benchmark timeout")) monkeypatch.setitem(sys.modules, "ray", fake) + monkeypatch.setattr(rs, "RESOURCE_ACQUIRE_TIMEOUT_SEC", 0.0, raising=False) lease = rs.ServingLease(num_gpus=1) - lease._actor = _FakeActor((0, "ok", "")) # pre-set: skip ensure() - rc, out, _err = lease.run_session_kill(["echo", "ok"], timeout=5) - assert rc == 0 - # The ray.get() for the benchmark call must have no timeout keyword. - assert all("timeout" not in kw for kw in fake.get_kwargs), ( - f"ServingLease must not pass timeout to ray.get; got kwargs: {fake.get_kwargs}" - ) + actor = _FakeActor((rs._ACTOR_TIMEOUT_RC, "", "benchmark timeout")) + lease._actor = actor + + try: + result = lease.run_session_kill(["echo", "ok"], timeout=5) + except subprocess.TimeoutExpired: + result = (rs._ACTOR_TIMEOUT_RC, "", "benchmark timeout") + + assert result == (1, "", "resource_acquire_timeout") + assert fake.killed == [actor] + assert lease._actor is None + + next_actor = _FakeActor((0, "next", "")) + monkeypatch.setattr(rb, "get_ray_backend", lambda: _StubBackendP2()) + monkeypatch.setattr(rs, "make_serving_actor", lambda _num_gpus, *, serving_slot: next_actor) + assert lease.run_session_kill(["echo", "next"], timeout=5) == (0, "next", "") + assert lease._actor is next_actor # ── Robustness: _RayLeaseProcess.poll dead-actor detection ─────────────────── diff --git a/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py b/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py index 2a78c6b788..17139f4ea6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_roofline_executor.py @@ -16,6 +16,7 @@ RooflineExecutor, _extract_trace_path, _failed, + _gpu_trace_unsupported_reason, make_roofline_executor, ) from hyperloom.orchestrator.trace.task_progress import progress_scope @@ -459,6 +460,131 @@ async def test_profile_no_trace_path(tmp_path): assert state.last_profile_trace == "" +def _selfcert_result(kernel_count: int, *, zero_ops: bool) -> dict: + """A profile result carrying only the two fields the stack-capability verdict reads.""" + return { + "status": "succeeded", + "trace_validate": {"verdict": {"measures": {"kernel_count": kernel_count}}}, + "trace_health": {"zero_ops": zero_ops}, + } + + +def test_zero_kernels_beside_a_populated_host_timeline_is_a_stack_verdict(): + """The ROCm 10 Kineto shape: the trace parses and carries host ops, yet no kernel was recorded.""" + reason = _gpu_trace_unsupported_reason(_selfcert_result(0, zero_ops=False)) + assert reason == ( + "the profiler recorded 0 GPU kernels beside a populated host timeline, so this stack cannot capture " + "GPU traces at all" + ) + + +@pytest.mark.parametrize( + "profile_result", + [ + pytest.param(_selfcert_result(0, zero_ops=True), id="empty-window"), + pytest.param(_selfcert_result(91, zero_ops=False), id="kernels-present"), + pytest.param({"status": "succeeded"}, id="no-certificate"), + pytest.param( + {"status": "succeeded", "trace_validate": {"verdict": {"measures": {"kernel_count": 0}}}}, + id="no-trace-health", + ), + pytest.param( + { + "status": "succeeded", + "trace_validate": {"verdict": {"measures": {"kernel_count": None}}}, + "trace_health": {"zero_ops": False}, + }, + id="kernel-count-unmeasured", + ), + ], +) +def test_a_transient_capture_failure_is_never_a_stack_verdict(profile_result): + """Only a parsed trace with host ops and zero kernels condemns the stack; anything less must stay retryable, + because condemning it wrongly gives up kernel attribution for the rest of the session. + """ + assert _gpu_trace_unsupported_reason(profile_result) == "" + + +@pytest.mark.asyncio +async def test_zero_kernel_trace_records_the_stack_verdict_on_shared_state(tmp_path): + """The verdict has to survive the action, since the enqueue gate reads it rather than the profile result.""" + state = _state() + profile_result = _profile_success() + profile_result["trace_validate"] = {"verdict": {"measures": {"kernel_count": 0}}} + profile_result["trace_health"] = {"zero_ops": False} + p1, p2 = _patch_subs(profile_result, _trace_analyze_success()) + executor = RooflineExecutor(shared_state=state) + with p1, p2: + await executor(_ctx(tmp_path)) + + assert state.gpu_trace_unsupported_reason == ( + "the profiler recorded 0 GPU kernels beside a populated host timeline, so this stack cannot capture " + "GPU traces at all" + ) + + +def _trace_reporting_backend(path: Path, backend_line: str) -> str: + """A gzipped trace whose top-level metadata names the Kineto backend, as PyTorch writes it.""" + import gzip as _gzip + + body = '{\n "schemaVersion": 1,\n' + backend_line + ' "traceEvents": [\n {"cat": "kernel"}\n ]\n}\n' + with _gzip.open(path, "wb") as handle: + handle.write(body.encode("utf-8")) + return str(path) + + +@pytest.mark.asyncio +async def test_the_condemning_verdict_names_the_backend_that_lost_the_kernels(tmp_path): + """Which backend recorded the trace is what matches the session against the upstream fix, so the verdict carries + it rather than leaving a reader to guess from the torch version. + """ + state = _state() + trace = _trace_reporting_backend(tmp_path / "r.trace.json.gz", ' "rocprofiler-sdk_version": 1.3,\n') + profile_result = _profile_success(trace) + profile_result["trace_validate"] = {"verdict": {"measures": {"kernel_count": 0}}} + profile_result["trace_health"] = {"zero_ops": False} + p1, p2 = _patch_subs(profile_result, _trace_analyze_success()) + with p1, p2: + await RooflineExecutor(shared_state=state)(_ctx(tmp_path)) + + assert state.gpu_trace_unsupported_reason == ( + "the profiler recorded 0 GPU kernels beside a populated host timeline, so this stack cannot capture " + "GPU traces at all (Kineto backend: rocprofiler-sdk 1.3)" + ) + assert state.stack_fingerprint_meta["kineto_backend"] == "rocprofiler-sdk 1.3" + + +@pytest.mark.asyncio +async def test_the_backend_is_recorded_even_when_the_capture_worked(tmp_path): + """The backend is stack provenance, not a failure detail: the healthy leg has to be identifiable too.""" + state = _state() + trace = _trace_reporting_backend(tmp_path / "ok.trace.json.gz", ' "roctracer_version": 4.1,\n') + profile_result = _profile_success(trace) + profile_result["trace_validate"] = {"verdict": {"measures": {"kernel_count": 99631}}} + profile_result["trace_health"] = {"zero_ops": False} + p1, p2 = _patch_subs(profile_result, _trace_analyze_success()) + with p1, p2: + await RooflineExecutor(shared_state=state)(_ctx(tmp_path)) + + assert state.stack_fingerprint_meta["kineto_backend"] == "roctracer 4.1" + assert state.gpu_trace_unsupported_reason == "" + + +@pytest.mark.asyncio +async def test_a_trace_with_kernels_leaves_the_stack_uncondemned(tmp_path): + """The default path must stay untouched: a healthy capture leaves the gate open.""" + state = _state() + profile_result = _profile_success() + profile_result["trace_validate"] = {"verdict": {"measures": {"kernel_count": 90488}}} + profile_result["trace_health"] = {"zero_ops": False} + p1, p2 = _patch_subs(profile_result, _trace_analyze_success()) + executor = RooflineExecutor(shared_state=state) + with p1, p2: + await executor(_ctx(tmp_path)) + + assert state.gpu_trace_unsupported_reason == "" + + @pytest.mark.asyncio async def test_profile_raises_exception(tmp_path): state = _state() diff --git a/src/hyperloom/inference_optimizer/tests/test_setup_cli.py b/src/hyperloom/inference_optimizer/tests/test_setup_cli.py index 4043da9d69..62ee9fe2c2 100644 --- a/src/hyperloom/inference_optimizer/tests/test_setup_cli.py +++ b/src/hyperloom/inference_optimizer/tests/test_setup_cli.py @@ -167,6 +167,91 @@ def _fake_run(cmd, *, env): assert key not in env +def _credential_functions() -> str: + """The credential half of ``install_baremetal.sh``, runnable on its own with stubbed log/die helpers.""" + install_script = Path(setup.__file__).resolve().parent / "assets" / "install_baremetal.sh" + script_text = install_script.read_text(encoding="utf-8") + return script_text[script_text.index("read_dotenv_var() {") : script_text.index("\nwrite_runtime_dotenv() {")] + + +def test_baremetal_setup_accepts_a_codex_only_gateway(tmp_path: Path): + """Hyperloom's runtime drives both claude and codex, and ``cli/preflight._provider_only_mode`` already treats an + OpenAI-only environment as a valid single provider; the bare-metal installer must not be the one place refusing it. + """ + dotenv = tmp_path / ".env" + dotenv.write_text("HYPERLOOM_RUN_MODE=baremetal\n", encoding="utf-8") + runner = tmp_path / "run.sh" + runner.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + f"DOTENV={dotenv}", + "CHECK_ONLY=0", + "DRY_RUN=0", + "OPENAI_BASE_URL_ARG=", + "log() { :; }", + "warn() { :; }", + 'die() { echo "$*" >&2; exit 99; }', + "is_interactive() { return 1; }", + _credential_functions(), + "unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL CLAUDE_CODE_OAUTH_TOKEN", + "unset DEEPSEEK_API_KEY DEEPSEEK_BASE_URL", + "OPENAI_BASE_URL=https://gw.example.com/api/v1/llm-proxy/v1", + "OPENAI_API_KEY=ak-codex-only", + "CODEX_MODEL=glm-5-3", + "resolve_credentials", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = subprocess.run(["bash", str(runner)], capture_output=True, text=True) + + assert proc.returncode == 0, proc.stderr + text = dotenv.read_text(encoding="utf-8") + assert "OPENAI_BASE_URL=https://gw.example.com/api/v1/llm-proxy/v1" in text + assert "OPENAI_API_KEY=ak-codex-only" in text + assert "CODEX_MODEL=glm-5-3" in text + + +def test_baremetal_setup_still_rejects_a_half_configured_openai_side(tmp_path: Path): + """A base URL with no key is not a codex deployment; accepting it would trade a clear setup error for a run that + fails much later, when the first agent call is made. + """ + dotenv = tmp_path / ".env" + dotenv.write_text("HYPERLOOM_RUN_MODE=baremetal\n", encoding="utf-8") + runner = tmp_path / "run.sh" + runner.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + f"DOTENV={dotenv}", + "CHECK_ONLY=0", + "DRY_RUN=0", + "OPENAI_BASE_URL_ARG=", + "log() { :; }", + "warn() { :; }", + 'die() { echo "$*" >&2; exit 99; }', + "is_interactive() { return 1; }", + _credential_functions(), + "unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL CLAUDE_CODE_OAUTH_TOKEN", + "unset DEEPSEEK_API_KEY DEEPSEEK_BASE_URL OPENAI_API_KEY", + "OPENAI_BASE_URL=https://gw.example.com/api/v1/llm-proxy/v1", + "resolve_credentials", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = subprocess.run(["bash", str(runner)], capture_output=True, text=True) + + assert proc.returncode == 99, proc.stdout + proc.stderr + + def test_baremetal_setup_authoritative_anthropic_env_removes_openai_keys(tmp_path: Path): install_script = Path(setup.__file__).resolve().parent / "assets" / "install_baremetal.sh" script_text = install_script.read_text(encoding="utf-8") @@ -2164,6 +2249,9 @@ def test_baremetal_sglang_installs_aiter_when_find_spec_succeeds_but_import_fail start = script_text.index("install_sglang_framework() {") end = script_text.index("\n}\n\n# Verify that the installed vLLM package resolves", start) + 3 install_sglang_framework = script_text[start:end] + # The import probe that decides whether to install lives in this helper, shared with the vLLM route. + aiter_start = script_text.index("ensure_aiter_for_python() {") + ensure_aiter_for_python = script_text[aiter_start : script_text.index("\n}\n", aiter_start) + 3] fake_py = tmp_path / "python" calls_file = tmp_path / "calls.txt" @@ -2214,6 +2302,7 @@ def test_baremetal_sglang_installs_aiter_when_find_spec_succeeds_but_import_fail "sglang_pypi_version_for_extra() { printf '7.2.4\\n'; }", "ensure_rocm_devel_headers() { :; }", f'install_compatible_aiter() {{ printf \'install_compatible_aiter %s %s\\n\' "$1" "$2" >> "$CALLS_FILE"; touch {import_flag}; }}', + ensure_aiter_for_python, install_sglang_framework, "install_sglang_framework", ] diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py index e953ee9f2d..139e0abc24 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field from pathlib import Path +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock @@ -417,15 +418,21 @@ async def test_dispatcher_hook_calls_bookkeeping_on_specialist_task( # 7. Point 2 — stalled-domain hard-trigger @pytest.fixture -def force_coord(tmp_path: Path): +def force_coord(tmp_path: Path, monkeypatch): """Coordinator stand-in with a real SharedState + mocked _handle_intent.""" from hyperloom.orchestrator.loop.coordinator import Coordinator + from hyperloom.orchestrator.specialists.dispatch import SpecialistDispatchCollaborator from hyperloom.orchestrator.state.shared_state import SharedState c = Coordinator.__new__(Coordinator) c.session_dir = tmp_path c.shared_state = SharedState() c.shared_state.phase = "FRAMEWORK_AGENT" + source_root = tmp_path / "framework" + (source_root / ".git").mkdir(parents=True) + c.shared_state.framework_repo_path = str(source_root) + c.tasks = SimpleNamespace(find_by_idempotency_key=AsyncMock(return_value=None)) + monkeypatch.setattr(SpecialistDispatchCollaborator, "_warm_specialist_params", AsyncMock()) c._handle_intent = AsyncMock() # type: ignore[method-assign] return c @@ -482,6 +489,74 @@ async def test_force_stalled_idempotency_key_is_cycle_scoped(force_coord): assert intent.payload["idempotency_key"].endswith("-c2") +@pytest.mark.asyncio +@pytest.mark.parametrize("task_state", ["queued", "running", "failed", "succeeded", "cancelled"]) +async def test_force_stalled_domain_does_not_resubmit_existing_round(force_coord, task_state): + state = force_coord.shared_state + for _ in range(10): + state.bump_domain_round_counters() + state.upsert_gap( + { + "canonical_id": "gap.framework.scheduler.s1", + "domain_hint": "serving_specialist", + "severity": "high", + } + ) + force_coord.tasks.find_by_idempotency_key.return_value = SimpleNamespace(state=task_state) + + await force_coord._maybe_force_stalled_domain_specialist() + + force_coord._handle_intent.assert_not_awaited() + force_coord.tasks.find_by_idempotency_key.assert_awaited_once_with("forced-stalled-framework-round0") + + +@pytest.mark.asyncio +async def test_force_stalled_source_patch_without_git_root_is_pruned_once(force_coord): + state = force_coord.shared_state + state.framework_repo_path = "" + for _ in range(10): + state.bump_domain_round_counters() + state.upsert_gap( + { + "canonical_id": "gap.framework.scheduler.s1", + "domain_hint": "serving_specialist", + "severity": "high", + } + ) + + await force_coord._maybe_force_stalled_domain_specialist() + await force_coord._maybe_force_stalled_domain_specialist() + + force_coord._handle_intent.assert_not_awaited() + assert state.pruned_families == ["source_patch"] + failures = [ + (row["action"], row["task_id"], row["error_class"], row["error_excerpt"]) for row in state.last_action_failures + ] + assert failures == [ + ( + "specialist", + "forced-stalled-framework-round0", + "no_git_framework_source_root", + "no_git_framework_source_root", + ) + ] + + +@pytest.mark.asyncio +async def test_force_stalled_research_specialist_ignores_source_patch_prune(force_coord): + state = force_coord.shared_state + state.framework_repo_path = "" + state.add_pruned_family("source_patch") + state.stalled_domains = lambda **_kwargs: ["pr_intelligence"] + state.best_gap_for_anchor = lambda _anchor: "gap.framework.discovery.s1" + + await force_coord._maybe_force_stalled_domain_specialist() + + force_coord._handle_intent.assert_awaited_once() + _, intent = force_coord._handle_intent.call_args.args + assert intent.payload["params"]["domain"] == "candidate_discovery_specialist" + + @pytest.mark.asyncio async def test_force_stalled_domain_noop_without_pending_gap(force_coord): state = force_coord.shared_state diff --git a/src/hyperloom/inference_optimizer/tests/test_stop_reason_outcomes.py b/src/hyperloom/inference_optimizer/tests/test_stop_reason_outcomes.py index 6240ad0552..bf8f492065 100644 --- a/src/hyperloom/inference_optimizer/tests/test_stop_reason_outcomes.py +++ b/src/hyperloom/inference_optimizer/tests/test_stop_reason_outcomes.py @@ -18,6 +18,7 @@ [ ("target_reached", "completed"), ("sweep_done", "completed"), + ("sweep_failed", "failed"), ("signal", "aborted"), ("", "aborted"), ("baseline_failed", "failed"), diff --git a/src/hyperloom/inference_optimizer/tests/test_vllm_image_source.py b/src/hyperloom/inference_optimizer/tests/test_vllm_image_source.py new file mode 100644 index 0000000000..2a32b6cf5b --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_vllm_image_source.py @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Prebuilt-image vLLM source activation tests.""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +from pathlib import Path + +from hyperloom.orchestrator.actions.executors import _workload_envs as we + +_INSTALL = Path(__file__).resolve().parents[1] / "assets" / "install.sh" +_COMMIT = "f46a9dfe2c5f57bebbd29556cbbb25eabd874226" + + +def _functions(*names: str) -> str: + text = _INSTALL.read_text(encoding="utf-8") + chunks = [] + for name in names: + match = re.search(rf"^{name}\(\) \{{.*?^\}}$", text, re.MULTILINE | re.DOTALL) + assert match, f"{name}() not found" + chunks.append(match.group()) + return "\n\n".join(chunks) + + +def _bash(names: tuple[str, ...], body: str) -> subprocess.CompletedProcess[str]: + script = f"set -euo pipefail\n{_functions(*names)}\n{body}\n" + return subprocess.run(["bash", "-c", script], text=True, capture_output=True, check=False) + + +def _git(*args: str, cwd: Path | None = None) -> str: + return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True, check=True).stdout.strip() + + +def _image_tree(tmp_path: Path) -> tuple[Path, Path, Path, str]: + upstream = tmp_path / "upstream" + upstream.mkdir() + _git("init", "-q", cwd=upstream) + _git("config", "user.email", "test@example.com", cwd=upstream) + _git("config", "user.name", "Test", cwd=upstream) + files = { + "vllm/__init__.py": "SOURCE = True\n", + "vllm/tracked.py": "tracked = 1\n", + "docker/Dockerfile.rocm": "upstream rocm\n", + "docker/Dockerfile.rocm_base": "upstream base\n", + "tools/vllm-rocm/pin_rocm_dependencies.py": "upstream pin\n", + } + for relative, content in files.items(): + path = upstream / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + _git("add", ".", cwd=upstream) + _git("commit", "-qm", "source", cwd=upstream) + commit = _git("rev-parse", "HEAD", cwd=upstream) + + source = tmp_path / "app-vllm" + shutil.copytree(upstream, source, ignore=shutil.ignore_patterns(".git")) + (source / "docker" / "Dockerfile.rocm").write_text("image rocm\n", encoding="utf-8") + (source / "docker" / "Dockerfile.rocm_base").write_text("image base\n", encoding="utf-8") + (source / "tools" / "vllm-rocm" / "pin_rocm_dependencies.py").unlink() + (source / "vllm" / "_C.fake.so").write_bytes(b"existing native") + wheel = tmp_path / "site-packages" / "vllm" + wheel.mkdir(parents=True) + (wheel / "tracked.py").write_text("wheel must not overwrite\n", encoding="utf-8") + (wheel / "_version.py").write_text("__version__ = 'test'\n", encoding="utf-8") + (wheel / "_C.fake.so").write_bytes(b"wheel native") + (wheel / "_rocm_C.fake.so").write_bytes(b"native") + return upstream, source, wheel, commit + + +def test_image_deltas_form_clean_synthetic_baseline(tmp_path: Path) -> None: + upstream, source, wheel, commit = _image_tree(tmp_path) + body = f""" +die() {{ echo "$*" >&2; return 1; }} +VLLM_IMAGE_REPO="{upstream}"; VLLM_IMAGE_SOURCE_COMMIT="{commit}"; PYTHON="{os.sys.executable}" +prepare_vllm_image_git_tree "{source}" +copy_missing_vllm_wheel_artifacts "{source}" "{wheel}"; prepare_vllm_image_git_tree "{source}" +""" + result = _bash(("prepare_vllm_image_git_tree", "copy_missing_vllm_wheel_artifacts"), body) + + assert result.returncode == 0, result.stderr + head = _git("rev-parse", "HEAD", cwd=source) + assert _git("rev-parse", "HEAD^", cwd=source) == commit + assert _git("rev-parse", "refs/hyperloom/upstream", cwd=source) == commit + assert _git("rev-parse", "refs/hyperloom/image-baseline", cwd=source) == head + assert _git("show", "-s", "--format=%s", cwd=source) == "Hyperloom prebuilt vLLM image baseline" + identity = _git("show", "-s", "--format=%an <%ae>%n%cn <%ce>", cwd=source) + assert identity.splitlines() == ["Hyperloom ", "Hyperloom "] + assert _git("status", "--porcelain", "--untracked-files=no", cwd=source) == "" + assert _git("diff", "--name-only", commit, "HEAD", cwd=source).splitlines() == [ + "docker/Dockerfile.rocm", + "docker/Dockerfile.rocm_base", + "tools/vllm-rocm/pin_rocm_dependencies.py", + ] + assert (source / "docker" / "Dockerfile.rocm").read_text() == "image rocm\n" + assert not (source / "tools" / "vllm-rocm" / "pin_rocm_dependencies.py").exists() + assert (source / "vllm" / "tracked.py").read_text() == "tracked = 1\n" + assert (source / "vllm" / "_C.fake.so").read_bytes() == b"existing native" + assert (source / "vllm" / "_version.py").read_text() == "__version__ = 'test'\n" + assert (source / "vllm" / "_rocm_C.fake.so").read_bytes() == b"native" + assert _git( + "check-ignore", "vllm/_C.fake.so", "vllm/_version.py", "vllm/_rocm_C.fake.so", cwd=source + ).splitlines() == [ + "vllm/_C.fake.so", + "vllm/_version.py", + "vllm/_rocm_C.fake.so", + ] + specialist = tmp_path / "specialist" + _git("worktree", "add", "--detach", str(specialist), cwd=source) + assert (specialist / "vllm" / "tracked.py").read_text() == "tracked = 1\n" + + +def test_dirty_synthetic_baseline_fails_without_changing_source(tmp_path: Path) -> None: + upstream, source, _wheel, commit = _image_tree(tmp_path) + tracked = source / "vllm" / "tracked.py" + body = f""" +die() {{ echo "$*" >&2; return 1; }} +VLLM_IMAGE_REPO="{upstream}"; VLLM_IMAGE_SOURCE_COMMIT="{commit}" +prepare_vllm_image_git_tree "{source}" +printf 'user change\\n' > "{tracked}" +prepare_vllm_image_git_tree "{source}" +""" + result = _bash(("prepare_vllm_image_git_tree",), body) + + assert result.returncode != 0 + assert tracked.read_text() == "user change\n" + assert "user changes" in result.stderr + + +def test_commit_mismatch_leaves_other_images_untouched() -> None: + body = f""" +log() {{ :; }}; die() {{ echo "$*" >&2; return 1; }} +probe_vllm_image_wheel() {{ printf '0.27.0+gdeadbee\\tdeadbee\\t/opt/python/site-packages/vllm\\n'; }} +prepare_vllm_image_git_tree() {{ echo MUTATION; }} +copy_missing_vllm_wheel_artifacts() {{ echo MUTATION; }} +verify_vllm_image_source_import() {{ echo MUTATION; }} +VLLM_IMAGE_SOURCE_ROOT=/app/vllm; VLLM_IMAGE_SOURCE_COMMIT={_COMMIT} +activate_vllm_image_source +""" + result = _bash(("activate_vllm_image_source",), body) + assert result.returncode == 0, result.stderr + assert result.stdout == "" + + +def test_matching_runtime_persists_verified_source_aliases(tmp_path: Path) -> None: + source = tmp_path / "app-vllm" + (source / "vllm").mkdir(parents=True) + env_file = tmp_path / "kernel.env" + body = f""" +log() {{ :; }}; die() {{ echo "$*" >&2; return 1; }} +probe_vllm_image_wheel() {{ printf '0.27.0+gf46a9dfe\\tf46a9dfe\\t/opt/python/site-packages/vllm\\n'; }} +prepare_vllm_image_git_tree() {{ + export VLLM_IMAGE_UPSTREAM_SHA="$VLLM_IMAGE_SOURCE_COMMIT" + export VLLM_IMAGE_BASELINE_SHA=0123456789abcdef +}} +copy_missing_vllm_wheel_artifacts() {{ :; }} +verify_vllm_image_source_import() {{ :; }} +VLLM_IMAGE_SOURCE_ROOT="{source}"; VLLM_IMAGE_SOURCE_COMMIT={_COMMIT} +VLLM_IMAGE_SOURCE_ACTIVE=0; KERNEL_AGENT_ENV="{env_file}"; DRY_RUN=0; CHECK_ONLY=0 +activate_vllm_image_source +persist_vllm_image_source_env +""" + result = _bash(("activate_vllm_image_source", "persist_vllm_image_source_env"), body) + + assert result.returncode == 0, result.stderr + assert env_file.read_text().splitlines() == [ + f"export FRAMEWORK_REPO_PATH='{source}'", + f"export VLLM_REPO_PATH='{source}'", + f"export VLLM_DIR='{source}'", + "export HYPERLOOM_VLLM_IMAGE_SOURCE='1'", + f"export VLLM_IMAGE_UPSTREAM_SHA='{_COMMIT}'", + "export VLLM_IMAGE_BASELINE_SHA='0123456789abcdef'", + ] + + +def test_failed_source_import_never_activates(tmp_path: Path) -> None: + source = tmp_path / "app-vllm" + (source / "vllm").mkdir(parents=True) + body = f""" +log() {{ :; }}; die() {{ echo "$*" >&2; return 1; }} +probe_vllm_image_wheel() {{ printf '0.27.0+gf46a9dfe\\tf46a9dfe\\t/opt/python/site-packages/vllm\\n'; }} +prepare_vllm_image_git_tree() {{ :; }} +copy_missing_vllm_wheel_artifacts() {{ :; }} +verify_vllm_image_source_import() {{ return 1; }} +VLLM_IMAGE_SOURCE_ROOT="{source}"; VLLM_IMAGE_SOURCE_COMMIT={_COMMIT} +VLLM_IMAGE_SOURCE_ACTIVE=0; DRY_RUN=0; CHECK_ONLY=0 +activate_vllm_image_source +""" + result = _bash(("activate_vllm_image_source",), body) + assert result.returncode != 0 + assert "import verification failed" in result.stderr + + +def test_vllm_launch_uses_one_source_root(monkeypatch) -> None: + monkeypatch.setenv("HYPERLOOM_VLLM_IMAGE_SOURCE", "1") + monkeypatch.setenv("FRAMEWORK_REPO_PATH", "/app/vllm") + monkeypatch.setenv("PYTHONPATH", "/workspace:/opt/python") + bench = {"framework": "vllm"} + envs: dict[str, str] = {} + + we._apply_vllm_source_runtime(bench, envs) + + assert {key: envs[key] for key in ("FRAMEWORK_REPO_PATH", "VLLM_REPO_PATH", "VLLM_DIR")} == { + "FRAMEWORK_REPO_PATH": "/app/vllm", + "VLLM_REPO_PATH": "/app/vllm", + "VLLM_DIR": "/app/vllm", + } + assert envs["PYTHONPATH"] == "/app/vllm:/workspace:/opt/python" + + +def test_non_vllm_launch_is_unchanged(monkeypatch) -> None: + monkeypatch.setenv("HYPERLOOM_VLLM_IMAGE_SOURCE", "1") + monkeypatch.setenv("FRAMEWORK_REPO_PATH", "/app/vllm") + envs = {"PYTHONPATH": "/original"} + + we._apply_vllm_source_runtime({"framework": "sglang"}, envs) + + assert envs == {"PYTHONPATH": "/original"} diff --git a/src/hyperloom/inference_optimizer/tests/test_xdit_integration_units.py b/src/hyperloom/inference_optimizer/tests/test_xdit_integration_units.py index c7c608d2f8..a868b5fd32 100644 --- a/src/hyperloom/inference_optimizer/tests/test_xdit_integration_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_xdit_integration_units.py @@ -494,10 +494,12 @@ def _tab(self): return tab - def test_build_spec_mi355x_removed(self): + def test_build_spec_mi355x_falls_back_to_vendor(self): tab = self._tab() spec = tab.build_hyperloom_arch_spec("mi355x") - assert spec is None + assert spec is not None + assert spec["mem_bw_gbps"] == pytest.approx(8000.0) + assert spec["max_achievable_tflops"]["matrix_bf16"] == pytest.approx(2516.6) def test_build_spec_case_insensitive_and_named(self): tab = self._tab() diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index dcfbe81d22..48d264c10f 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -1036,6 +1036,7 @@ async def run_grid( session_deadline_sec: float | None = None, variant_expected_sec: float | None = None, deadline_stop: StoppedByTheRun = STOPPED_BY_THE_RUN[SESSION_TIME_EXHAUSTED_CLASS], + lifecycle_boot_only: bool = False, ) -> list[VariantResult]: """Execute variants; ``deadline_stop`` names the owner of the supplied deadline.""" silence_timeout_sec, benchmark_timeout_sec = resolve_benchmark_timeouts() @@ -2001,6 +2002,22 @@ def _skip_rest_for_budget(idx: int, *, spent_on: str, rounds_left: int = variant warnings.append(f"warmup_round_tput:{float(warmup_tput):.1f}") if not measurement.get("valid_measurement"): + if lifecycle_boot_only and rc == 0: + results.append( + VariantResult( + name=variant.name, + extra_server_args=variant.extra_server_args, + extra_envs=dict(variant.extra_envs), + status="succeeded", + workspace=str(workspace), + report_path=str(report_path) if report_path.exists() else None, + returncode=rc, + nonfatal_warnings=warnings, + note="server_lifecycle_boot_only", + ) + ) + await _report_finished_variant(i) + continue death_excerpt = server_log_death_excerpt(str(server_log)) if rc != 0: error = death_excerpt or redact_secret_values((stderr or stdout)[-2000:]) diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_variant_filter.py b/src/hyperloom/orchestrator/actions/executors/_grid_variant_filter.py index 90e1e89000..a9e0edd84f 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_variant_filter.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_variant_filter.py @@ -10,14 +10,17 @@ import logging import os import re +import shlex import subprocess import time +from pathlib import Path from hyperloom.common.env import is_truthy from ._grid_base import ( GridVariant, ) +from ._grid_server_args import compose_server_args log = logging.getLogger(__name__) @@ -329,11 +332,110 @@ def _detect_model_class(model_path: str) -> tuple[bool, bool]: return is_mla, is_moe +_UNSAFE_UNIFIED_ATTN_STACK = { + "sglang": "0.5.20.dev20260920+gc610c40399", + "aiter": "4ad99832823dde2315b361cbd3b54b1c5c12acd5", + "rocm": "10.0.0", +} +_UNSAFE_UNIFIED_ATTN_REASON = "SGLANG_USE_AITER_UNIFIED_ATTN=1 is unsafe on the exact ROCm 10 Qwen3-14B-FP8 stack" +_FULL_SHA_RE = re.compile(r"[0-9a-f]{40}") +_DIST_VERSION_SHA_RE = re.compile(r"\+g([0-9a-f]{7,})") + + +def _fingerprint_component_matches(actual: str, expected: str) -> bool: + """Match one stack-fingerprint component against a pinned commit, accepting the git-describe dist version a + package reports when no explicit commit env var was exported. + """ + if actual == expected: + return True + if not _FULL_SHA_RE.fullmatch(expected): + return False + found = _DIST_VERSION_SHA_RE.search(actual) + return found is not None and expected.startswith(found.group(1)) + + +def _matches_unsafe_unified_attn_stack( + *, + framework: str, + model_path: str, + gpu_type: str, + stack_fingerprint: dict | None, +) -> bool: + if framework.strip().lower() != "sglang" or gpu_type.strip().lower() not in {"mi355x", "gfx950"}: + return False + stack = stack_fingerprint if isinstance(stack_fingerprint, dict) else {} + if any( + not _fingerprint_component_matches(str(stack.get(key) or ""), value) + for key, value in _UNSAFE_UNIFIED_ATTN_STACK.items() + ): + return False + model_dir = Path(model_path) + if "14b" not in model_dir.name.lower() or "fp8" not in model_dir.name.lower(): + return False + try: + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return False + if not isinstance(config, dict): + return False + quant = config.get("quantization_config") or {} + if not isinstance(quant, dict): + return False + return ( + config.get("architectures") == ["Qwen3ForCausalLM"] + and str(config.get("model_type") or "").lower() == "qwen3" + and str(config.get("torch_dtype") or "").lower() in {"bf16", "bfloat16"} + and config.get("head_dim") == 128 + and config.get("num_attention_heads") == 40 + and config.get("num_key_value_heads") == 8 + and str(quant.get("quant_method") or "").lower() == "fp8" + ) + + +def _last_server_arg(args: str, flags: tuple[str, ...], default: str) -> str: + try: + tokens = shlex.split(args) + except ValueError: + return "" + value = default + for idx, token in enumerate(tokens): + for flag in flags: + if token == flag and idx + 1 < len(tokens): + value = tokens[idx + 1] + elif token.startswith(f"{flag}="): + value = token.split("=", 1)[1] + return value + + +def _unsafe_unified_attn_reason(variant: GridVariant, *, base_server_args: str) -> str | None: + envs = getattr(variant, "extra_envs", None) or {} + if not is_truthy(envs.get("SGLANG_USE_AITER_UNIFIED_ATTN"), default=False): + return None + effective_args = compose_server_args( + base_extra_args=base_server_args, + variant_extra_args=variant.extra_server_args, + remove_args=variant.remove_args, + args_mode=variant.args_mode, + ) + page_size = _last_server_arg(effective_args, ("--page-size", "--page_size"), "1") + kv_dtype = _last_server_arg( + effective_args, + ("--kv-cache-dtype", "--kv_cache_dtype"), + "auto", + ).lower() + if page_size == "1" and kv_dtype in {"auto", "bf16", "bfloat16"}: + return _UNSAFE_UNIFIED_ATTN_REASON + return None + + def apply_compatibility_filter( grid: list["GridVariant"], *, framework: str, model_path: str, + gpu_type: str = "", + stack_fingerprint: dict | None = None, + base_server_args: str = "", ) -> tuple[list["GridVariant"], list[dict]]: """Skip variants known to be incompatible with current model/framework.""" if model_path: @@ -347,12 +449,29 @@ def apply_compatibility_filter( help_available = bool(help_text) is_xdit = fw == "xdit" + unsafe_unified_attn_stack = _matches_unsafe_unified_attn_stack( + framework=fw, + model_path=model_path, + gpu_type=gpu_type, + stack_fingerprint=stack_fingerprint, + ) kept: list[GridVariant] = [] dropped: list[dict] = [] for v in grid: args = v.extra_server_args or "" skip_reason: str | None = None + if unsafe_unified_attn_stack: + skip_reason = _unsafe_unified_attn_reason(v, base_server_args=base_server_args) + if skip_reason: + dropped.append( + { + "name": v.name, + "source": "compatibility_filter", + "reason": skip_reason, + } + ) + continue # xDiT do-not-set blacklist (env-keyed; precision lock + known crashes). if is_xdit: skip_reason = xdit_blacklist_reason(getattr(v, "extra_envs", None)) diff --git a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py index 97b2458680..5b6d8fdc93 100644 --- a/src/hyperloom/orchestrator/actions/executors/_ray_serving.py +++ b/src/hyperloom/orchestrator/actions/executors/_ray_serving.py @@ -27,6 +27,9 @@ # Timeout for ray.get probes on specialist actor methods (is_alive/exit_code/stop). _LEASE_PROBE_TIMEOUT_SEC: float = 30.0 +# How long a serving round may wait for its actor to acquire resources. +RESOURCE_ACQUIRE_TIMEOUT_SEC: float = _LEASE_PROBE_TIMEOUT_SEC + # How often the submitter of a round looks up from ``ray.wait`` to see whether the action it belongs to has been # cancelled. _CANCEL_POLL_SEC: float = 0.25 @@ -380,6 +383,7 @@ def run_session_kill( # Registered before the round is submitted, so a cancel that arrives while Ray is still scheduling it is one # this call is counted as able to hear -- the same window the local path opens around its spawn. with cancel_scope_listener() as cancel_scope: + acquire_ref = self._actor.pid.remote() ref = self._actor.run_blocking.remote( cmd, env=env, @@ -390,12 +394,19 @@ def run_session_kill( server_already_ready=server_already_ready, session_remaining_sec=session_remaining_sec, ) - return self._collect_round(ref, cmd=cmd, timeout=timeout, cancel_scope=cancel_scope) + return self._collect_round( + ref, + acquire_ref=acquire_ref, + cmd=cmd, + timeout=timeout, + cancel_scope=cancel_scope, + ) def _collect_round( self, ref: Any, *, + acquire_ref: Any, cmd: list[str], timeout: int | float | None, cancel_scope: Any, @@ -410,10 +421,11 @@ def _collect_round( _actor_err: Any = getattr(_ray_exc, "RayActorError", ()) if _ray_exc else () _task_err: Any = getattr(_ray_exc, "RayTaskError", ()) if _ray_exc else () try: - if cancel_scope is None: - rc, out, err = ray.get(ref) - else: - rc, out, err = self._await_or_cancel(ref, cancel_scope=cancel_scope) + rc, out, err = self._await_or_cancel( + ref, + acquire_ref=acquire_ref, + cancel_scope=cancel_scope, + ) except _actor_err as exc: # type: ignore[misc] # The actor (worker) itself died — e.g. its server OOM-killed the worker, or raylet reaped it. Drop the # dead handle so the next round re-creates a fresh actor via ``ensure()`` and this round surfaces as a @@ -441,17 +453,38 @@ def _collect_round( raise _sp.TimeoutExpired(cmd, timeout or 0, output=out or None, stderr=err or None) return rc, out, err - def _await_or_cancel(self, ref: Any, *, cancel_scope: Any) -> tuple[int, str, str]: - """Block on a round, asking the actor to stop it if the scope is cancelled.""" + def _await_or_cancel(self, ref: Any, *, acquire_ref: Any, cancel_scope: Any) -> tuple[int, str, str]: + """Wait for actor scheduling, then forward cancellation while the round runs.""" import ray # noqa: PLC0415 from ._subprocess_kill import ORCHESTRATOR_CANCELLED_RETURNCODE # noqa: PLC0415 asked_at: float | None = None + acquire_deadline = time.monotonic() + RESOURCE_ACQUIRE_TIMEOUT_SEC + acquired = False while True: - ready, _ = ray.wait([ref], num_returns=1, timeout=_CANCEL_POLL_SEC) - if ready: + refs = [ref] if acquired else [ref, acquire_ref] + wait_timeout = _CANCEL_POLL_SEC + if not acquired: + wait_timeout = min(wait_timeout, max(0.0, acquire_deadline - time.monotonic())) + ready, _ = ray.wait(refs, num_returns=1, timeout=wait_timeout) + if ref in ready: return ray.get(ref) + if acquire_ref in ready: + ray.get(acquire_ref) + acquired = True + if cancel_scope is None: + return ray.get(ref) + continue + if not acquired and time.monotonic() >= acquire_deadline: + log.warning( + "ServingLease: actor did not acquire resources within %.0fs; killing it", + RESOURCE_ACQUIRE_TIMEOUT_SEC, + ) + self._kill_actor() + return 1, "", "resource_acquire_timeout" + if cancel_scope is None: + continue if asked_at is None: if not cancel_scope.cancelled: continue @@ -645,16 +678,12 @@ def stop(self) -> bool: return False def close(self) -> bool: - """Stop the specialist, then kill the actor to release the GPU lease. - - ``ray.kill`` skips ``__ray_terminate__``, so an unconfirmed stop keeps - the same actor available for the release callback's next cleanup attempt. - A pending start without an acknowledgement remains unconfirmed too. - """ + """Bound the stop attempt, then kill the actor to release its GPU lease.""" if self._actor is None: return True - if not self.stop(): - return False + stopped = self.stop() + if not stopped: + log.warning("GpuSpecialistLease.close: stop unconfirmed; forcing actor kill") import ray # noqa: PLC0415 try: diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 037b543218..fc7d96309e 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -667,6 +667,23 @@ def _resolve_framework_repo_path( return "" +def _apply_vllm_source_runtime(bench: dict[str, Any], envs: dict[str, Any]) -> None: + """Route a prepared image checkout into the vLLM server launch.""" + if str(bench.get("framework") or "").strip().lower() != "vllm": + return + if os.environ.get("HYPERLOOM_VLLM_IMAGE_SOURCE", "").strip() != "1": + return + repo_path = _resolve_framework_repo_path(envs, framework="vllm") + if not repo_path: + return + for name in ("FRAMEWORK_REPO_PATH", "VLLM_REPO_PATH", "VLLM_DIR"): + envs[name] = repo_path + os.environ[name] = repo_path + existing = str(envs.get("PYTHONPATH") or os.environ.get("PYTHONPATH") or "") + entries = [repo_path, *(part for part in existing.split(os.pathsep) if part)] + envs["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(entries)) + + def _custom_script_path(runner_type: str) -> str: """Locate the operator's entrypoint inside ``$HYPERLOOM_BYPASS_SCRIPTS_DIR``. @@ -1607,21 +1624,22 @@ def materialize_config_with_envs( ] # ``profiler`` and ``torch_profiler_dir`` are normally set by # Magpie's launcher script, not by this layer -- but that script - # appends its own flags *after* EXTRA_VLLM_ARGS in the real + # appends its own flags *before* EXTRA_VLLM_ARGS in the real # ``vllm serve`` invocation, so the argv preflight probe (which # only sees EXTRA_VLLM_ARGS) checks capture_torch_profiler/ # delay_iterations/max_iterations against a ProfilerConfig that # never saw ``profiler=torch`` or a trace dir. vLLM's validator # requires both whenever those bounds are present, so the probe # fails an argv that will be valid once Magpie's flags are - # appended, and this layer's profiler bounds get treated as - # invalid and dropped instead of launched. Asserting placeholders - # here keeps the probed fragment self-consistent; the actual - # ``torch_profiler_dir`` Magpie computes from ``$WORKSPACE_DIR`` - # overrides this one at real launch time via vLLM's dotted-flag - # last-wins merge, so the value here only has to be a valid - # absolute path, not the directory the trace ends up under. An - # operator-set flag is left untouched either way. + # prepended, and this layer's profiler bounds get treated as + # invalid and dropped instead of launched. Asserting the flags + # here keeps the probed fragment self-consistent, and because + # EXTRA_VLLM_ARGS comes last, this ``torch_profiler_dir`` is the + # one vLLM keeps -- it decides where the trace lands. Magpie reads + # the same directory from VLLM_TORCH_PROFILER_DIR below, so both + # sides agree no matter which flag wins. An operator-set flag is + # left untouched either way. + envs.setdefault("VLLM_TORCH_PROFILER_DIR", str(output_dir)) if _profiler_flag_value(existing_vllm_args, "profiler") is None: profiler_flags.append(("profiler", "--profiler-config.profiler torch")) if _profiler_flag_value(existing_vllm_args, "torch_profiler_dir") is None: @@ -2159,6 +2177,7 @@ def materialize_config_with_envs( extra = str(envs.get("EXTRA_ATOM_ARGS", "")).strip() if "--mark-trace" not in extra: envs["EXTRA_ATOM_ARGS"] = f"{extra} --mark-trace".strip() + _apply_vllm_source_runtime(bench, envs) # The rendered YAML is persisted, so credentials must not reach it. filtered_envs, dropped_credentials = filter_untrusted_env_mapping( envs, diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index fdaeb2465c..ecfa78881b 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -715,6 +715,14 @@ async def _run_explore(self, ctx) -> dict[str, Any]: runnable, framework=framework, model_path=resolved_model, + gpu_type=resolved_gpu or str(getattr(ss, "gpu_type", "") or ""), + stack_fingerprint=getattr(ss, "stack_fingerprint_meta", None), + base_server_args=compose_server_args( + inherited_args=_effective_inherited_args, + base_extra_args=base_extra_args, + remove_args=base_remove_args, + args_mode=base_args_mode, + ), ) # Operator-supplied --skip-variants patterns. runnable, _skip_dropped = apply_user_skip_list( diff --git a/src/hyperloom/orchestrator/actions/executors/profile.py b/src/hyperloom/orchestrator/actions/executors/profile.py index eb48c293de..446287703d 100644 --- a/src/hyperloom/orchestrator/actions/executors/profile.py +++ b/src/hyperloom/orchestrator/actions/executors/profile.py @@ -778,8 +778,12 @@ def _candidate_trace_dirs(workspace: Path) -> list[Path]: """Trace directories to probe for a Magpie profile workspace.""" return [ workspace / "torch_trace", + workspace, workspace / "capture_traces", workspace.parent / "capture_traces", + # vLLM writes rank traces straight into the run's output dir, which is the + # workspace's parent rather than anything under the Magpie workspace. + workspace.parent, ] diff --git a/src/hyperloom/orchestrator/actions/executors/roofline.py b/src/hyperloom/orchestrator/actions/executors/roofline.py index 5c3d27ac79..2401b78557 100644 --- a/src/hyperloom/orchestrator/actions/executors/roofline.py +++ b/src/hyperloom/orchestrator/actions/executors/roofline.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any, Awaitable, Callable, Sequence +from hyperloom.common.provenance import detect_kineto_backend from hyperloom.common.timeutil import now_iso from ...loop.sub_agent_runner import RunnerContext from ...trace.task_progress import report_progress @@ -130,6 +131,25 @@ async def _reclaim_gpus_for_retry(session_dir: Path | str, *, attempt: int) -> N log.debug("roofline: post-reclaim probe failed", exc_info=True) +def _gpu_trace_unsupported_reason(profile_result: dict[str, Any]) -> str: + """Why this stack can never record GPU kernels, or empty when the capture merely failed this time. Demands a + parsed trace carrying host ops beside zero kernels, so one transient empty capture cannot condemn the session. + """ + if not isinstance(profile_result, dict): + return "" + measures = ((profile_result.get("trace_validate") or {}).get("verdict") or {}).get("measures") or {} + kernel_count = measures.get("kernel_count") if isinstance(measures, dict) else None + if not isinstance(kernel_count, int) or kernel_count != 0: + return "" + health = profile_result.get("trace_health") + if not isinstance(health, dict) or health.get("zero_ops") is not False: + return "" + return ( + f"the profiler recorded {kernel_count} GPU kernels beside a populated host timeline, " + "so this stack cannot capture GPU traces at all" + ) + + def _trace_is_high_idle(ta_result: dict[str, Any]) -> bool: """Whether trace_analyze flagged the profiled step as host-bound (high GPU idle), i.e. carries a ``high_gpu_idle_pct`` trace-health warning.""" if not isinstance(ta_result, dict): @@ -938,6 +958,23 @@ async def _note_profile_run( successful_profile_params or ctx.task.params or {}, arm=roofline_arm, ) + _backend = detect_kineto_backend(trace_path) + if _backend: + _fingerprint = dict(self.shared_state.stack_fingerprint_meta or {}) + if _fingerprint.get("kineto_backend") != _backend: + _fingerprint["kineto_backend"] = _backend + self.shared_state.stack_fingerprint_meta = _fingerprint + if not self.shared_state.gpu_trace_unsupported_reason: + _unsupported = _gpu_trace_unsupported_reason(profile_result) + if _unsupported: + if _backend: + _unsupported = f"{_unsupported} (Kineto backend: {_backend})" + self.shared_state.gpu_trace_unsupported_reason = _unsupported + log.error( + "roofline: %s; automatic profile/roofline enqueues will be suppressed (stack=%s)", + _unsupported, + self.shared_state.stack_fingerprint_meta or "(unknown)", + ) # The host-side rewrite evidence is produced by the profile sub-step and is what the framework specialist is # given instead of guessing landing points from source. from ._framework_rewrite_evidence import promote_evidence_path diff --git a/src/hyperloom/orchestrator/kernel/attempt_summary.py b/src/hyperloom/orchestrator/kernel/attempt_summary.py index 683d7bb318..f1994c8edf 100644 --- a/src/hyperloom/orchestrator/kernel/attempt_summary.py +++ b/src/hyperloom/orchestrator/kernel/attempt_summary.py @@ -24,9 +24,10 @@ CATEGORY_ATTEMPTED_REJECTED = "ATTEMPTED_REJECTED" CATEGORY_IN_FLIGHT = "IN_FLIGHT" -#: Closed 4-value terminal kernel-outcome bucket the dashboard reads directly. +#: Closed terminal kernel-outcome bucket the dashboard reads directly. #: ``IN_FLIGHT`` (no terminal decision) folds into ``fail``. OUTCOME_SUCCESS = "success" +OUTCOME_UNVALIDATED = "unvalidated" OUTCOME_FAIL = "fail" OUTCOME_TIMEOUT = "timeout" OUTCOME_SKIP = "skip" @@ -310,6 +311,99 @@ def _session_kernel_opt_outcome(by_kernel: list[dict[str, Any]]) -> str: return OUTCOME_FAIL +def _lane_totals( + attempted: int, + success: int, + unvalidated: int, + failed: int, + *, + outcome: str = "", +) -> dict[str, Any]: + """Build one lane's counters and terminal outcome.""" + if not outcome: + if attempted == 0: + outcome = OUTCOME_SKIP + elif success: + outcome = OUTCOME_SUCCESS + elif unvalidated: + outcome = OUTCOME_UNVALIDATED + else: + outcome = OUTCOME_FAIL + return { + "attempted": attempted, + "success": success, + "unvalidated": unvalidated, + "failed": failed, + "outcome": outcome, + } + + +def _geak_lane_totals(state: Any) -> dict[str, Any]: + """Summarize GEAK's E2E result independently of source rewrites.""" + result = getattr(state, "geak_result", {}) or {} + if not isinstance(result, dict): + return _lane_totals(0, 0, 0, 0) + accepted = [ + row + for key in ("accepted_kernels", "accepted_heads") + for row in (result.get(key) or []) + if isinstance(row, (dict, str)) and bool(row) + ] + if accepted: + count = len(accepted) + return _lane_totals(count, count, 0, 0) + status = str(result.get("status") or "").strip().lower() + if not status or status == OUTCOME_SKIP or status == "skipped": + return _lane_totals(0, 0, 0, 0) + return _lane_totals(1, 0, 0, 1) + + +def _gemm_tuning_lane_totals(state: Any) -> dict[str, Any]: + """Summarize GEMM micro-tuning without treating candidates as E2E wins.""" + attempts = list(getattr(state, "gemm_tuning_attempts", []) or []) + if not attempts: + last = getattr(state, "last_gemm_tuning", {}) or {} + if isinstance(last, dict) and last: + attempts = [last] + attempted = success = unvalidated = failed = 0 + for result in attempts: + if not isinstance(result, dict): + continue + rows = [row for row in (result.get("tuners_run") or []) if isinstance(row, dict)] + if rows: + winners = sum( + 1 for row in rows if row.get("kept") is True or (_to_float(row.get("best_micro_speedup")) or 0.0) > 1.0 + ) + attempted += len(rows) + if result.get("requires_e2e_validation"): + unvalidated += winners + else: + success += winners + failed += len(rows) - winners + continue + status = str(result.get("status") or "").strip().lower() + if not status or status in (OUTCOME_SKIP, "skipped"): + continue + attempted += 1 + kept = str(result.get("decision") or "").upper() == "KEEP" + if kept and result.get("requires_e2e_validation"): + unvalidated += 1 + elif kept: + success += 1 + else: + failed += 1 + return _lane_totals(attempted, success, unvalidated, failed) + + +def _overall_lane_outcome(lanes: dict[str, dict[str, Any]]) -> str: + """Roll lane outcomes up without calling unvalidated work successful.""" + outcomes = {str(row.get("outcome") or "") for row in lanes.values()} + for outcome in (OUTCOME_SUCCESS, OUTCOME_UNVALIDATED, OUTCOME_TIMEOUT, OUTCOME_FAIL): + if outcome in outcomes: + return outcome + return OUTCOME_SKIP + + def _summary_integrated( entry: dict[str, Any], backend_ladder: list[dict[str, Any]], @@ -402,7 +496,7 @@ def build_kernel_optimization_summary( state: Any, session_dir: Path | str, *, - schema_version: int = 1, + schema_version: int = 2, ) -> dict[str, Any]: """Build the full summary block for one session.""" sd_path = Path(session_dir) @@ -519,13 +613,35 @@ def build_kernel_optimization_summary( rejection_breakdown=rejection_breakdown, failure_reason_breakdown=failure_reason_breakdown, ) - - return { + source_outcome = _session_kernel_opt_outcome(by_kernel) + lane_totals: dict[str, dict[str, Any]] = { + "source_level": _lane_totals( + counts["attempted"], + counts["integrated"], + counts["keep_pending"] + counts["in_flight"], + counts["rejected"], + outcome=( + OUTCOME_UNVALIDATED + if counts["attempted"] and not counts["integrated"] and (counts["keep_pending"] or counts["in_flight"]) + else source_outcome + ), + ), + "geak": _geak_lane_totals(state), + "gemm_tuning": _gemm_tuning_lane_totals(state), + } + overall_outcome = source_outcome + if schema_version >= 2: + counts["attempted"] = sum(int(lane["attempted"]) for lane in lane_totals.values()) + overall_outcome = _overall_lane_outcome(lane_totals) + if counts["attempted"] and lane_totals["source_level"]["attempted"] == 0: + top_takeaways[0] = "No source-level kernel rewrites were attempted; other kernel lanes did run." + + summary = { "schema_version": schema_version, "session_id": session_id, "model_name": str(getattr(state, "model_name", "") or ""), "cumulative_gain_validated_pct": float(getattr(state, "cumulative_gain_validated", 0.0) or 0.0), - "kernel_opt_outcome": _session_kernel_opt_outcome(by_kernel), + "kernel_opt_outcome": overall_outcome, "totals": counts, "rejection_breakdown": rejection_breakdown, "failure_reason_breakdown": failure_reason_breakdown, @@ -533,6 +649,9 @@ def build_kernel_optimization_summary( "by_kernel": by_kernel, "top_takeaways": top_takeaways, } + if schema_version >= 2: + summary["lane_totals"] = lane_totals + return summary def _render_attempted_row( @@ -762,6 +881,7 @@ def _to_float(v: Any) -> float | None: __all__ = [ "build_kernel_optimization_summary", "OUTCOME_SUCCESS", + "OUTCOME_UNVALIDATED", "OUTCOME_FAIL", "OUTCOME_TIMEOUT", "OUTCOME_SKIP", diff --git a/src/hyperloom/orchestrator/kernel/conc_sweep.py b/src/hyperloom/orchestrator/kernel/conc_sweep.py index ec71d34a6b..2ad7fcae9e 100644 --- a/src/hyperloom/orchestrator/kernel/conc_sweep.py +++ b/src/hyperloom/orchestrator/kernel/conc_sweep.py @@ -675,6 +675,7 @@ async def _sweep_ladder_by_restart() -> list[VariantResult]: budget_state=_budget_state, base_yaml_path=base_yaml_path, base_extra_args="", + base_extra_envs={"MAGPIE_RUN_PHASE": "server"}, grid=[boot_variant], output_root=workspace, model_path=model_path, @@ -685,6 +686,7 @@ async def _sweep_ladder_by_restart() -> list[VariantResult]: preclean_before_run=True, warmup_before_measure=False, serving_lease=arm_lease, + lifecycle_boot_only=True, ) except Exception as exc: # noqa: BLE001 log.warning( @@ -774,6 +776,7 @@ async def _sweep_ladder_by_restart() -> list[VariantResult]: # Boot succeeded (br is not None here — boot_failed guarded above). assert br is not None boot_succeeded = True + boot_only = str(getattr(br, "note", "") or "") == "server_lifecycle_boot_only" # Commit the higher-CONC failed boots (genuine capacity failures) first. for fb in failed_boots: arm_results.append(fb) @@ -785,8 +788,9 @@ async def _sweep_ladder_by_restart() -> list[VariantResult]: conc=variant_conc(fb), point=_point_from_variant(fb, arm=arm_name), ) - arm_results.append(br) - _all_results_ref.append(br) + if not boot_only: + arm_results.append(br) + _all_results_ref.append(br) _record_rung( recorder, arm_name, @@ -860,7 +864,11 @@ async def _sweep_ladder_by_restart() -> list[VariantResult]: # Server is up: sweep remaining CONCs by reuse. reuse_failure: BaseException | None = None try: - reuse_grid = grid[boot_idx + 1 :] + reuse_grid = ( + grid[boot_idx:] + if str(getattr(br, "note", "") or "") == "server_lifecycle_boot_only" + else grid[boot_idx + 1 :] + ) for r_idx, variant in enumerate(reuse_grid): _reuse_remaining = session_deadline_to_remaining_sec(session_deadline_sec) # Check session deadline before each reuse point. diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index ed9073b1e8..8ee995cb4f 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -1921,7 +1921,7 @@ def _forge_gemm_tune_available() -> bool: def _resolve_aiter_root_for_forge() -> str: - """Resolve AITER's source root, including split ``aiter_meta`` wheels.""" + """Resolve AITER's source root from split wheels or editable installs.""" explicit = os.environ.get("AITER_ROOT_DIR", "").strip() if explicit: return explicit @@ -1934,6 +1934,19 @@ def _resolve_aiter_root_for_forge() -> str: root = Path(location) if (root / "csrc").is_dir(): return str(root) + try: + spec = importlib.util.find_spec("aiter") + except (ModuleNotFoundError, ValueError): + spec = None + locations = list(getattr(spec, "submodule_search_locations", None) or []) + origin = getattr(spec, "origin", None) + if origin: + locations.append(str(Path(origin).parent)) + for location in locations: + package = Path(location) + root = package.parent + if package.name == "aiter" and (root / "csrc").is_dir(): + return str(root) return "" diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index ee233da98e..20648ff577 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -758,6 +758,7 @@ def router(self) -> IntentRouter: "_kernel_enabled": "phase_machine", "_optimize_enabled": "phase_machine", "_advance_phase_if_needed": "phase_machine", + "_await_kernel_entry_task": "phase_machine", "_on_phase_entered": "phase_machine", "_reseed_orch_prompt_for_phase": "phase_machine", "_record_phase_entry_evidence": "phase_machine", @@ -1758,6 +1759,10 @@ async def run( # Normal path: no stop signal within the tick interval. pass finally: + try: + await self._await_kernel_entry_task() + except (asyncio.CancelledError, Exception): # noqa: BLE001 + log.exception("Coordinator: KERNEL entry hook did not settle before shutdown") final_signals: AbstractSet[int] = frozenset() if self._signals is not None: final_signals = self._signals.close() diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 75d783f01b..5b327fe036 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -881,6 +881,24 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: # Specialist pre-dispatch warmup via KnowledgePlane. if action_name == "specialist": await self._warm_specialist_params(params) + from ..specialists.runner import specialist_patch_preflight_error + + preflight_error = specialist_patch_preflight_error( + params, + framework_repo_path=str(getattr(self.shared_state, "framework_repo_path", "") or ""), + ) + if preflight_error: + if self.shared_state.add_pruned_family("source_patch"): + self.shared_state.record_action_failure( + action="specialist", + task_id=str(intent.payload.get("idempotency_key") or nested_idempotency_key or ""), + result={"error_class": preflight_error, "error": preflight_error}, + ) + try: + self.shared_state.save(self.session_dir) + except Exception: # noqa: BLE001 + log.exception("delegate specialist: source-patch prune save failed") + return # Idempotency-key chain: top-level -> nested compat alias -> content-fingerprint auto-key. raw_key = intent.payload.get("idempotency_key") or nested_idempotency_key if not raw_key: diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index cc990a73d2..6099fb326c 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -4963,7 +4963,7 @@ async def _promote_conc_sweep( summary instead. """ outcome.early_return = True - # Write last_conc_sweep so exit_normal_sweep can fire sweep_done. + # Write last_conc_sweep so exit_normal_sweep can distinguish an honest sweep_done from a no-pair sweep_failed. self.shared_state.record_conc_sweep(result) self.shared_state.save(self.session_dir) diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index d2b5d36dc4..9dfe2a7517 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -59,6 +59,14 @@ def _derive_close_stop_reason(self) -> str: if (row.get("to_phase") or "").strip().upper() != _phase_state.PHASE_CLOSE: continue reason = (row.get("reason") or "").strip() + evidence = row.get("evidence") if isinstance(row.get("evidence"), dict) else {} + if ( + reason == "sweep_done" + and evidence.get("sweep_was_skipped") + and evidence.get("sweep_skip_budget_exhausted") + and str(evidence.get("sweep_skip_reason") or "") == "budget_exhausted_no_successful_pairs" + ): + return "sweep_failed" if reason and _phase_state.is_valid_stop_reason(reason): return reason # Newest CLOSE-bound row had no usable reason — stop rather than use a stale older one. @@ -88,6 +96,8 @@ async def _maybe_run_close_post_opt_roofline(self) -> None: # Roofline disabled for this run; nothing to profile. return task = await self._enqueue_internal_analysis_task(reason="close_post_opt") + if task is None: + return log.info( "CLOSE step 0: running post-opt roofline task=%s (timeout=%.0fs)", task.task_id, diff --git a/src/hyperloom/orchestrator/phases/kernel.py b/src/hyperloom/orchestrator/phases/kernel.py index ecbca31dae..93228b97c8 100644 --- a/src/hyperloom/orchestrator/phases/kernel.py +++ b/src/hyperloom/orchestrator/phases/kernel.py @@ -305,6 +305,16 @@ def _note_reprofile(**fields: Any) -> None: reason=idempotency_reason, inline_event=recorder.event_id if recorder is not None else "", ) + if reprofile_task is None: + _note_reprofile( + ran=False, + task_kind=task_kind, + trigger=trigger, + skipped_reason="gpu_trace_unsupported", + idempotency_reason=idempotency_reason, + snapshot_id_before=snapshot_id_before, + ) + return # An idempotent reuse can return a task that already reached a terminal state (its snapshot from a prior # cycle is still valid). run_task would then attempt succeeded->running -> IllegalTransition, so reuse the # existing snapshot instead of re-running. @@ -4180,6 +4190,8 @@ def _last_measured_roofline_tput(self) -> float: def _needs_roofline_for_watermark(self) -> bool: """True iff projected tput crossed the watermark over ``last_roofline_tput`` (False until PRELUDE roofline ran, or while auto_roofline_pending_task_id is in-flight).""" state = self.shared_state + if str(getattr(state, "gpu_trace_unsupported_reason", "") or ""): + return False try: last_rl = float(state.last_roofline_tput or 0.0) except (TypeError, ValueError): @@ -4241,6 +4253,8 @@ async def _maybe_enqueue_watermark_roofline( exc, ) return False + if task is None: + return False self.shared_state.auto_roofline_pending_task_id = task.task_id log.info( "watermark-roofline (%s): enqueued task=%s (cur=%.2f, last_roofline=%.2f, ratio>=%.2f)", diff --git a/src/hyperloom/orchestrator/phases/machine.py b/src/hyperloom/orchestrator/phases/machine.py index 830c2785d4..5f48ee7bd4 100644 --- a/src/hyperloom/orchestrator/phases/machine.py +++ b/src/hyperloom/orchestrator/phases/machine.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +import asyncio import logging as _logging from typing import Any from . import geak_rebench as _geak_rebench @@ -20,6 +21,21 @@ class MachinePhase(PhaseHandler): """Extracted phase handler; delegates unknown attrs to its Coordinator.""" + _kernel_entry_task: "asyncio.Task[Any] | None" = None + + def _kernel_entry_in_flight(self) -> bool: + """True while the out-of-band KERNEL entry hook (GEAK run, reprofile, GEMM tuning) has not returned.""" + task = self._kernel_entry_task + return task is not None and not task.done() + + async def _await_kernel_entry_task(self) -> None: + """Let a still-running KERNEL entry hook finish before the session tears down.""" + task = self._kernel_entry_task + if task is None or task.done(): + return + log.info("Coordinator: waiting for the KERNEL entry hook to finish before shutdown") + await asyncio.wait({task}) + def _ensure_phase_initialised(self) -> None: """Set ``phase`` + persist ``phase_budget_pct`` once per session (idempotent).""" state = self.shared_state @@ -157,7 +173,7 @@ async def _track_kernel_idle_streak(self) -> None: state.kernel_idle_ticks = 0 state.kernel_idle_since_unix = now return - if inflight or _phase_state.kernel_inline_step_running(state, now_unix=now): + if inflight or _phase_state.kernel_inline_step_running(state, now_unix=now) or self._kernel_entry_in_flight(): state.kernel_idle_since_unix = now return # Only reachable after a tick that opened the streak above, so ``kernel_idle_since_unix`` is already stamped @@ -185,6 +201,10 @@ async def _advance_phase_if_needed(self) -> None: await self._maybe_enqueue_trajectory_reviewer() if next_phase is None: return + if self._kernel_entry_in_flight(): + # The entry hook owns KERNEL until it returns; leaving earlier hands GPUs GEAK still holds to the next phase. + log.debug("phase_machine: holding %s -> %s until the KERNEL entry hook returns", state.phase, next_phase[0]) + return target, reason, evidence = next_phase if target == (state.phase or "").upper(): return # already there @@ -319,18 +339,39 @@ async def _advance_phase_if_needed(self) -> None: ) except Exception: # noqa: BLE001 — defensive log.exception("Coordinator: phase_transition event bus write failed") - # Phase-entry side effects are additive; hook failures are logged only. + # Phase-entry side effects are additive; hook failures are logged only. Only KERNEL entry runs out-of-band so + # its GEAK/reprofile/tuning work does not freeze the tick loop; the phase is held until it returns. + entry_kwargs: dict[str, Any] = { + "from_phase": prior or "", + "to_phase": target, + "reason": reason or "", + "evidence": evidence if isinstance(evidence, dict) else None, + } + if target_phase == _phase_state.PHASE_KERNEL_AGENT: + task = asyncio.create_task(self._on_phase_entered(**entry_kwargs)) + task.add_done_callback(self._record_phase_entry_task_result) + self._kernel_entry_task = task + return + # Every other phase keeps its entry effects on the transition itself, which is what callers advancing into + # CLOSE rely on to see the sequencer's settlement once the transition returns. try: - await self._on_phase_entered( - from_phase=prior or "", - to_phase=target, - reason=reason or "", - evidence=evidence if isinstance(evidence, dict) else None, - ) + await self._on_phase_entered(**entry_kwargs) + except Exception as exc: # noqa: BLE001 — a failed hook must not block the transition + log.exception("Coordinator: _on_phase_entered hook failed") + # This hook is also what closes the left phase's event, so a raise here is the case where that event never + # got its exit evidence. + self._record_coordinator_exception(stage="phase_entered", exc=exc) + + def _record_phase_entry_task_result(self, task: "asyncio.Task[Any]") -> None: + """Log phase-entry hook failures after the transition tick has returned.""" + try: + task.result() + except asyncio.CancelledError: + log.warning("Coordinator: _on_phase_entered hook was cancelled") except Exception as exc: # noqa: BLE001 — a failed hook must not block the transition log.exception("Coordinator: _on_phase_entered hook failed") - # This hook is also what closes the left phase's event, so a raise - # here is the case where that event never got its exit evidence. + # This hook is also what closes the left phase's event, so a raise here is the case where that event never + # got its exit evidence. self._record_coordinator_exception(stage="phase_entered", exc=exc) async def _on_phase_entered( diff --git a/src/hyperloom/orchestrator/phases/machine_state.py b/src/hyperloom/orchestrator/phases/machine_state.py index ba31be9e01..225d84908d 100644 --- a/src/hyperloom/orchestrator/phases/machine_state.py +++ b/src/hyperloom/orchestrator/phases/machine_state.py @@ -1583,6 +1583,13 @@ def exit_normal_sweep( evidence["sweep_was_skipped"] = True evidence["sweep_skip_budget_exhausted"] = bool(last_conc.get("budget_exhausted")) evidence["sweep_skip_reason"] = str(last_conc.get("skip_reason") or "") + summary = last_conc.get("summary") if isinstance(last_conc.get("summary"), dict) else {} + spent_budget_without_pair = bool(last_conc.get("budget_exhausted")) or ( + str(last_conc.get("skip_reason") or "") == "budget_exhausted_no_successful_pairs" + ) + ran_but_reported_no_pair = bool(summary) and int(summary.get("successful_pairs") or 0) <= 0 + if spent_budget_without_pair or (ran_but_reported_no_pair and not evidence["sweep_skip_reason"]): + return "sweep_failed", evidence return "sweep_done", evidence remaining = phase_budget_remaining_seconds( state, diff --git a/src/hyperloom/orchestrator/phases/macro_cycle.py b/src/hyperloom/orchestrator/phases/macro_cycle.py index 682f634645..2fddd9decf 100644 --- a/src/hyperloom/orchestrator/phases/macro_cycle.py +++ b/src/hyperloom/orchestrator/phases/macro_cycle.py @@ -318,6 +318,8 @@ async def _on_cycle_start_reprofile(self, *, from_phase: str) -> None: task = await self._enqueue_internal_analysis_task( reason="cycle_start", ) + if task is None: + return self.shared_state.auto_roofline_pending_task_id = task.task_id log.info( "cycle %d start: forced reprofile task=%s", diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index aa5fb7cc60..5690453b8b 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -2644,6 +2644,8 @@ async def _maybe_enqueue_prelude_initial_analysis_after_baseline( rl_task = await self._enqueue_internal_analysis_task( reason="prelude_initial", ) + if rl_task is None: + return state.auto_roofline_pending_task_id = rl_task.task_id log.info( "PRELUDE: baseline landed (tput=%.2f); auto-enqueued initial %s task=%s", @@ -2667,11 +2669,26 @@ def _analysis_attempt_suffix(self, kind: str) -> str: streak = 0 return f"-a{streak}" if streak > 0 else "" - async def _enqueue_internal_analysis_task(self, *, reason: str, inline_event: str = "") -> Task: - """Build + enqueue a Coordinator-internal analysis task (roofline or profile). Idempotency key internal-analysis-.""" + async def _enqueue_internal_analysis_task(self, *, reason: str, inline_event: str = "") -> "Task | None": + """Build + enqueue a Coordinator-internal analysis task (roofline or profile). Idempotency key + internal-analysis-. Returns None when the stack cannot produce a GPU trace for it to analyze. + """ from hyperloom.inference_optimizer.breakdown.recorder.event_ids import INLINE_EVENT_PARAM state = self.shared_state + unsupported = str(getattr(state, "gpu_trace_unsupported_reason", "") or "") + if unsupported: + log.error( + "internal-analysis (%s): not enqueued -- %s; relying on static-source evidence for the rest of " + "the session", + reason, + unsupported, + ) + self._record_prelude_arm_dropped( + "internal_analysis", + {"reason": reason, "gpu_trace_unsupported_reason": unsupported}, + ) + return None kind = self._internal_analysis_kind() params: dict[str, Any] = { "source": "coordinator_internal", diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index f3207a0ce0..1afa378c23 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -406,6 +406,9 @@ def _trace_path_allowlist() -> tuple[str, ...]: "recipe_finalize_status", "recipe_finalize_attempts", "recipe_finalize_outcome", + # Measured stack incapability; forging it either revives analysis that cannot succeed or silences the one + # that can. + "gpu_trace_unsupported_reason", # KB tag completeness (Coordinator-populated; LLM reads via prompt). "stack_fingerprint_meta", "baseline_workload_extra", diff --git a/src/hyperloom/orchestrator/specialists/dispatch.py b/src/hyperloom/orchestrator/specialists/dispatch.py index debe0e60aa..a35a2b7842 100644 --- a/src/hyperloom/orchestrator/specialists/dispatch.py +++ b/src/hyperloom/orchestrator/specialists/dispatch.py @@ -24,7 +24,7 @@ PolicyDenied, validate_freeform_wave_task, ) -from .runner import SpecialistFailureType +from .runner import SpecialistFailureType, specialist_patch_preflight_error if TYPE_CHECKING: from ..loop.sub_agent_runner import SubAgentResult @@ -34,6 +34,8 @@ __all__ = ["SpecialistDispatchCollaborator"] +_SOURCE_PATCH_FAMILY = "source_patch" + class SpecialistDispatchCollaborator(CoordinatorCollaborator): """Specialist dispatch: warmup, auto-retry, wave fan-out, stalled-domain forcing, and round-entry construction.""" @@ -551,12 +553,49 @@ async def _maybe_force_stalled_domain_specialist(self) -> None: "source": "coordinator_internal", "reason": f"stalled_domain_force:{anchor}", } + from .profile import MODE_PATCH, resolve_specialist_profile + + is_source_patch = resolve_specialist_profile(params, domain=dom).mode == MODE_PATCH + if is_source_patch and state.is_pruned(_SOURCE_PATCH_FAMILY): + continue + idempotency_key = f"forced-stalled-{anchor}-round{round_id}{self._cycle_idem_suffix()}" + lookup = getattr(self.tasks, "find_by_idempotency_key", None) + if callable(lookup): + existing = await lookup(idempotency_key) + if existing is not None: + continue + await self._warm_specialist_params(params) + if is_source_patch: + preflight_error = specialist_patch_preflight_error( + params, + framework_repo_path=str(getattr(state, "framework_repo_path", "") or ""), + ) + if preflight_error: + if state.add_pruned_family(_SOURCE_PATCH_FAMILY): + state.record_action_failure( + action="specialist", + task_id=idempotency_key, + result={ + "error_class": preflight_error, + "error": preflight_error, + }, + ) + try: + state.save(self.session_dir) + except Exception: # noqa: BLE001 + log.exception("stalled-domain force: source-patch prune save failed") + log.error( + "stalled-domain force: pruned %s after deterministic failure: %s", + _SOURCE_PATCH_FAMILY, + preflight_error, + ) + continue intent = Intent( type=IntentType.DELEGATE, payload={ "action_name": "specialist", "params": params, - "idempotency_key": (f"forced-stalled-{anchor}-round{round_id}{self._cycle_idem_suffix()}"), + "idempotency_key": idempotency_key, }, ) # Zero the counter up-front so a slow enqueue can't re-fire next tick. diff --git a/src/hyperloom/orchestrator/specialists/runner.py b/src/hyperloom/orchestrator/specialists/runner.py index 1fd53632a1..a85171872d 100644 --- a/src/hyperloom/orchestrator/specialists/runner.py +++ b/src/hyperloom/orchestrator/specialists/runner.py @@ -40,6 +40,7 @@ SPECIALIST_DOMAIN_KEYS, SpecialistDomain, domain_for_tag, + get_domain, normalize_dispatch_tags, ) from .subprocess_ import ( @@ -62,6 +63,33 @@ log = logging.getLogger(__name__) +NO_GIT_FRAMEWORK_SOURCE_ROOT = "no_git_framework_source_root" + + +def specialist_patch_preflight_error( + params: dict[str, Any] | None, + *, + framework_repo_path: str = "", + framework_source_roots: tuple[str, ...] = (), +) -> str: + """Return the deterministic source-root error for a patch specialist.""" + task_params = params or {} + domain = get_domain(str(task_params.get("domain") or "")) + if resolve_specialist_profile(task_params, domain=domain).mode != MODE_PATCH: + return "" + roots = tuple( + path + for path in ( + str(framework_repo_path or "").strip(), + *(framework_source_roots or tuple(task_params.get("framework_source_roots") or ())), + ) + if path + ) + preferred = str(task_params.get("session_framework_tree") or "").strip() or resolve_framework_tree( + str(task_params.get("framework") or "") + ) + return "" if _pick_worktree_base(roots, preferred=preferred) is not None else NO_GIT_FRAMEWORK_SOURCE_ROOT + def _ctx_deadline(ctx: RunnerContext) -> Deadline | None: """Read the dispatch deadline off a task context. @@ -1503,12 +1531,16 @@ def _maybe_setup_worktree( if profile is not None: if profile.mode != MODE_PATCH: return None, None, "" + preflight_error = specialist_patch_preflight_error( + ctx.task.params, + framework_source_roots=self.subprocess_config.framework_source_roots, + ) + if preflight_error: + return None, None, preflight_error base = _pick_worktree_base( self.subprocess_config.framework_source_roots, preferred=resolve_framework_tree(str((ctx.task.params or {}).get("framework") or "")), ) - if base is None: - return None, None, "no_git_framework_source_root" worktree_path = workspace / "worktree" branch = f"specialist-{ctx.task.task_id}" wt, err = _setup_worktree(base, worktree_path, branch) @@ -1728,6 +1760,7 @@ def _write_specialist_done_partial( __all__ = [ + "NO_GIT_FRAMEWORK_SOURCE_ROOT", "RETRYABLE_SPECIALIST_FAILURES", "SPECIALIST_TOOL_DENYLIST", "SpecialistFailureType", @@ -1736,4 +1769,5 @@ def _write_specialist_done_partial( "SpecialistSubprocessConfig", "build_empty_specialist_done", "classify_specialist_failure", + "specialist_patch_preflight_error", ] diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 589fdf54a8..322c49e5d3 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -689,6 +689,9 @@ class SharedState(_RenderMixin, GapsStateMixin, _PhaseStateMixin): roofline_snapshots: list[dict[str, Any]] = field(default_factory=list) # Outer roofline failure counter; bumped on fail, reset on success. roofline_failure_streak: int = 0 + # Why this stack can never record GPU kernels; set once from a parsed trace with host ops and no kernels, so the + # Coordinator stops auto-enqueueing analysis that cannot succeed. Empty means GPU tracing is still presumed usable. + gpu_trace_unsupported_reason: str = "" # Feature toggles (mirrored from ``cli.py`` flags at session start). framework_agent_phase_enabled: bool = True diff --git a/src/kernelforge/gemm_tune/cli.py b/src/kernelforge/gemm_tune/cli.py index cc17e6a8cf..747b05ba66 100644 --- a/src/kernelforge/gemm_tune/cli.py +++ b/src/kernelforge/gemm_tune/cli.py @@ -429,7 +429,8 @@ def run( [g.gpu_id for g in busy], ) - # aiter tune/serve alignment preflight (warn-only, best-effort). + # aiter tune/serve alignment preflight. + preflight_hard: list[str] = [] try: from .aiter_preflight import collect as _aiter_collect @@ -439,10 +440,13 @@ def run( log.warning("aiter preflight: %s", _m) for _m in _pf["hard"]: log.warning("aiter preflight PROBLEM: %s", _m) + preflight_hard = list(_pf["hard"]) if _pf["aligned"]: log.info("aiter preflight: serve aiter aligned with tuner root") - except Exception as _exc: # noqa: BLE001 - preflight must never break tuning + except Exception as _exc: # noqa: BLE001 - unavailable diagnostics remain best-effort log.debug("aiter preflight skipped: %s", _exc) + if preflight_hard: + raise click.ClickException("aiter preflight failed: " + "; ".join(preflight_hard)) # Analyze model try: diff --git a/src/kernelforge/gemm_tune/tests/test_cli.py b/src/kernelforge/gemm_tune/tests/test_cli.py index 3b0cc240fb..f4acac984e 100644 --- a/src/kernelforge/gemm_tune/tests/test_cli.py +++ b/src/kernelforge/gemm_tune/tests/test_cli.py @@ -21,10 +21,10 @@ def _model_dir(tmp_path): return model -def _stub_preflight(monkeypatch): +def _stub_preflight(monkeypatch, *, soft=None, hard=None): monkeypatch.setattr( "kernelforge.gemm_tune.aiter_preflight.collect", - lambda: {"soft": [], "hard": [], "aligned": False}, + lambda: {"soft": soft or [], "hard": hard or [], "aligned": False}, ) @@ -39,7 +39,7 @@ def test_help_carries_no_knowledge_base_options(): def test_every_runnable_tuner_is_executed(tmp_path, monkeypatch): """Nothing may stand between a runnable tuner and a real tuning run.""" - _stub_preflight(monkeypatch) + _stub_preflight(monkeypatch, soft=["AITER_COMMIT unset"]) model = _model_dir(tmp_path) output = tmp_path / "output" executed: list[str] = [] @@ -80,6 +80,42 @@ def execute(self): assert all("kb_cache" not in t for t in report["tuners_run"]) +def test_hard_aiter_mismatch_aborts_before_tuning(tmp_path, monkeypatch): + _stub_preflight(monkeypatch, hard=["MISALIGNED: serving and tuner roots differ"]) + model = _model_dir(tmp_path) + executed = [] + + class _Tuner: + def execute(self): + executed.append(True) + return TuneResult(tuner_name="a8w8", status="no_improvement") + + monkeypatch.setattr(cli_mod, "_create_tuner", lambda _name, _ctx: _Tuner()) + result = CliRunner().invoke( + gemm_tune, + [ + "run", + "--model-path", + str(model), + "--framework", + "vllm", + "--precision", + "fp8", + "--gpu-type", + "mi300x", + "--tuner", + "a8w8", + "--skip-gpu-check", + "--output-dir", + str(tmp_path / "output"), + ], + ) + + assert result.exit_code != 0 + assert "MISALIGNED" in result.output + assert executed == [] + + def test_cli_resolves_auto_once_and_records_effective_gpu(tmp_path, monkeypatch): from kernelforge.gemm_tune import router