From 2fbc7bc6cd11024d1dc0f5df01072a9e81c24650 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 00:08:57 -0600 Subject: [PATCH 1/3] feat(bench): execute runtime candidates through Pier --- bench/HARNESS.md | 45 +- bench/README.md | 10 +- .../no-model-task/environment/Dockerfile | 16 + .../environment/seed/src/status.txt | 1 + .../pier-agent/no-model-task/instruction.md | 6 + .../pier-agent/no-model-task/pre_artifacts.sh | 6 + .../pier-agent/no-model-task/task.toml | 35 + .../pier-agent/no-model-task/tests/Dockerfile | 17 + .../no-model-task/tests/seed/src/status.txt | 1 + .../pier-agent/no-model-task/tests/test.sh | 19 + bench/package.json | 12 +- bench/pier_agents/__init__.py | 5 + bench/pier_agents/candidate_contract.py | 746 ++++++++++++++ bench/pier_agents/process_boundary.py | 321 +++++++ bench/pier_agents/tangle_candidate.py | 906 ++++++++++++++++++ bench/pier_agents/tangle_candidate_test.py | 693 ++++++++++++++ bench/pier_agents/workspace_boundary.py | 264 +++++ bench/pnpm-lock.yaml | 16 +- bench/scripts/run-package-tests.mjs | 4 +- bench/scripts/verify-packed-consumer.mjs | 87 +- bench/scripts/verify-pier-agent.mts | 744 ++++++++++++++ bench/scripts/verify-pier-pair.mts | 57 ++ bench/src/index.ts | 18 + bench/src/pier-agent.test.mts | 342 +++++++ bench/src/pier-agent.ts | 579 +++++++++++ bench/src/pier-result-grader.mjs | 30 + bench/src/pier-result-grader.test.mts | 62 ++ bench/src/pier-result-grader.ts | 108 +++ bench/src/pier-task-outcome.test.mts | 81 ++ bench/src/pier-task-outcome.ts | 234 +++++ bench/src/pier-workspace-archive.test.mts | 58 ++ bench/src/pier-workspace-archive.ts | 182 ++++ 32 files changed, 5687 insertions(+), 18 deletions(-) create mode 100644 bench/fixtures/pier-agent/no-model-task/environment/Dockerfile create mode 100644 bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt create mode 100644 bench/fixtures/pier-agent/no-model-task/instruction.md create mode 100755 bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh create mode 100644 bench/fixtures/pier-agent/no-model-task/task.toml create mode 100644 bench/fixtures/pier-agent/no-model-task/tests/Dockerfile create mode 100644 bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt create mode 100755 bench/fixtures/pier-agent/no-model-task/tests/test.sh create mode 100644 bench/pier_agents/__init__.py create mode 100644 bench/pier_agents/candidate_contract.py create mode 100644 bench/pier_agents/process_boundary.py create mode 100644 bench/pier_agents/tangle_candidate.py create mode 100644 bench/pier_agents/tangle_candidate_test.py create mode 100644 bench/pier_agents/workspace_boundary.py create mode 100644 bench/scripts/verify-pier-agent.mts create mode 100644 bench/scripts/verify-pier-pair.mts create mode 100644 bench/src/pier-agent.test.mts create mode 100644 bench/src/pier-agent.ts create mode 100644 bench/src/pier-result-grader.mjs create mode 100644 bench/src/pier-result-grader.test.mts create mode 100644 bench/src/pier-result-grader.ts create mode 100644 bench/src/pier-task-outcome.test.mts create mode 100644 bench/src/pier-task-outcome.ts create mode 100644 bench/src/pier-workspace-archive.test.mts create mode 100644 bench/src/pier-workspace-archive.ts diff --git a/bench/HARNESS.md b/bench/HARNESS.md index 681ce992..0ce1f4af 100644 --- a/bench/HARNESS.md +++ b/bench/HARNESS.md @@ -3,7 +3,7 @@ If you're an agent picking this up: read this page, then run `pnpm help` + `pnpm gate` — do NOT re-derive the harness from source. This map is SHORT on purpose; if it disagrees with the code, the code wins — fix this page in the same turn (the anti-rediscovery law). -Verified against source 2026-07-07 · agent-eval pinned `^0.106.1`. The CANONICAL surface is now +Verified against source 2026-07-09 · agent-eval pinned `^0.108.1`. The CANONICAL surface is now the published optimization suite (`@tangle-network/agent-runtime/loops`): `Environment` + `Strategy`/`defineStrategy` + `runBenchmark` — see the section below FIRST. The recursive diverse-vs-blind gate runs through the keystone (`gate-cli.mts` → `runGate`); @@ -134,6 +134,7 @@ the gate + measurement tools: run-benchmarks-cli.mts runBenchmarks: any subset of the ADAPTERS registry × model/harness cells, one combined ranked report (#420) commit0-env-run.mts the HARD domain through `runBenchmark` (the optimization suite) terminal-compare.ts Terminal-Bench compare (own main) + pnpm verify:pier zero-model failure/pass Pier controls through a separate verifier unit tests (the only fully-green, cred-free runnable surface besides offline replay): node --test --import tsx src/{selector,refine-loop}.test.mts tsx src/gate.test.mts # offline plumbing test (no creds) @@ -215,6 +216,48 @@ stdin-piping runner (`runVenvScriptStdin`). - **Absent (not built):** swe-gym, swe-bench-multimodal, and the rest of the survey set. Every unbuilt/scaffold adapter fails LOUD (throws with the integration step) rather than faking a score — no silent zeros in any corpus. Offline fixture tests: `benchmarks/{aec-bench,commit0,programbench,appworld,rag-benchmarks}.test.mts` (`tsx --test`). +## Pier candidate bridge + +`pier_agents.tangle_candidate:TangleCandidateAgent` is the reusable Pier custom-agent path for frozen Tangle candidates. +`executePreparedPierCandidate()` is the only public entry point; its private staging step writes the runtime's execution-plan and materialization-receipt bytes verbatim. +The Python bridge rechecks those bytes and their signed task, candidate, profile, repository, instruction, and workspace identities before launch. +It rejects a raw candidate bundle, never projects an `AgentProfile`, and leaves task isolation, patch transfer, verification, retries, and result storage to Pier. + +The adapter fails the trial when any prepared identity drifts, when the task checkout or immutable OCI image differs from the signed identity, or when the candidate exits nonzero or exceeds its signed deadline. +Candidate code runs as an unprivileged numeric user, while evaluator inputs and timeout evidence remain root-owned. +It never accepts candidate-authored token, cost, or trace receipts; `executePreparedPierCandidate()` uses the runtime's atomic execution path and reconciles the protected `TraceStore` with the model-gateway ledger before returning a gradable receipt. +The prepared object contains no credentials. +The runtime passes model and trace bindings only to the trusted executor request. +The Pier launcher inherits their values through its protected process environment and passes only `${NAME}` references on the command line, so credentials never enter prepared bytes, CLI arguments, or job files. +The executor builds fresh task, candidate, and profile trees from the request's exact verified file bytes; prepared staging directories are never launch authority. +Isolated memory and knowledge-bearing candidates currently fail closed until Pier has executor-owned mount and after-state capture. +The signed wall deadline is a hard stop: the runtime aborts, Pier kills the process tree, and the executor acknowledges process and container death. +The signed tool-step count is a post-run validity check over protected traces, not a pre-tool stop; generic black-box Pier processes cannot honestly prevent step N+1. +The executable zero-model fixture is `fixtures/pier-agent/`; run it against the R360 Pier checkout with `PIER_REPO=/path/to/pier pnpm verify:pier`. +That command runs both a no-change candidate that must score 0/1 and a known-good candidate that must score 1/1, then checks that each official result and exact task patch is bound into its own runtime receipt with zero model usage. + +For a real frozen candidate, start Pier synchronously inside the atomic callback, append `agentArgs` and `attemptArgs`, pass each executor-only `evaluatorEnv` entry through Pier's evaluator-owned environment mechanism, and return a handle that can kill and reap the process and remove its task container: + +```ts +const result = await executePreparedPierCandidate({ + prepared, + directory: '/sealed/candidate', + pierVersion: '0.3.0', + traceStore, + claimStore, + outputArtifacts, + grader, + start: (staged, context) => startOnePierTrial(staged, context), +}) +``` + +The handle's result resolves only after normal process/container cleanup. +On a signed deadline or external abort, the adapter waits for `terminateAndWait()` to acknowledge both process exit and container removal before it returns control to the runtime. + +One prepared execution always maps to one Pier attempt (`--n-attempts 1 --max-retries 0`). +Production callers pass a long-lived `FileAgentCandidateExecutionClaimStore`; an in-memory claim store is test-only and cannot prevent a second process from replaying the same attempt. +Any allowed pre-model infrastructure retry is a new prepared execution with its own counted attempt identity. + ## Is it runnable RIGHT NOW? (verify the map, don't trust it blindly) ``` ls src/*.mts src/*.ts # the real tool list (each its own main — source of truth) diff --git a/bench/README.md b/bench/README.md index 522525fc..1175b010 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,6 +1,6 @@ # agent-runtime-bench -Private experiment workspace nested in agent-runtime; decoupled from its build/lint/release (the package builds `src/`, lints `src tests examples` — `bench/` is none of those). +Published as `@tangle-network/agent-bench`, with independent CI and release checks for its TypeScript and Python surfaces. **Read [`bench/HARNESS.md`](./HARNESS.md) FIRST.** It is the one maintained map: the commands, the `rollout → corpus → selector → CI → gate` data flow, the canonical-suite table, the wired/needs-creds/scaffolded matrix, and the gate one-liners — kept verified against source. @@ -13,3 +13,11 @@ pnpm install # tsx + link parent ``` The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API_KEY`, or a direct provider). + +## Pier custom candidates + +The package executes a branded `PreparedAgentCandidateExecution` from `@tangle-network/agent-runtime` through one atomic API and ships `pier_agents.tangle_candidate:TangleCandidateAgent` as its thin Pier transport. +The executor recreates every input from runtime-verified file bytes and reveals model credentials only inside the claimed execution callback. +Pier owns the task container and verifier; protected model usage and traces stay in `@tangle-network/agent-eval` and are finalized by the shared runtime. +Run `PIER_REPO=/path/to/pier pnpm verify:pier` for the zero-model failure/pass proof, and see `HARNESS.md` for the exact invocation and failure contract. +From an installed npm package, expose the shipped Python module with `export PYTHONPATH="$(npm root)/@tangle-network/agent-bench${PYTHONPATH:+:$PYTHONPATH}"` before invoking Pier. diff --git a/bench/fixtures/pier-agent/no-model-task/environment/Dockerfile b/bench/fixtures/pier-agent/no-model-task/environment/Dockerfile new file mode 100644 index 00000000..775ff6ce --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/environment/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY seed/ /app/ +RUN git init \ + && git config user.email fixture@tangle.tools \ + && git config user.name "Tangle Fixture" \ + && git add -A \ + && GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \ + GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + git commit -m baseline \ + && git tag benchmark-base diff --git a/bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt b/bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt new file mode 100644 index 00000000..aee83dc4 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/environment/seed/src/status.txt @@ -0,0 +1 @@ +not-ready diff --git a/bench/fixtures/pier-agent/no-model-task/instruction.md b/bench/fixtures/pier-agent/no-model-task/instruction.md new file mode 100644 index 00000000..1ab44251 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/instruction.md @@ -0,0 +1,6 @@ +Change src/status.txt so it contains exactly: + +ready +owner=tangle + +Commit the change to git when finished. diff --git a/bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh b/bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh new file mode 100755 index 00000000..b9cc6aec --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/pre_artifacts.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p /logs/artifacts +cd /app +git -c safe.directory=/app diff --binary benchmark-base..HEAD > /logs/artifacts/model.patch diff --git a/bench/fixtures/pier-agent/no-model-task/task.toml b/bench/fixtures/pier-agent/no-model-task/task.toml new file mode 100644 index 00000000..2080342b --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/task.toml @@ -0,0 +1,35 @@ +version = "1.0" + +[task] +name = "agent-bench/pier-candidate-no-model" +authors = [] +keywords = ["pier", "candidate", "no-model"] + +[metadata] +author_name = "Tangle Research" +author_email = "research@tangle.tools" +difficulty = "synthetic" +category = "software-engineering" +tags = ["no-model", "separate-verifier"] + +[agent] +timeout_sec = 60.0 + +[verifier] +timeout_sec = 60.0 +environment_mode = "separate" + +[verifier.environment] + +[environment] +build_timeout_sec = 300.0 +cpus = 1 +memory_mb = 1024 +storage_mb = 2048 +gpus = 0 +allow_internet = false +mcp_servers = [] + +[verifier.env] + +[solution.env] diff --git a/bench/fixtures/pier-agent/no-model-task/tests/Dockerfile b/bench/fixtures/pier-agent/no-model-task/tests/Dockerfile new file mode 100644 index 00000000..323cf830 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/tests/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY seed/ /app/ +RUN git init \ + && git config user.email fixture@tangle.tools \ + && git config user.name "Tangle Fixture" \ + && git add -A \ + && GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \ + GIT_COMMITTER_DATE=2000-01-01T00:00:00Z \ + git commit -m baseline \ + && git tag benchmark-base +COPY test.sh /tests/test.sh diff --git a/bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt b/bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt new file mode 100644 index 00000000..aee83dc4 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/tests/seed/src/status.txt @@ -0,0 +1 @@ +not-ready diff --git a/bench/fixtures/pier-agent/no-model-task/tests/test.sh b/bench/fixtures/pier-agent/no-model-task/tests/test.sh new file mode 100755 index 00000000..bdd162f6 --- /dev/null +++ b/bench/fixtures/pier-agent/no-model-task/tests/test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +reward=0 +patch_applied=0 +patch=/logs/artifacts/model.patch + +if [ -s "$patch" ] \ + && git -c safe.directory=/app apply --check "$patch" \ + && git -c safe.directory=/app apply "$patch"; then + patch_applied=1 +fi +if [ "$(cat /app/src/status.txt 2>/dev/null || true)" = $'ready\nowner=tangle' ] \ + && [ ! -e /app/AGENTS.md ]; then + reward=1 +fi + +printf '{"reward":%d,"patch_applied":%d}\n' "$reward" "$patch_applied" \ + > /logs/verifier/reward.json diff --git a/bench/package.json b/bench/package.json index ff3b5359..b3db33e5 100644 --- a/bench/package.json +++ b/bench/package.json @@ -18,10 +18,12 @@ "terminal-compare": "tsx src/terminal-compare.ts", "test": "node scripts/run-package-tests.mjs", "typecheck:public": "tsc -p tsconfig.public.json", - "verify:package": "node scripts/verify-packed-consumer.mjs" + "verify:package": "node scripts/verify-packed-consumer.mjs", + "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { "@tangle-network/agent-eval": "^0.108.1", + "@tangle-network/agent-interface": "^0.23.0", "@tangle-network/agent-runtime": "^0.90.1", "@tangle-network/sandbox": "^0.9.7" }, @@ -34,9 +36,15 @@ "src", "fixtures", "scripts", + "pier_agents/__init__.py", + "pier_agents/candidate_contract.py", + "pier_agents/process_boundary.py", + "pier_agents/tangle_candidate.py", + "pier_agents/workspace_boundary.py", "tb_agents/*.py", "steerers", - "README.md" + "README.md", + "HARNESS.md" ], "publishConfig": { "access": "public" diff --git a/bench/pier_agents/__init__.py b/bench/pier_agents/__init__.py new file mode 100644 index 00000000..6f65e392 --- /dev/null +++ b/bench/pier_agents/__init__.py @@ -0,0 +1,5 @@ +"""Pier custom agents shipped by ``@tangle-network/agent-bench``.""" + +from .tangle_candidate import TangleCandidateAgent + +__all__ = ["TangleCandidateAgent"] diff --git a/bench/pier_agents/candidate_contract.py b/bench/pier_agents/candidate_contract.py new file mode 100644 index 00000000..ce52275e --- /dev/null +++ b/bench/pier_agents/candidate_contract.py @@ -0,0 +1,746 @@ +"""Strict reader for runtime-prepared candidate execution artifacts. + +The TypeScript runtime is the authority that verifies bundles and creates the +canonical execution plan and materialization receipt. This module performs the +small, security-critical replay checks needed before Pier copies those prepared +bytes into a task container. It deliberately does not define another plan. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +import re +import stat +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + + +_SHA256_RE = re.compile(r"^sha256:[a-f0-9]{64}$") +_GIT_OBJECT_RE = re.compile(r"^(?:[a-f0-9]{40}|[a-f0-9]{64})$") +_ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_EXECUTABLE_RE = re.compile(r"^[A-Za-z0-9._+/-]+$") +_DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") +_SHELLS = {"sh", "bash", "zsh", "fish", "cmd", "cmd.exe", "powershell", "pwsh"} +_BLOCKED_GATEWAY_DOMAINS = { + "localhost", + "metadata.google.internal", + "metadata.google", +} +_RESERVED_PATH_PARTS = {".git", ".sidecar"} +_PLAN_KIND = "agent-candidate-execution-plan-material" +_RECEIPT_KIND = "agent-candidate-materialization" +_PROFILE_PLAN_KIND = "agent-profile-workspace-plan" +_EXECUTION_EVIDENCE_KIND = "agent-candidate-execution-plan" +_STDIN_TASK_PATH = "/tangle/input/stdin.txt" + + +class CandidateContractError(ValueError): + """Prepared execution bytes are missing, malformed, or inconsistent.""" + + +@dataclass(frozen=True) +class InstructionEvidence: + sha256: str + byte_length: int + delivery_kind: str + delivery_env: str | None = None + delivery_path: str | None = None + + +@dataclass(frozen=True) +class WorkspaceFile: + path: str + mode: int + sha256: str + byte_length: int + + +@dataclass(frozen=True) +class ProfileFile: + path: str + mode: int + sha256: str + + +@dataclass(frozen=True) +class PreparedCandidateContract: + plan: dict[str, Any] + plan_raw: bytes + receipt: dict[str, Any] + receipt_raw: bytes + bundle_digest: str + execution_id: str + execution_plan_digest: str + materialization_receipt_digest: str + instruction: InstructionEvidence + repository_base_commit: str + repository_base_tree: str + task_root: str + candidate_root: str | None + task_files: tuple[WorkspaceFile, ...] + candidate_files: tuple[WorkspaceFile, ...] + profile_target: str + profile_files: tuple[ProfileFile, ...] + executable: str + args: tuple[str, ...] + env: dict[str, str] + cwd_root: str + cwd_path: str + timeout_ms: int + requested_model: str + resolved_model: dict[str, Any] + model_network_domains: tuple[str, ...] + container: dict[str, Any] + + +def sha256_bytes(data: bytes) -> str: + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +def _object(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise CandidateContractError(f"{label} must be a JSON object") + return value + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or "\0" in value: + raise CandidateContractError(f"{label} must be a non-empty string without NUL") + return value + + +def _integer(value: Any, label: str, *, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise CandidateContractError(f"{label} must be an integer >= {minimum}") + return value + + +def _number(value: Any, label: str, *, minimum: float = 0) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < minimum + ): + raise CandidateContractError(f"{label} must be a finite number >= {minimum}") + return float(value) + + +def _digest(value: Any, label: str) -> str: + digest = _string(value, label) + if not _SHA256_RE.fullmatch(digest): + raise CandidateContractError(f"{label} must be sha256:<64 lowercase hex>") + return digest + + +def _git_object(value: Any, label: str) -> str: + object_id = _string(value, label) + if not _GIT_OBJECT_RE.fullmatch(object_id): + raise CandidateContractError(f"{label} must be a full Git object id") + return object_id + + +def _safe_relative(value: Any, label: str, *, allow_dot: bool = False) -> str: + path = _string(value, label) + if any(ord(char) < 32 or ord(char) == 127 for char in path) or "\\" in path: + raise CandidateContractError(f"{label} contains a forbidden character") + parsed = PurePosixPath(path) + if parsed.is_absolute(): + raise CandidateContractError(f"{label} must be relative") + if path == ".": + if allow_dot: + return path + raise CandidateContractError(f"{label} must identify a child path") + parts = path.split("/") + if any( + not part or part in {".", ".."} or part in _RESERVED_PATH_PARTS + for part in parts + ): + raise CandidateContractError(f"{label} must be a canonical safe relative path") + return path + + +def _absolute(value: Any, label: str) -> str: + path = _string(value, label) + if any(ord(char) < 32 or ord(char) == 127 for char in path) or "\\" in path: + raise CandidateContractError(f"{label} contains a forbidden character") + parsed = PurePosixPath(path) + if ( + not parsed.is_absolute() + or path == "/" + or ".." in parsed.parts + or str(parsed) != path + ): + raise CandidateContractError(f"{label} must be a canonical absolute POSIX path") + return path + + +def _public(value: Any, label: str) -> str: + obj = _object(value, label) + if set(obj) != {"kind", "value"} or obj.get("kind") != "public": + raise CandidateContractError(f"{label} must be one explicit public value") + return _string(obj.get("value"), f"{label}.value") + + +def _gateway_domain(value: Any, label: str) -> str: + domain = _string(value, label) + labels = domain.split(".") + if ( + len(domain) > 253 + or domain != domain.lower() + or domain.endswith(".") + or ":" in domain + or domain in _BLOCKED_GATEWAY_DOMAINS + or domain.endswith(".localhost") + or len(labels) < 2 + or not all(1 <= len(part) <= 63 and _DNS_LABEL_RE.fullmatch(part) for part in labels) + or not re.search(r"[a-z]", labels[-1]) + ): + raise CandidateContractError( + f"{label} must be an exact lowercase public DNS name" + ) + return domain + + +def _read_json(path: Path, label: str) -> tuple[dict[str, Any], bytes]: + try: + raw = path.read_bytes() + except OSError as exc: + raise CandidateContractError(f"cannot read {label} {path}: {exc}") from exc + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CandidateContractError(f"{label} is not valid UTF-8 JSON: {exc}") from exc + return _object(value, label), raw + + +def _embedded_bytes(value: Any, label: str) -> bytes: + artifact = _object(value, label) + if artifact.get("encoding") != "base64" or not isinstance( + artifact.get("content"), str + ): + raise CandidateContractError(f"{label} must embed exact base64 bytes") + try: + raw = base64.b64decode(artifact["content"], validate=True) + except ValueError as exc: + raise CandidateContractError(f"{label} contains invalid base64") from exc + expected_digest = _digest(artifact.get("sha256"), f"{label}.sha256") + expected_length = _integer(artifact.get("byteLength"), f"{label}.byteLength") + if len(raw) != expected_length or sha256_bytes(raw) != expected_digest: + raise CandidateContractError(f"{label} bytes do not match their identity") + return raw + + +def _workspace_files(value: Any, label: str) -> tuple[WorkspaceFile, ...]: + snapshot = _object(value, label) + if ( + snapshot.get("schemaVersion") != 1 + or snapshot.get("kind") != "agent-candidate-workspace-snapshot" + ): + raise CandidateContractError(f"{label} has the wrong snapshot kind") + material = _object(snapshot.get("material"), f"{label}.material") + if ( + material.get("schemaVersion") != 1 + or material.get("kind") != "agent-candidate-workspace-manifest" + ): + raise CandidateContractError(f"{label}.material has the wrong manifest kind") + values = material.get("files") + if not isinstance(values, list): + raise CandidateContractError(f"{label}.material.files must be an array") + files: list[WorkspaceFile] = [] + for index, value in enumerate(values): + obj = _object(value, f"{label}.material.files[{index}]") + mode = _integer(obj.get("mode"), f"{label}.material.files[{index}].mode") + if mode not in {0o644, 0o755}: + raise CandidateContractError( + f"{label} contains unsupported file mode {mode:o}" + ) + files.append( + WorkspaceFile( + path=_safe_relative( + obj.get("path"), f"{label}.material.files[{index}].path" + ), + mode=mode, + sha256=_digest( + obj.get("sha256"), f"{label}.material.files[{index}].sha256" + ), + byte_length=_integer( + obj.get("byteLength"), f"{label}.material.files[{index}].byteLength" + ), + ) + ) + paths = [file.path for file in files] + if paths != sorted(set(paths)): + raise CandidateContractError(f"{label} file paths must be unique and sorted") + return tuple(files) + + +def _profile_files(value: Any) -> tuple[ProfileFile, ...]: + values = value.get("files") if isinstance(value, dict) else None + if not isinstance(values, list): + raise CandidateContractError("profilePlan.material.files must be an array") + files: list[ProfileFile] = [] + for index, value in enumerate(values): + obj = _object(value, f"profilePlan.material.files[{index}]") + mode = _integer(obj.get("mode"), f"profilePlan.material.files[{index}].mode") + if mode not in {0o644, 0o755}: + raise CandidateContractError( + f"profile plan contains unsupported mode {mode:o}" + ) + files.append( + ProfileFile( + path=_safe_relative( + obj.get("relPath"), f"profilePlan.material.files[{index}].relPath" + ), + mode=mode, + sha256=_digest( + obj.get("contentSha256"), + f"profilePlan.material.files[{index}].contentSha256", + ), + ) + ) + paths = [file.path for file in files] + if paths != sorted(set(paths)): + raise CandidateContractError( + "profile plan file paths must be unique and sorted" + ) + return tuple(files) + + +def _instruction(value: Any) -> InstructionEvidence: + obj = _object(value, "task.instruction") + if obj.get("encoding") != "utf8": + raise CandidateContractError("task.instruction.encoding must equal utf8") + delivery = _object(obj.get("delivery"), "task.instruction.delivery") + kind = delivery.get("kind") + if kind == "argv-append" or kind == "stdin-utf8": + if set(delivery) != {"kind"}: + raise CandidateContractError(f"{kind} delivery has unexpected fields") + env = path = None + elif kind == "utf8-file": + if ( + delivery.get("env") != "TANGLE_CANDIDATE_TASK_PATH" + or delivery.get("path") != "/tangle/input/task.txt" + ): + raise CandidateContractError( + "utf8-file delivery must use the fixed env and path" + ) + env = "TANGLE_CANDIDATE_TASK_PATH" + path = "/tangle/input/task.txt" + else: + raise CandidateContractError("unsupported task instruction delivery") + return InstructionEvidence( + sha256=_digest(obj.get("sha256"), "task.instruction.sha256"), + byte_length=_integer( + obj.get("byteLength"), "task.instruction.byteLength", minimum=1 + ), + delivery_kind=kind, + delivery_env=env, + delivery_path=path, + ) + + +def load_prepared_candidate_contract( + plan_path: Path, + receipt_path: Path, + expected_receipt_digest: str, +) -> PreparedCandidateContract: + """Load exact runtime bytes and prove the receipt binds the plan.""" + + plan, plan_raw = _read_json(plan_path, "execution plan") + receipt, receipt_raw = _read_json(receipt_path, "materialization receipt") + receipt_digest = _digest(expected_receipt_digest, "expected receipt digest") + if sha256_bytes(receipt_raw) != receipt_digest: + raise CandidateContractError( + "materialization receipt bytes do not match the expected digest" + ) + if plan.get("schemaVersion") != 1 or plan.get("kind") != _PLAN_KIND: + raise CandidateContractError("execution plan has the wrong schema or kind") + if receipt.get("schemaVersion") != 1 or receipt.get("kind") != _RECEIPT_KIND: + raise CandidateContractError( + "materialization receipt has the wrong schema or kind" + ) + if receipt.get("digestAlgorithm") != "rfc8785-sha256" or "digest" in receipt: + raise CandidateContractError( + "materialization receipt must be exact digest-free canonical material" + ) + + execution_evidence = _object(receipt.get("executionPlan"), "receipt.executionPlan") + plan_digest = _digest( + execution_evidence.get("digest"), "receipt.executionPlan.digest" + ) + if ( + execution_evidence.get("schemaVersion") != 1 + or execution_evidence.get("kind") != _EXECUTION_EVIDENCE_KIND + ): + raise CandidateContractError("receipt execution evidence has the wrong kind") + if ( + sha256_bytes(plan_raw) != plan_digest + or execution_evidence.get("material") != plan + ): + raise CandidateContractError( + "receipt does not bind the exact execution plan material" + ) + if ( + _embedded_bytes( + execution_evidence.get("artifact"), "receipt.executionPlan.artifact" + ) + != plan_raw + ): + raise CandidateContractError( + "receipt execution-plan artifact differs from the executed bytes" + ) + + bundle_digest = _digest(plan.get("bundleDigest"), "plan.bundleDigest") + if receipt.get("bundleDigest") != bundle_digest: + raise CandidateContractError("receipt and plan bundle digests differ") + execution_id = _string(plan.get("executionId"), "plan.executionId") + + task = _object(plan.get("task"), "plan.task") + repository = _object(task.get("repository"), "plan.task.repository") + base_commit = _git_object( + repository.get("baseCommit"), "plan.task.repository.baseCommit" + ) + base_tree = _git_object(repository.get("baseTree"), "plan.task.repository.baseTree") + if len(base_commit) != len(base_tree): + raise CandidateContractError("task Git objects use different hash formats") + instruction = _instruction(task.get("instruction")) + task_files = _workspace_files(task.get("workspace"), "plan.task.workspace") + if not task_files: + raise CandidateContractError("task workspace cannot be empty") + + roots = _object(plan.get("workspaces"), "plan.workspaces") + task_root = _absolute(roots.get("taskRoot"), "plan.workspaces.taskRoot") + candidate_root_value = roots.get("candidateRoot") + candidate_root = ( + _absolute(candidate_root_value, "plan.workspaces.candidateRoot") + if candidate_root_value is not None + else None + ) + if candidate_root is not None and ( + candidate_root == task_root + or candidate_root.startswith(f"{task_root}/") + or task_root.startswith(f"{candidate_root}/") + ): + raise CandidateContractError("task and candidate workspace roots overlap") + protected_instruction_path = ( + instruction.delivery_path + if instruction.delivery_path is not None + else _STDIN_TASK_PATH + if instruction.delivery_kind == "stdin-utf8" + else None + ) + if protected_instruction_path is not None and ( + protected_instruction_path == task_root + or protected_instruction_path.startswith(f"{task_root}/") + or task_root.startswith(f"{protected_instruction_path}/") + or ( + candidate_root is not None + and ( + protected_instruction_path == candidate_root + or protected_instruction_path.startswith(f"{candidate_root}/") + or candidate_root.startswith(f"{protected_instruction_path}/") + ) + ) + ): + raise CandidateContractError("instruction path overlaps an execution workspace") + + code_kind = plan.get("codeKind") + candidate_snapshot = plan.get("candidateWorkspace") + active_code = code_kind in {"no-op", "git-patch"} + if code_kind not in {"disabled", "no-op", "git-patch"}: + raise CandidateContractError("plan.codeKind is unsupported") + if active_code != (candidate_snapshot is not None) or active_code != ( + candidate_root is not None + ): + raise CandidateContractError( + "active code and candidate workspace presence disagree" + ) + candidate_files = ( + _workspace_files(candidate_snapshot, "plan.candidateWorkspace") + if candidate_snapshot is not None + else () + ) + if active_code and not candidate_files: + raise CandidateContractError("active candidate workspace cannot be empty") + if receipt.get("candidateWorkspace") != candidate_snapshot: + raise CandidateContractError("receipt and plan candidate workspaces differ") + + profile_evidence = _object(receipt.get("profilePlan"), "receipt.profilePlan") + if ( + profile_evidence.get("schemaVersion") != 1 + or profile_evidence.get("kind") != _PROFILE_PLAN_KIND + ): + raise CandidateContractError("receipt profile evidence has the wrong kind") + profile_digest = _digest( + profile_evidence.get("digest"), "receipt.profilePlan.digest" + ) + profile_raw = _embedded_bytes( + profile_evidence.get("artifact"), "receipt.profilePlan.artifact" + ) + if sha256_bytes(profile_raw) != profile_digest: + raise CandidateContractError("profile artifact does not match its digest") + try: + profile_artifact_material = json.loads(profile_raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise CandidateContractError("profile artifact is not UTF-8 JSON") from exc + profile_material = _object( + profile_evidence.get("material"), "receipt.profilePlan.material" + ) + if profile_artifact_material != profile_material: + raise CandidateContractError("profile artifact and material differ") + profile_files = _profile_files(profile_material) + profile_application = _object(plan.get("profile"), "plan.profile") + if profile_application.get("planDigest") != profile_digest: + raise CandidateContractError("plan does not bind the profile digest") + profile_target = profile_application.get("targetWorkspace") + if profile_target not in {"task", "candidate"}: + raise CandidateContractError("profile target workspace is unsupported") + if profile_target == "candidate" and candidate_root is None: + raise CandidateContractError( + "candidate-targeted profile lacks a candidate workspace" + ) + if profile_application.get("mountPaths") != [file.path for file in profile_files]: + raise CandidateContractError( + "profile mount paths differ from the exact profile files" + ) + + launch = _object(plan.get("launch"), "plan.launch") + executable = _string(launch.get("executable"), "plan.launch.executable") + executable_parts = executable.split("/") + if executable.startswith("/"): + executable_parts = executable_parts[1:] + if ( + not _EXECUTABLE_RE.fullmatch(executable) + or not executable_parts + or any(part in {"", ".", ".."} for part in executable_parts) + or executable_parts[-1].lower() in _SHELLS + ): + raise CandidateContractError("plan launch executable is unsafe") + args_value = launch.get("args") + if not isinstance(args_value, list): + raise CandidateContractError("plan.launch.args must be an array") + args = tuple( + _public(value, f"plan.launch.args[{index}]") + for index, value in enumerate(args_value) + ) + env_value = _object(launch.get("env"), "plan.launch.env") + env: dict[str, str] = {} + for name, value in env_value.items(): + if not _ENV_RE.fullmatch(name): + raise CandidateContractError(f"invalid launch environment name: {name}") + env[name] = _public(value, f"plan.launch.env.{name}") + if instruction.delivery_kind == "utf8-file": + if env.get("TANGLE_CANDIDATE_TASK_PATH") != instruction.delivery_path: + raise CandidateContractError( + "launch env does not bind the fixed instruction path" + ) + elif "TANGLE_CANDIDATE_TASK_PATH" in env: + raise CandidateContractError( + "non-file instruction delivery exposes a task path" + ) + + cwd = _object(launch.get("cwd"), "plan.launch.cwd") + cwd_workspace = cwd.get("workspace") + if cwd_workspace == "task": + cwd_root = task_root + elif cwd_workspace == "candidate" and candidate_root is not None: + cwd_root = candidate_root + else: + raise CandidateContractError("launch cwd names an unavailable workspace") + cwd_path = _safe_relative(cwd.get("path"), "plan.launch.cwd.path", allow_dot=True) + limits = _object(plan.get("limits"), "plan.limits") + timeout_ms = _integer(limits.get("timeoutMs"), "plan.limits.timeoutMs", minimum=1) + _integer(limits.get("maxSteps"), "plan.limits.maxSteps", minimum=1) + max_model_calls = _integer( + limits.get("maxModelCalls"), "plan.limits.maxModelCalls" + ) + _integer(limits.get("maxInputTokens"), "plan.limits.maxInputTokens") + _integer(limits.get("maxOutputTokens"), "plan.limits.maxOutputTokens") + _number(limits.get("maxCostUsd"), "plan.limits.maxCostUsd") + attempt = _object(plan.get("attempt"), "plan.attempt") + _integer(attempt.get("number"), "plan.attempt.number", minimum=1) + _integer(attempt.get("maxAttempts"), "plan.attempt.maxAttempts", minimum=1) + if attempt.get("retryPolicy") != "none": + raise CandidateContractError("Pier execution requires runtime-owned retries") + container = _object(plan.get("container"), "plan.container") + if receipt.get("container") != container: + raise CandidateContractError("receipt and plan container identities differ") + _digest(container.get("indexDigest"), "plan.container.indexDigest") + _digest(container.get("manifestDigest"), "plan.container.manifestDigest") + image = _string(container.get("image"), "plan.container.image") + if "@" in image or "://" in image or any(char.isspace() for char in image): + raise CandidateContractError( + "container image must be an unpinned OCI reference without credentials" + ) + if container.get("source") not in { + "pinned-container", + "evaluator-task-container", + }: + raise CandidateContractError("plan container source is unsupported") + platform = _object(container.get("platform"), "plan.container.platform") + if platform.get("os") != "linux" or platform.get("architecture") not in { + "amd64", + "arm64", + }: + raise CandidateContractError("Pier adapter requires a supported Linux platform") + model = _object(plan.get("model"), "plan.model") + resolved_model = _object(model.get("resolved"), "plan.model.resolved") + requested_model = _string( + resolved_model.get("requested"), "plan.model.resolved.requested" + ) + _string(resolved_model.get("provider"), "plan.model.resolved.provider") + _string(resolved_model.get("model"), "plan.model.resolved.model") + _string(resolved_model.get("snapshot"), "plan.model.resolved.snapshot") + if receipt.get("resolvedModel") != resolved_model: + raise CandidateContractError("receipt and plan resolved models differ") + model_access = _object(model.get("access"), "plan.model.access") + if set(model_access) != {"kind", "grantDigest", "network"}: + raise CandidateContractError("plan.model.access has unexpected fields") + if model_access.get("kind") != "evaluator-mediated": + raise CandidateContractError( + "plan.model.access.kind must equal evaluator-mediated" + ) + _digest(model_access.get("grantDigest"), "plan.model.access.grantDigest") + model_network = _object( + model_access.get("network"), "plan.model.access.network" + ) + if max_model_calls == 0: + if model_network != {"mode": "disabled"}: + raise CandidateContractError( + "zero-call plans cannot expose a model gateway" + ) + model_network_domains: tuple[str, ...] = () + else: + if set(model_network) != {"mode", "domains"} or model_network.get( + "mode" + ) != "gateway-only": + raise CandidateContractError( + "model-calling plans require one frozen gateway allowlist" + ) + domain_values = model_network.get("domains") + if not isinstance(domain_values, list) or not domain_values: + raise CandidateContractError( + "plan.model.access.network.domains must be a non-empty array" + ) + model_network_domains = tuple( + _gateway_domain( + value, f"plan.model.access.network.domains[{index}]" + ) + for index, value in enumerate(domain_values) + ) + if list(model_network_domains) != sorted(set(model_network_domains)): + raise CandidateContractError( + "model gateway domains must be unique and sorted" + ) + if receipt.get("harness") != plan.get("harness") or receipt.get( + "harnessVersion" + ) != plan.get("harnessVersion"): + raise CandidateContractError("receipt and plan harness identities differ") + if receipt.get("codeKind") != code_kind: + raise CandidateContractError("receipt and plan code kinds differ") + memory = _object(plan.get("memory"), "plan.memory") + if memory != {"mode": "disabled"}: + raise CandidateContractError( + "Pier adapter does not yet support isolated candidate memory" + ) + if plan.get("knowledgeManifestDigest") is not None: + raise CandidateContractError( + "Pier adapter does not yet support candidate knowledge snapshots" + ) + network = _object(plan.get("network"), "plan.network") + if network != {"mode": "disabled"}: + raise CandidateContractError( + "Pier candidate execution requires disabled network" + ) + + return PreparedCandidateContract( + plan=plan, + plan_raw=plan_raw, + receipt=receipt, + receipt_raw=receipt_raw, + bundle_digest=bundle_digest, + execution_id=execution_id, + execution_plan_digest=plan_digest, + materialization_receipt_digest=receipt_digest, + instruction=instruction, + repository_base_commit=base_commit, + repository_base_tree=base_tree, + task_root=task_root, + candidate_root=candidate_root, + task_files=task_files, + candidate_files=candidate_files, + profile_target=profile_target, + profile_files=profile_files, + executable=executable, + args=args, + env=env, + cwd_root=cwd_root, + cwd_path=cwd_path, + timeout_ms=timeout_ms, + requested_model=requested_model, + resolved_model=resolved_model, + model_network_domains=model_network_domains, + container=container, + ) + + +def verify_local_files( + root: Path, + expected: tuple[WorkspaceFile, ...] | tuple[ProfileFile, ...], + label: str, +) -> None: + """Require exact regular files, modes, and bytes below one staging root.""" + + if root.is_symlink() or not root.is_dir(): + raise CandidateContractError(f"{label} must be a real directory: {root}") + if root.resolve(strict=True) != root.absolute(): + raise CandidateContractError(f"{label} has a symlinked path component: {root}") + observed: dict[str, tuple[int, str, int]] = {} + for entry in root.rglob("*"): + relative = entry.relative_to(root).as_posix() + info = entry.lstat() + if stat.S_ISLNK(info.st_mode): + raise CandidateContractError(f"{label} contains a symlink: {relative}") + if stat.S_ISDIR(info.st_mode): + continue + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise CandidateContractError( + f"{label} contains a non-regular or hard-linked file: {relative}" + ) + raw = entry.read_bytes() + observed[relative] = (stat.S_IMODE(info.st_mode), sha256_bytes(raw), len(raw)) + expected_map = {file.path: file for file in expected} + if set(observed) != set(expected_map): + raise CandidateContractError( + f"{label} file set differs from the signed manifest" + ) + for path, expected_file in expected_map.items(): + actual = observed[path] + if isinstance(expected_file, ProfileFile): + if actual[:2] != (expected_file.mode, expected_file.sha256): + raise CandidateContractError(f"{label} bytes or mode differ: {path}") + elif actual != ( + expected_file.mode, + expected_file.sha256, + expected_file.byte_length, + ): + raise CandidateContractError( + f"{label} bytes, length, or mode differ: {path}" + ) + + +def immutable_snapshot( + source: Path, destination: Path, expected: tuple[Any, ...], label: str +) -> None: + """Copy a verified staging tree and prove it did not change during capture.""" + + import shutil + + verify_local_files(source, expected, label) + shutil.copytree(source, destination, symlinks=True, copy_function=shutil.copy2) + verify_local_files(destination, expected, f"{label} snapshot") diff --git a/bench/pier_agents/process_boundary.py b/bench/pier_agents/process_boundary.py new file mode 100644 index 00000000..ec91cc07 --- /dev/null +++ b/bench/pier_agents/process_boundary.py @@ -0,0 +1,321 @@ +"""Evaluator-owned process isolation for one Pier candidate invocation.""" + +from __future__ import annotations + +import secrets +import shlex +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from pier.environments.base import BaseEnvironment + +from .candidate_contract import sha256_bytes + + +class ProcessBoundaryHost(Protocol): + async def _exec( + self, environment: BaseEnvironment, command: str, **kwargs: Any + ) -> Any: ... + + async def _ensure_real_directory( + self, environment: BaseEnvironment, path: str, *, anchor: str, mode: int = 0o755 + ) -> None: ... + + async def _assert_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + label: str, + user: str | int | None = "root", + ) -> None: ... + + async def _write_verified_file( + self, + environment: BaseEnvironment, + source: Path, + target: str, + *, + anchor: str, + mode: int, + digest: str, + byte_length: int, + ) -> None: ... + + +@dataclass(frozen=True) +class ProcessBoundaryConfig: + uid: int + gid: int + evaluator_root: str + control_root: str + home: str + temporary: str + isolate_user: bool = True + + +class CandidateProcessBoundary: + def __init__(self, host: ProcessBoundaryHost, config: ProcessBoundaryConfig): + self._host = host + self.config = config + + async def prepare_identity( + self, environment: BaseEnvironment, workspace_roots: tuple[str, ...] + ) -> None: + config = self.config + await self._host._ensure_real_directory( + environment, config.evaluator_root, anchor="/", mode=0o755 + ) + await self._host._exec( + environment, + f"chmod 0755 -- {shlex.quote(config.evaluator_root)}", + user="root", + ) + for path, mode in ( + (config.control_root, 0o700), + (config.home, 0o700), + (config.temporary, 0o700), + ): + await self._host._ensure_real_directory( + environment, path, anchor="/", mode=mode + ) + await self._host._exec( + environment, + f"chmod {mode:o} -- {shlex.quote(path)}", + user="root", + ) + if not config.isolate_user: + return + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + "command -v setsid >/dev/null", + "command -v setpriv >/dev/null", + f"! grep -l '^Uid:[[:space:]]*{config.uid}[[:space:]]' " + "/proc/[0-9]*/status >/dev/null 2>&1", + f"chown {config.uid}:{config.gid} -- " + f"{shlex.quote(config.home)} {shlex.quote(config.temporary)}", + f"chown -R {config.uid}:{config.gid} -- " + + " ".join(shlex.quote(root) for root in workspace_roots), + ] + ), + user="root", + ) + + async def stage_wrapper(self, environment: BaseEnvironment) -> tuple[str, str]: + token = secrets.token_hex(24) + wrapper = f"{self.config.control_root}/run-{token}.sh" + marker = f"{self.config.control_root}/timeout-{token}" + raw = self._runner_script() + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(raw) + local_path = Path(handle.name) + try: + await self._host._write_verified_file( + environment, + local_path, + wrapper, + anchor=self.config.control_root, + mode=0o700, + digest=sha256_bytes(raw), + byte_length=len(raw), + ) + finally: + local_path.unlink(missing_ok=True) + return wrapper, marker + + async def kill_and_wait(self, environment: BaseEnvironment) -> None: + """Kill every process in the candidate identity and prove none remain.""" + if not self.config.isolate_user: + return + command = """set -eu +candidate_pids() { + for status in /proc/[0-9]*/status; do + test -r "$status" || continue + pid=${status#/proc/} + pid=${pid%/status} + while IFS=: read -r key value; do + if test "$key" = Uid; then + set -- $value + if test "${1:-}" = "$TARGET_UID"; then + printf '%s\n' "$pid" + fi + break + fi + done < "$status" + done +} + +for attempt in $(seq 1 100); do + pids=$(candidate_pids) + test -n "$pids" || exit 0 + for pid in $pids; do + kill -KILL "$pid" 2>/dev/null || true + done + sleep 0.02 +done +remaining=$(candidate_pids) +test -z "$remaining" || { + printf 'candidate processes survived SIGKILL: %s\n' "$remaining" >&2 + exit 1 +} +""" + await self._host._exec( + environment, + command, + env={"TARGET_UID": str(self.config.uid)}, + user="root", + timeout_sec=5, + ) + + async def remove_control_files( + self, + environment: BaseEnvironment, + wrapper: str | None, + marker: str | None, + ) -> None: + paths = [path for path in (wrapper, marker) if path is not None] + if not paths: + return + await self._host._assert_real_directory( + environment, + self.config.control_root, + label="candidate control directory", + ) + await self._host._exec( + environment, + "rm -f -- " + " ".join(shlex.quote(path) for path in paths), + user="root", + ) + + async def timeout_marker_exists( + self, environment: BaseEnvironment, marker: str + ) -> bool: + quoted = shlex.quote(marker) + result = await self._host._exec( + environment, + f'test -f {quoted} && test ! -L {quoted} && test "$(cat -- {quoted})" = timeout', + user="root", + check=False, + ) + return result.return_code == 0 + + def _runner_script(self) -> bytes: + if not self.config.isolate_user: + return b"""#!/bin/sh +set -eu +marker=$1 +uid=$2 +gid=$3 +deadline=$4 +grace=$5 +stdin_path=$6 +shift 6 +rm -f -- "$marker" "$marker.tmp" +set +e +if test "$stdin_path" = -; then + timeout --signal=TERM --kill-after="${grace}s" "${deadline}s" env "$@" +else + timeout --signal=TERM --kill-after="${grace}s" "${deadline}s" env "$@" < "$stdin_path" +fi +status=$? +set -e +if test "$status" = 124; then + printf timeout > "$marker" +fi +exit "$status" +""" + return b"""#!/bin/sh +set -eu +marker=$1 +uid=$2 +gid=$3 +deadline=$4 +grace=$5 +stdin_path=$6 +shift 6 +rm -f -- "$marker" "$marker.tmp" + +kill_candidate_uid() { + signal=$1 + for status in /proc/[0-9]*/status; do + test -r "$status" || continue + pid=${status#/proc/} + pid=${pid%/status} + while IFS=: read -r key value; do + if test "$key" = Uid; then + set -- $value + if test "${1:-}" = "$uid"; then + kill "-$signal" "$pid" 2>/dev/null || true + fi + break + fi + done < "$status" + done +} + +candidate_uid_is_dead() { + for status in /proc/[0-9]*/status; do + test -r "$status" || continue + while IFS=: read -r key value; do + if test "$key" = Uid; then + set -- $value + test "${1:-}" = "$uid" && return 1 + break + fi + done < "$status" + done + return 0 +} + +kill_and_wait_candidate_uid() { + kill_candidate_uid KILL + attempt=0 + while ! candidate_uid_is_dead; do + attempt=$((attempt + 1)) + test "$attempt" -lt 100 || { + printf 'candidate processes survived SIGKILL\n' >&2 + return 1 + } + kill_candidate_uid KILL + sleep 0.02 + done +} + +if test "$stdin_path" = -; then + setsid setpriv --reuid="$uid" --regid="$gid" --clear-groups --no-new-privs \ + --bounding-set=-all --inh-caps=-all --ambient-caps=-all --pdeathsig=KILL -- env "$@" & +else + setsid setpriv --reuid="$uid" --regid="$gid" --clear-groups --no-new-privs \ + --bounding-set=-all --inh-caps=-all --ambient-caps=-all --pdeathsig=KILL -- env "$@" < "$stdin_path" & +fi +candidate_pid=$! +( + sleep "$deadline" + if kill -0 "$candidate_pid" 2>/dev/null; then + umask 077 + printf timeout > "$marker.tmp" + mv -fT -- "$marker.tmp" "$marker" + kill_candidate_uid TERM + sleep "$grace" + kill_and_wait_candidate_uid + fi +) & +watchdog_pid=$! + +set +e +wait "$candidate_pid" +candidate_status=$? +set -e +kill "$watchdog_pid" 2>/dev/null || true +wait "$watchdog_pid" 2>/dev/null || true +kill_and_wait_candidate_uid +if test -f "$marker"; then + exit 124 +fi +exit "$candidate_status" +""" diff --git a/bench/pier_agents/tangle_candidate.py b/bench/pier_agents/tangle_candidate.py new file mode 100644 index 00000000..52aa19d5 --- /dev/null +++ b/bench/pier_agents/tangle_candidate.py @@ -0,0 +1,906 @@ +"""Execute one runtime-prepared candidate inside a Pier task container. + +Pier owns the task container, patch transfer, and verifier. Agent Runtime owns +candidate verification, exact execution bytes, model access, and trace usage. +This bridge only rechecks the prepared bytes, copies their signed workspaces, +delivers the exact task instruction, and reports truthful process termination. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import math +import os +import re +import secrets +import shlex +import tempfile +import time +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version as package_version +from pathlib import Path, PurePosixPath +from typing import Any + +from pier.agents.base import BaseAgent +from pier.agents.installed.base import NonZeroAgentExitCodeError +from pier.environments.base import BaseEnvironment +from pier.models.agent.context import AgentContext +from pier.models.agent.network import NetworkAllowlist + +from .candidate_contract import ( + CandidateContractError, + PreparedCandidateContract, + immutable_snapshot, + load_prepared_candidate_contract, + sha256_bytes, +) +from .process_boundary import CandidateProcessBoundary, ProcessBoundaryConfig +from .workspace_boundary import ( + CandidateWorkspaceBoundary, + WorkspaceBoundaryConfig, + candidate_git_env, +) + + +_ADAPTER_VERSION = "2.0.0" +_ENV_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_PROTECTED_TASK_PATH = "/tangle/input/stdin.txt" +_PIER_PROTECTED_ROOTS = ("/logs", "/tests", "/solution", "/tangle") +_CANDIDATE_UID = 65532 +_CANDIDATE_GID = 65532 +_EVALUATOR_ROOT = "/tangle" +_CANDIDATE_HOME = "/tangle/home" +_CANDIDATE_TMP = "/tangle/tmp" +_CONTROL_ROOT = "/tangle/control" +_FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +_RESERVED_PROCESS_ENV = { + "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", + "GIT_CONFIG_VALUE_0", + "GIT_CONFIG_NOSYSTEM", + "GIT_CONFIG_GLOBAL", + "GIT_CONFIG_SYSTEM", + "GIT_NO_REPLACE_OBJECTS", + "GIT_TERMINAL_PROMPT", + "HOME", + "LC_ALL", + "PATH", + "TMPDIR", +} +_TRACE_ENV_NAMES = { + "TANGLE_CANDIDATE_EXECUTION_ID", + "TANGLE_CANDIDATE_BUNDLE_DIGEST", + "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST", + "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST", + "TANGLE_TRACE_RUN_ID", +} + + +@dataclass(frozen=True) +class OriginalProfileFile: + path: str + existed: bool + tracked: bool + mode: int | None + content: bytes | None + + +class PierCandidateError(CandidateContractError): + """The prepared candidate cannot be replayed safely by Pier.""" + + +def _runtime_pier_version() -> str: + try: + return package_version("datacurve-pier") + except PackageNotFoundError as exc: + raise PierCandidateError( + "datacurve-pier package metadata is unavailable" + ) from exc + + +def _same_or_descendant(path: str, root: str) -> bool: + return path == root or path.startswith(f"{root}/") + + +def _join_workspace(root: str, relative: str) -> str: + root_path = PurePosixPath(root) + joined = root_path if relative == "." else root_path / PurePosixPath(relative) + if joined != root_path and root_path not in joined.parents: + raise PierCandidateError(f"cwd escapes its workspace: {relative}") + return str(joined) + + +def _trace_env( + contract: PreparedCandidateContract, trace_run_id: str +) -> dict[str, str]: + return { + "TANGLE_CANDIDATE_EXECUTION_ID": contract.execution_id, + "TANGLE_CANDIDATE_BUNDLE_DIGEST": contract.bundle_digest, + "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": contract.execution_plan_digest, + "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": contract.materialization_receipt_digest, + "TANGLE_TRACE_RUN_ID": trace_run_id, + } + + +class TangleCandidateAgent(BaseAgent): + """Pier adapter for a branded ``PreparedAgentCandidateExecution``.""" + + SUPPORTS_ATIF = False + SUPPORTS_WINDOWS = False + ISOLATE_CANDIDATE_USER = True + + def __init__( + self, + logs_dir: Path, + model_name: str | None = None, + plan_path: str | None = None, + receipt_path: str | None = None, + expected_receipt_digest: str | None = None, + trace_run_id: str | None = None, + task_dir: str | None = None, + profile_dir: str | None = None, + candidate_dir: str | None = None, + pier_version: str | None = None, + extra_env: dict[str, str] | None = None, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(logs_dir=logs_dir, model_name=model_name, *args, **kwargs) + if ( + not plan_path + or not receipt_path + or not expected_receipt_digest + or not trace_run_id + ): + raise PierCandidateError( + "plan_path, receipt_path, expected_receipt_digest, and trace_run_id are required" + ) + if not task_dir or not profile_dir or not pier_version: + raise PierCandidateError("task_dir, profile_dir, and pier_version are required") + + self._expected_pier_version = pier_version + self._contract = load_prepared_candidate_contract( + Path(plan_path).expanduser().resolve(), + Path(receipt_path).expanduser().resolve(), + expected_receipt_digest, + ) + if model_name != self._contract.requested_model: + raise PierCandidateError( + "Pier model does not match the runtime-resolved request: " + f"expected={self._contract.requested_model} observed={model_name}" + ) + self._extra_env = self._validate_protected_env( + extra_env or {}, trace_run_id + ) + self._validate_execution_roots() + + self._snapshots = tempfile.TemporaryDirectory( + prefix="agent-bench-pier-candidate-" + ) + snapshot_root = Path(self._snapshots.name) + self._task_snapshot = snapshot_root / "task" + self._profile_snapshot = snapshot_root / "profile" + self._candidate_snapshot: Path | None = None + try: + immutable_snapshot( + Path(os.path.abspath(Path(task_dir).expanduser())), + self._task_snapshot, + self._contract.task_files, + "prepared task directory", + ) + immutable_snapshot( + Path(os.path.abspath(Path(profile_dir).expanduser())), + self._profile_snapshot, + self._contract.profile_files, + "prepared profile directory", + ) + if self._contract.candidate_root is not None: + if not candidate_dir: + raise PierCandidateError("active code requires candidate_dir") + self._candidate_snapshot = snapshot_root / "candidate" + immutable_snapshot( + Path(os.path.abspath(Path(candidate_dir).expanduser())), + self._candidate_snapshot, + self._contract.candidate_files, + "prepared candidate directory", + ) + elif candidate_dir: + raise PierCandidateError("disabled code cannot receive candidate_dir") + except Exception: + self._snapshots.cleanup() + raise + + self._original_profile_files: tuple[OriginalProfileFile, ...] = () + self._process_boundary = CandidateProcessBoundary( + self, + ProcessBoundaryConfig( + uid=_CANDIDATE_UID, + gid=_CANDIDATE_GID, + evaluator_root=_EVALUATOR_ROOT, + control_root=_CONTROL_ROOT, + home=_CANDIDATE_HOME, + temporary=_CANDIDATE_TMP, + isolate_user=self.ISOLATE_CANDIDATE_USER, + ), + ) + self._workspace_boundary = CandidateWorkspaceBoundary( + self, + WorkspaceBoundaryConfig( + contract=self._contract, + task_snapshot=self._task_snapshot, + protected_roots=_PIER_PROTECTED_ROOTS, + error_type=PierCandidateError, + ), + ) + + def _validate_protected_env( + self, values: dict[str, str], trace_run_id: str + ) -> dict[str, str]: + protected: dict[str, str] = {} + public_names = set(self._contract.env) + reserved_public = sorted(public_names & _RESERVED_PROCESS_ENV) + if reserved_public: + raise PierCandidateError( + "signed public env collides with evaluator Git safety fields: " + + ", ".join(reserved_public) + ) + expected_trace = _trace_env(self._contract, trace_run_id) + for name, expected in expected_trace.items(): + if values.get(name) != expected: + raise PierCandidateError( + f"evaluator trace environment is missing or mismatched: {name}" + ) + for name, value in values.items(): + if not _ENV_RE.fullmatch(name) or name in _RESERVED_PROCESS_ENV: + raise PierCandidateError( + f"invalid protected model environment name: {name}" + ) + if name in public_names: + raise PierCandidateError( + f"protected model environment overlaps signed public env: {name}" + ) + if name.startswith("TANGLE_CANDIDATE_") and name not in _TRACE_ENV_NAMES: + raise PierCandidateError( + f"unknown evaluator candidate environment: {name}" + ) + if not isinstance(value, str) or not value or "\0" in value: + raise PierCandidateError( + f"protected model environment value is empty or invalid: {name}" + ) + protected[name] = value + return protected + + def _validate_execution_roots(self) -> None: + roots = [self._contract.task_root] + if self._contract.candidate_root is not None: + roots.append(self._contract.candidate_root) + for root in roots: + protected = next( + ( + path + for path in _PIER_PROTECTED_ROOTS + if _same_or_descendant(root, path) + ), + None, + ) + if protected is not None: + raise PierCandidateError( + f"execution workspace cannot be below Pier-owned path {protected}" + ) + + def name(self) -> str: + short = self._contract.bundle_digest.removeprefix("sha256:")[:16] + return f"tangle-candidate-{short}" + + def version(self) -> str: + return _ADAPTER_VERSION + + def network_allowlist(self) -> NetworkAllowlist: + return NetworkAllowlist(domains=list(self._contract.model_network_domains)) + + async def _exec( + self, + environment: BaseEnvironment, + command: str, + *, + cwd: str | None = None, + env: dict[str, str] | None = None, + user: str | int | None = None, + timeout_sec: float | None = None, + check: bool = True, + ) -> Any: + result = await environment.exec( + command=command, + cwd=cwd, + env=environment.agent_process_env(env), + user=user, + timeout_sec=timeout_sec, + ) + if check and result.return_code != 0: + raise PierCandidateError( + f"candidate bridge command failed ({result.return_code}): {command}; " + f"stderr={(result.stderr or '')[:500]}" + ) + return result + + async def _assert_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + label: str, + user: str | int | None = "root", + ) -> None: + quoted = shlex.quote(path) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"test -d {quoted}", + f"test ! -L {quoted}", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user=user, + ) + + async def _ensure_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + anchor: str, + mode: int = 0o755, + ) -> None: + parsed = PurePosixPath(path) + parsed_anchor = PurePosixPath(anchor) + if parsed != parsed_anchor and parsed_anchor not in parsed.parents: + raise PierCandidateError(f"directory escapes its trusted anchor: {path}") + await self._assert_real_directory( + environment, anchor, label="trusted directory anchor" + ) + current = parsed_anchor + relative_parts = parsed.relative_to(parsed_anchor).parts + for part in relative_parts: + current /= part + value = str(current) + quoted = shlex.quote(value) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"if test -e {quoted} || test -L {quoted}; then", + f" test -d {quoted}", + f" test ! -L {quoted}", + "else", + f" mkdir -- {quoted}", + f" chmod {mode:o} -- {quoted}", + "fi", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user="root", + ) + + async def _write_verified_file( + self, + environment: BaseEnvironment, + source: Path, + target: str, + *, + anchor: str, + mode: int, + digest: str, + byte_length: int, + ) -> None: + parent = str(PurePosixPath(target).parent) + await self._ensure_real_directory(environment, parent, anchor=anchor) + temporary = f"{parent}/.tangle-{secrets.token_hex(24)}" + quoted_temporary = shlex.quote(temporary) + quoted_target = shlex.quote(target) + expected_hash = digest.removeprefix("sha256:") + await self._exec( + environment, + f"test ! -e {quoted_temporary} && test ! -L {quoted_temporary}", + user="root", + ) + try: + await environment.upload_file(source, temporary) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"test -f {quoted_temporary}", + f"test ! -L {quoted_temporary}", + f'test "$(stat -c %h -- {quoted_temporary})" = 1', + f'test "$(wc -c < {quoted_temporary})" = {byte_length}', + f"test \"$(sha256sum {quoted_temporary} | cut -d' ' -f1)\" = {expected_hash}", + f"chmod {mode:o} -- {quoted_temporary}", + f'test "$(stat -c %a -- {quoted_temporary})" = {mode:o}', + f"rm -rf -- {quoted_target}", + f"mv -fT -- {quoted_temporary} {quoted_target}", + f"test -f {quoted_target}", + f"test ! -L {quoted_target}", + f'test "$(stat -c %h -- {quoted_target})" = 1', + f'test "$(stat -c %a -- {quoted_target})" = {mode:o}', + f'test "$(wc -c < {quoted_target})" = {byte_length}', + f"test \"$(sha256sum {quoted_target} | cut -d' ' -f1)\" = {expected_hash}", + ] + ), + user="root", + ) + finally: + await self._exec( + environment, + f"rm -f -- {quoted_temporary}", + user="root", + check=False, + ) + + async def _verify_pier_execution_identity( + self, environment: BaseEnvironment + ) -> None: + await self._workspace_boundary.verify_pier_execution_identity(environment) + + async def _capture_original_profile_files( + self, environment: BaseEnvironment + ) -> tuple[OriginalProfileFile, ...]: + if self._contract.profile_target != "task": + return () + originals: list[OriginalProfileFile] = [] + root = self._contract.task_root + git = f"git -c core.hooksPath=/dev/null -C {shlex.quote(root)}" + for file in self._contract.profile_files: + target = f"{root}/{file.path}" + quoted = shlex.quote(target) + parent = shlex.quote(str(PurePosixPath(target).parent)) + presence = await self._exec( + environment, + "\n".join( + [ + "set -eu", + f'test "$(realpath -m -- {parent})" = {parent}', + f"if test -e {quoted} || test -L {quoted}; then", + f" test -f {quoted}", + f" test ! -L {quoted}", + f' test "$(stat -c %h -- {quoted})" = 1', + "else", + " exit 2", + "fi", + ] + ), + user="root", + check=False, + ) + tracked = ( + await self._exec( + environment, + f"{git} ls-files --error-unmatch -- {shlex.quote(file.path)}", + env=candidate_git_env(root), + user="root", + check=False, + ) + ).return_code == 0 + if presence.return_code == 2: + originals.append( + OriginalProfileFile(file.path, False, tracked, None, None) + ) + continue + if presence.return_code != 0: + raise PierCandidateError( + f"profile mount target is not one regular file: {file.path}" + ) + encoded = await self._exec( + environment, f"base64 -w0 -- {quoted}", user="root" + ) + mode = await self._exec(environment, f"stat -c %a -- {quoted}", user="root") + try: + content = base64.b64decode( + (encoded.stdout or "").strip(), validate=True + ) + parsed_mode = int((mode.stdout or "").strip(), 8) + except (ValueError, TypeError) as exc: + raise PierCandidateError( + f"could not capture original profile target: {file.path}" + ) from exc + originals.append( + OriginalProfileFile(file.path, True, tracked, parsed_mode, content) + ) + return tuple(originals) + + async def _apply_profile(self, environment: BaseEnvironment) -> None: + target_root = ( + self._contract.task_root + if self._contract.profile_target == "task" + else self._contract.candidate_root + ) + if target_root is None: + raise PierCandidateError("profile target workspace is unavailable") + self._original_profile_files = await self._capture_original_profile_files( + environment + ) + for file in self._contract.profile_files: + target = f"{target_root}/{file.path}" + source = self._profile_snapshot / file.path + await self._write_verified_file( + environment, + source, + target, + anchor=target_root, + mode=file.mode, + digest=file.sha256, + byte_length=source.stat().st_size, + ) + + async def setup(self, environment: BaseEnvironment) -> None: + try: + observed_pier = _runtime_pier_version() + if observed_pier != self._expected_pier_version: + raise PierCandidateError( + f"Pier version mismatch: expected={self._expected_pier_version} " + f"installed={observed_pier}" + ) + await self._verify_pier_execution_identity(environment) + await self._workspace_boundary.materialize_task_workspace(environment) + await self._workspace_boundary.verify_repository(environment) + await self._workspace_boundary.verify_container_workspace( + environment, + self._contract.task_root, + self._contract.task_files, + allow_task_metadata=True, + ) + if self._contract.candidate_root is not None: + if self._candidate_snapshot is None: + raise PierCandidateError("candidate snapshot is unavailable") + quoted = shlex.quote(self._contract.candidate_root) + candidate_parent = str( + PurePosixPath(self._contract.candidate_root).parent + ) + await self._ensure_real_directory( + environment, candidate_parent, anchor="/" + ) + await self._exec( + environment, + "\n".join( + [ + "set -eu", + f"test ! -e {quoted}", + f"test ! -L {quoted}", + f"mkdir -- {quoted}", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user="root", + ) + await self._workspace_boundary.verify_resolved_root_isolation( + environment + ) + await environment.upload_dir( + self._candidate_snapshot, self._contract.candidate_root + ) + await self._workspace_boundary.verify_container_workspace( + environment, + self._contract.candidate_root, + self._contract.candidate_files, + allow_task_metadata=False, + ) + else: + await self._workspace_boundary.verify_resolved_root_isolation( + environment + ) + workspace_roots = (self._contract.task_root,) + ( + (self._contract.candidate_root,) + if self._contract.candidate_root is not None + else () + ) + await self._process_boundary.prepare_identity(environment, workspace_roots) + await self._apply_profile(environment) + + self.logs_dir.mkdir(parents=True, exist_ok=True) + (self.logs_dir / "execution-plan.json").write_bytes(self._contract.plan_raw) + (self.logs_dir / "materialization-receipt.json").write_bytes( + self._contract.receipt_raw + ) + finally: + self._snapshots.cleanup() + + def _identity_metadata(self) -> dict[str, Any]: + return { + "executionId": self._contract.execution_id, + "bundleDigest": self._contract.bundle_digest, + "executionPlanDigest": self._contract.execution_plan_digest, + "materializationReceiptDigest": self._contract.materialization_receipt_digest, + "usageSource": "protected-agent-eval-trace", + } + + async def _upload_instruction( + self, environment: BaseEnvironment, instruction: str + ) -> tuple[list[str], str | None, str | None]: + try: + raw = instruction.encode("utf-8") + except UnicodeEncodeError as exc: + raise PierCandidateError("Pier instruction is not valid UTF-8") from exc + evidence = self._contract.instruction + if len(raw) != evidence.byte_length or sha256_bytes(raw) != evidence.sha256: + raise PierCandidateError( + "Pier instruction bytes do not match the signed task instruction" + ) + + argv = [self._contract.executable, *self._contract.args] + if evidence.delivery_kind == "argv-append": + argv.append(instruction) + return argv, None, None + + target = evidence.delivery_path or _PROTECTED_TASK_PATH + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(raw) + local_path = Path(handle.name) + try: + await self._write_verified_file( + environment, + local_path, + target, + anchor="/", + mode=0o644, + digest=evidence.sha256, + byte_length=len(raw), + ) + finally: + local_path.unlink(missing_ok=True) + return ( + argv, + target if evidence.delivery_kind == "stdin-utf8" else None, + target, + ) + + async def _remove_instruction_file( + self, environment: BaseEnvironment, path: str | None + ) -> None: + if path is None: + return + parent = str(PurePosixPath(path).parent) + await self._assert_real_directory( + environment, parent, label="protected instruction directory" + ) + await self._exec( + environment, + f"rm -f -- {shlex.quote(path)}", + user="root", + ) + + async def _restore_profile_and_capture_solution( + self, environment: BaseEnvironment + ) -> None: + root = self._contract.task_root + quoted_root = shlex.quote(root) + git = f"git -c core.hooksPath=/dev/null -C {quoted_root}" + git_env = candidate_git_env(root) + + # Ignore candidate-authored commits and derive one evaluator-owned final + # tree from the actual working files relative to the signed base commit. + await self._exec( + environment, + f"{git} reset --mixed {shlex.quote(self._contract.repository_base_commit)}", + env=git_env, + user="root", + ) + for original in self._original_profile_files: + target = f"{root}/{original.path}" + quoted = shlex.quote(target) + if original.existed: + if original.content is None or original.mode is None: + raise PierCandidateError( + f"profile backup is incomplete: {original.path}" + ) + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(original.content) + local_path = Path(handle.name) + try: + await self._write_verified_file( + environment, + local_path, + target, + anchor=root, + mode=original.mode, + digest=sha256_bytes(original.content), + byte_length=len(original.content), + ) + finally: + local_path.unlink(missing_ok=True) + else: + parent = str(PurePosixPath(target).parent) + await self._ensure_real_directory(environment, parent, anchor=root) + await self._exec(environment, f"rm -rf -- {quoted}", user="root") + + await self._exec(environment, f"{git} add -A -- .", env=git_env, user="root") + for original in self._original_profile_files: + if original.tracked: + continue + await self._exec( + environment, + f"{git} rm -r -f --cached --ignore-unmatch -- {shlex.quote(original.path)}", + env=git_env, + user="root", + ) + staged = await self._exec( + environment, + f"{git} diff --cached --quiet", + env=git_env, + user="root", + check=False, + ) + if staged.return_code not in {0, 1}: + raise PierCandidateError("could not inspect the sanitized solution tree") + if staged.return_code == 1: + commit_env = { + **git_env, + "GIT_AUTHOR_NAME": "Tangle Evaluator", + "GIT_AUTHOR_EMAIL": "evaluator@tangle.tools", + "GIT_COMMITTER_NAME": "Tangle Evaluator", + "GIT_COMMITTER_EMAIL": "evaluator@tangle.tools", + } + await self._exec( + environment, + f"{git} commit -m 'capture candidate solution'", + env=commit_env, + user="root", + ) + + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + argv, stdin_path, instruction_path = await self._upload_instruction( + environment, instruction + ) + cwd = _join_workspace(self._contract.cwd_root, self._contract.cwd_path) + await self._exec(environment, f"test -d {shlex.quote(cwd)}") + + process_env = { + **self._extra_env, + **candidate_git_env(self._contract.task_root), + "HOME": _CANDIDATE_HOME, + "TMPDIR": _CANDIDATE_TMP, + "PATH": _FIXED_PATH, + } + wrapper, timeout_marker = await self._process_boundary.stage_wrapper( + environment + ) + timeout_seconds = self._contract.timeout_ms / 1000 + command = shlex.join( + [ + wrapper, + timeout_marker, + str(_CANDIDATE_UID), + str(_CANDIDATE_GID), + f"{timeout_seconds:.3f}", + "2", + stdin_path or "-", + *(f"{name}={value}" for name, value in self._contract.env.items()), + *argv, + ] + ) + context.metadata = { + **self._identity_metadata(), + "termination": {"kind": "running"}, + } + + started = time.monotonic() + try: + result = await environment.exec( + command=command, + cwd=cwd, + env=environment.agent_process_env(process_env), + user="root", + timeout_sec=math.ceil(timeout_seconds) + 12, + ) + except (asyncio.CancelledError, asyncio.TimeoutError): + elapsed = int((time.monotonic() - started) * 1000) + cleanup_errors: list[str] = [] + try: + await asyncio.shield( + self._process_boundary.kill_and_wait(environment) + ) + except Exception as cleanup_exc: + cleanup_errors.append(str(cleanup_exc)) + try: + await asyncio.shield( + self._remove_instruction_file(environment, instruction_path) + ) + except Exception as cleanup_exc: + cleanup_errors.append(str(cleanup_exc)) + try: + await asyncio.shield( + self._restore_profile_and_capture_solution(environment) + ) + except ( + Exception + ) as cleanup_exc: # cleanup failure is evidence, never hidden + cleanup_errors.append(str(cleanup_exc)) + try: + await asyncio.shield( + self._process_boundary.remove_control_files( + environment, wrapper, timeout_marker + ) + ) + except Exception as cleanup_exc: + cleanup_errors.append(str(cleanup_exc)) + context.metadata = { + **self._identity_metadata(), + "termination": {"kind": "cancelled", "observedElapsedMs": elapsed}, + **( + {"cleanupError": "; ".join(cleanup_errors)} + if cleanup_errors + else {} + ), + } + raise + except Exception: + await asyncio.shield(self._process_boundary.kill_and_wait(environment)) + await asyncio.shield( + self._remove_instruction_file(environment, instruction_path) + ) + await asyncio.shield( + self._restore_profile_and_capture_solution(environment) + ) + await asyncio.shield( + self._process_boundary.remove_control_files( + environment, wrapper, timeout_marker + ) + ) + raise + + elapsed = int((time.monotonic() - started) * 1000) + timed_out = await self._process_boundary.timeout_marker_exists( + environment, timeout_marker + ) + await self._process_boundary.kill_and_wait(environment) + await self._remove_instruction_file(environment, instruction_path) + await self._restore_profile_and_capture_solution(environment) + await self._process_boundary.remove_control_files( + environment, wrapper, timeout_marker + ) + self.logs_dir.mkdir(parents=True, exist_ok=True) + (self.logs_dir / "stdout.txt").write_text(result.stdout or "", encoding="utf-8") + (self.logs_dir / "stderr.txt").write_text(result.stderr or "", encoding="utf-8") + (self.logs_dir / "process.json").write_text( + json.dumps( + { + "executionId": self._contract.execution_id, + "exitCode": result.return_code, + "elapsedMs": elapsed, + "cwd": cwd, + "argv": argv, + "instructionDelivery": self._contract.instruction.delivery_kind, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + context.metadata = { + **self._identity_metadata(), + "termination": ( + {"kind": "timeout", "timeoutMs": self._contract.timeout_ms} + if timed_out + else {"kind": "exit", "exitCode": result.return_code} + ), + "observedElapsedMs": elapsed, + } + if timed_out: + raise asyncio.TimeoutError( + f"candidate exceeded signed timeout {self._contract.timeout_ms}ms" + ) + if result.return_code != 0: + raise NonZeroAgentExitCodeError( + f"candidate exited {result.return_code}: " + f"stdout={(result.stdout or '')[:500]} " + f"stderr={(result.stderr or '')[:500]}" + ) diff --git a/bench/pier_agents/tangle_candidate_test.py b/bench/pier_agents/tangle_candidate_test.py new file mode 100644 index 00000000..29af6759 --- /dev/null +++ b/bench/pier_agents/tangle_candidate_test.py @@ -0,0 +1,693 @@ +import asyncio +import base64 +import hashlib +import importlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import types +import unittest +from pathlib import Path + + +class _BaseAgent: + def __init__(self, logs_dir, model_name=None, *args, **kwargs): + self.logs_dir = Path(logs_dir) + self.model_name = model_name + + +class _NonZeroAgentExitCodeError(RuntimeError): + pass + + +class _AgentContext: + def __init__(self): + self.n_input_tokens = None + self.n_cache_tokens = None + self.n_output_tokens = None + self.cost_usd = None + self.metadata = None + + +class _NetworkAllowlist: + def __init__(self, domains=None): + self.domains = list(domains or []) + + +def _install_pier_stubs(): + modules = { + name: types.ModuleType(name) + for name in ( + "pier", + "pier.agents", + "pier.agents.base", + "pier.agents.installed", + "pier.agents.installed.base", + "pier.environments", + "pier.environments.base", + "pier.models", + "pier.models.agent", + "pier.models.agent.context", + "pier.models.agent.network", + ) + } + modules["pier.agents.base"].BaseAgent = _BaseAgent + modules[ + "pier.agents.installed.base" + ].NonZeroAgentExitCodeError = _NonZeroAgentExitCodeError + modules["pier.environments.base"].BaseEnvironment = type("BaseEnvironment", (), {}) + modules["pier.models.agent.context"].AgentContext = _AgentContext + modules["pier.models.agent.network"].NetworkAllowlist = _NetworkAllowlist + sys.modules.update(modules) + + +_install_pier_stubs() +candidate = importlib.import_module("pier_agents.tangle_candidate") +contract_module = importlib.import_module("pier_agents.candidate_contract") +process_boundary_module = importlib.import_module("pier_agents.process_boundary") +candidate._runtime_pier_version = lambda: "0.3.0" + + +class _LocalTangleCandidateAgent(candidate.TangleCandidateAgent): + ISOLATE_CANDIDATE_USER = False + + async def _verify_pier_execution_identity(self, environment): + del environment + + +class _Result: + def __init__(self, return_code=0, stdout="", stderr=""): + self.return_code = return_code + self.stdout = stdout + self.stderr = stderr + + +class _LocalEnvironment: + def agent_process_env(self, env): + return {**os.environ, **(env or {})} + + async def upload_dir(self, source_dir, target_dir): + shutil.copytree(source_dir, target_dir, dirs_exist_ok=True) + + async def upload_file(self, source_path, target_path): + target = Path(target_path) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, target) + + async def exec(self, command, cwd=None, env=None, user=None, timeout_sec=None): + del user + completed = subprocess.run( + ["bash", "-c", command], + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + timeout=timeout_sec, + ) + return _Result(completed.returncode, completed.stdout, completed.stderr) + + +class _LocalBoundaryHost: + async def _exec(self, environment, command, **kwargs): + check = kwargs.pop("check", True) + result = await environment.exec(command=command, **kwargs) + if check and result.return_code != 0: + raise RuntimeError(result.stderr or f"boundary command exited {result.return_code}") + return result + + +def _sha(raw): + return f"sha256:{hashlib.sha256(raw).hexdigest()}" + + +def _canonical(value): + return json.dumps( + value, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ).encode() + + +def _embedded(raw): + return { + "encoding": "base64", + "content": base64.b64encode(raw).decode(), + "sha256": _sha(raw), + "byteLength": len(raw), + } + + +def _workspace(files): + material = { + "schemaVersion": 1, + "kind": "agent-candidate-workspace-manifest", + "files": [ + { + "path": path, + "mode": mode, + "sha256": _sha(raw), + "byteLength": len(raw), + } + for path, mode, raw in sorted(files) + ], + } + manifest = _canonical(material) + return { + "schemaVersion": 1, + "kind": "agent-candidate-workspace-snapshot", + "digest": _sha(manifest), + "material": material, + "manifest": _embedded(manifest), + "archive": _embedded(b"fixture-archive"), + } + + +def _git(root, *args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True).strip() + + +def _fixture(root: Path): + task_root = root / "execution-task" + task_staging = root / "task-staging" + candidate_root = root / "execution-candidate" + candidate_staging = root / "candidate-staging" + profile_staging = root / "profile-staging" + sealed = root / "sealed" + logs = root / "logs" + task_root.mkdir() + (task_staging / "src").mkdir(parents=True) + candidate_staging.mkdir() + profile_staging.mkdir() + sealed.mkdir() + + status = b"not-ready\n" + (task_root / "src").mkdir() + (task_root / "src/status.txt").write_bytes(status) + os.chmod(task_root / "src/status.txt", 0o644) + (task_staging / "src/status.txt").write_bytes(status) + os.chmod(task_staging / "src/status.txt", 0o644) + subprocess.run( + ["git", "init", "-b", "main", str(task_root)], check=True, capture_output=True + ) + _git(task_root, "config", "user.email", "fixture@example.com") + _git(task_root, "config", "user.name", "Fixture") + _git(task_root, "add", "-A") + _git(task_root, "commit", "-m", "base") + base_commit = _git(task_root, "rev-parse", "HEAD") + base_tree = _git(task_root, "rev-parse", "HEAD^{tree}") + + runner = ( + "import os, pathlib, sys, time\n" + "root = pathlib.Path.cwd()\n" + "profile = root / 'AGENTS.md'\n" + "if not profile.exists():\n" + " profile = pathlib.Path(__file__).parent / 'AGENTS.md'\n" + "assert profile.read_text() == 'fixture-profile\\n'\n" + "assert sys.argv[-1] == 'make the task ready'\n" + "if os.environ.get('FIXTURE_SLEEP'):\n" + " time.sleep(float(os.environ['FIXTURE_SLEEP']))\n" + "attack = os.environ.get('FIXTURE_ATTACK')\n" + "if attack == 'symlink':\n" + " profile.unlink()\n" + " profile.symlink_to('src/status.txt')\n" + "elif attack == 'hardlink':\n" + " profile.unlink()\n" + " os.link(root / 'src/status.txt', profile)\n" + "(root / 'src/status.txt').write_text('ready\\n')\n" + ).encode() + (candidate_staging / "runner.py").write_bytes(runner) + os.chmod(candidate_staging / "runner.py", 0o755) + profile = b"fixture-profile\n" + (profile_staging / "AGENTS.md").write_bytes(profile) + os.chmod(profile_staging / "AGENTS.md", 0o644) + + instruction = "make the task ready".encode() + task_snapshot = _workspace([("src/status.txt", 0o644, status)]) + candidate_snapshot = _workspace([("runner.py", 0o755, runner)]) + profile_material = { + "version": 1, + "harness": "codex", + "files": [ + { + "relPath": "AGENTS.md", + "mode": 0o644, + "contentSha256": _sha(profile), + } + ], + "env": {}, + "flags": [], + "unsupported": [], + } + profile_raw = _canonical(profile_material) + profile_digest = _sha(profile_raw) + bundle_digest = f"sha256:{'b' * 64}" + plan = { + "schemaVersion": 1, + "kind": "agent-candidate-execution-plan-material", + "bundleDigest": bundle_digest, + "executionId": "pier-fixture-execution", + "attempt": {"number": 1, "maxAttempts": 1, "retryPolicy": "none"}, + "task": { + "benchmark": "pier-fixture", + "benchmarkVersion": "1", + "taskId": "fixture-1", + "splitDigest": f"sha256:{'1' * 64}", + "instruction": { + "encoding": "utf8", + "sha256": _sha(instruction), + "byteLength": len(instruction), + "delivery": {"kind": "argv-append"}, + }, + "repository": { + "identity": "fixture/repository", + "rootIdentity": "fixture/repository", + "baseCommit": base_commit, + "baseTree": base_tree, + }, + "workspace": task_snapshot, + }, + "workspaces": { + "taskRoot": str(task_root), + "candidateRoot": str(candidate_root), + }, + "codeKind": "no-op", + "candidateWorkspace": candidate_snapshot, + "profile": { + "planDigest": profile_digest, + "targetWorkspace": "task", + "mountPaths": ["AGENTS.md"], + }, + "harness": "codex", + "harnessVersion": "fixture", + "container": { + "source": "evaluator-task-container", + "image": "ghcr.io/tangle-network/fixture:latest", + "indexDigest": f"sha256:{'2' * 64}", + "manifestDigest": f"sha256:{'3' * 64}", + "platform": {"os": "linux", "architecture": "amd64"}, + }, + "model": { + "policy": "single", + "resolved": { + "requested": "openai/gpt-5.4", + "provider": "openai", + "model": "gpt-5.4", + "snapshot": "fixture", + "reasoningEffort": "xhigh", + }, + "access": { + "kind": "evaluator-mediated", + "grantDigest": f"sha256:{'4' * 64}", + "network": {"mode": "disabled"}, + }, + "routes": [{"kind": "primary", "requested": "openai/gpt-5.4"}], + }, + "launch": { + "executable": "python3", + "args": [{"kind": "public", "value": str(candidate_root / "runner.py")}], + "env": {}, + "cwd": {"workspace": "task", "path": "."}, + }, + "memory": {"mode": "disabled"}, + "limits": { + "timeoutMs": 60_000, + "maxSteps": 8, + "maxModelCalls": 0, + "maxInputTokens": 0, + "maxOutputTokens": 0, + "maxCostUsd": 0, + }, + "network": {"mode": "disabled"}, + } + plan_raw = _canonical(plan) + plan_digest = _sha(plan_raw) + execution_evidence = { + "schemaVersion": 1, + "kind": "agent-candidate-execution-plan", + "digest": plan_digest, + "material": plan, + "artifact": _embedded(plan_raw), + } + profile_evidence = { + "schemaVersion": 1, + "kind": "agent-profile-workspace-plan", + "digest": profile_digest, + "material": profile_material, + "artifact": _embedded(profile_raw), + } + receipt = { + "schemaVersion": 1, + "kind": "agent-candidate-materialization", + "digestAlgorithm": "rfc8785-sha256", + "bundleDigest": bundle_digest, + "profilePlan": profile_evidence, + "executionPlan": execution_evidence, + "candidateWorkspace": candidate_snapshot, + "codeKind": "no-op", + "materializedTree": base_tree, + "harness": "codex", + "harnessVersion": "fixture", + "container": plan["container"], + "resolvedModel": plan["model"]["resolved"], + "entrypoint": { + "path": "runner.py", + "sha256": _sha(runner), + "byteLength": len(runner), + }, + } + receipt_raw = _canonical(receipt) + plan_path = sealed / "execution-plan.json" + receipt_path = sealed / "materialization-receipt.json" + plan_path.write_bytes(plan_raw) + receipt_path.write_bytes(receipt_raw) + return { + "plan_path": plan_path, + "receipt_path": receipt_path, + "receipt_digest": _sha(receipt_raw), + "candidate_staging": candidate_staging, + "task_staging": task_staging, + "profile_staging": profile_staging, + "candidate_root": candidate_root, + "task_root": task_root, + "logs": logs, + "base_commit": base_commit, + "plan": plan, + } + + +def _rewrite_signed_plan(fixture): + plan_raw = _canonical(fixture["plan"]) + plan_digest = _sha(plan_raw) + receipt = json.loads(fixture["receipt_path"].read_text()) + receipt["executionPlan"] = { + "schemaVersion": 1, + "kind": "agent-candidate-execution-plan", + "digest": plan_digest, + "material": fixture["plan"], + "artifact": _embedded(plan_raw), + } + receipt["container"] = fixture["plan"]["container"] + receipt["resolvedModel"] = fixture["plan"]["model"]["resolved"] + receipt_raw = _canonical(receipt) + fixture["plan_path"].write_bytes(plan_raw) + fixture["receipt_path"].write_bytes(receipt_raw) + fixture["receipt_digest"] = _sha(receipt_raw) + + +def _agent(fixture): + evaluator_root = fixture["plan_path"].parent / "evaluator" + candidate._CANDIDATE_UID = os.getuid() + candidate._CANDIDATE_GID = os.getgid() + candidate._EVALUATOR_ROOT = str(evaluator_root) + candidate._CONTROL_ROOT = str(evaluator_root / "control") + candidate._CANDIDATE_HOME = str(evaluator_root / "home") + candidate._CANDIDATE_TMP = str(evaluator_root / "tmp") + plan_digest = _sha(fixture["plan_path"].read_bytes()) + trace_env = { + "TANGLE_CANDIDATE_EXECUTION_ID": fixture["plan"]["executionId"], + "TANGLE_CANDIDATE_BUNDLE_DIGEST": fixture["plan"]["bundleDigest"], + "TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST": plan_digest, + "TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST": fixture["receipt_digest"], + "TANGLE_TRACE_RUN_ID": fixture["plan"]["executionId"], + } + return _LocalTangleCandidateAgent( + logs_dir=fixture["logs"], + model_name=fixture["plan"]["model"]["resolved"]["requested"], + plan_path=str(fixture["plan_path"]), + receipt_path=str(fixture["receipt_path"]), + expected_receipt_digest=fixture["receipt_digest"], + trace_run_id=fixture["plan"]["executionId"], + task_dir=str(fixture["task_staging"]), + candidate_dir=str(fixture["candidate_staging"]), + profile_dir=str(fixture["profile_staging"]), + pier_version="0.3.0", + extra_env=trace_env, + ) + + +class CandidateContractTest(unittest.TestCase): + def test_loads_exact_runtime_artifacts_and_rejects_plan_substitution(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + loaded = contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + self.assertEqual(loaded.execution_id, "pier-fixture-execution") + self.assertEqual(loaded.instruction.delivery_kind, "argv-append") + self.assertEqual(loaded.model_network_domains, ()) + + fixture["plan"]["executionId"] = "substituted" + fixture["plan_path"].write_bytes(_canonical(fixture["plan"])) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "does not bind" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_accepts_only_frozen_public_model_gateway_domains(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["limits"]["maxModelCalls"] = 1 + fixture["plan"]["model"]["access"]["network"] = { + "mode": "gateway-only", + "domains": ["router.tangle.tools"], + } + _rewrite_signed_plan(fixture) + loaded = contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + self.assertEqual( + loaded.model_network_domains, ("router.tangle.tools",) + ) + agent = _agent(fixture) + try: + self.assertEqual( + agent.network_allowlist().domains, + ["router.tangle.tools"], + ) + finally: + agent._snapshots.cleanup() + + def test_rejects_wildcard_or_unsorted_model_gateway_domains(self): + for domains in ( + ["*.tangle.tools"], + ["router.tangle.tools", "api.openai.com"], + ): + with ( + self.subTest(domains=domains), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + fixture["plan"]["limits"]["maxModelCalls"] = 1 + fixture["plan"]["model"]["access"]["network"] = { + "mode": "gateway-only", + "domains": domains, + } + _rewrite_signed_plan(fixture) + with self.assertRaises(contract_module.CandidateContractError): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_model_gateway_when_call_budget_is_zero(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["model"]["access"]["network"] = { + "mode": "gateway-only", + "domains": ["router.tangle.tools"], + } + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "zero-call plans" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + def test_rejects_unsigned_candidate_workspace_files(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + (fixture["candidate_staging"] / "unsigned.py").write_text("bad\n") + with self.assertRaisesRegex( + contract_module.CandidateContractError, "file set differs" + ): + _agent(fixture) + + def test_rejects_container_image_with_embedded_digest(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["container"]["image"] += f"@sha256:{'9' * 64}" + _rewrite_signed_plan(fixture) + with self.assertRaisesRegex( + contract_module.CandidateContractError, "unpinned OCI reference" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + + +class TangleCandidateAgentTest(unittest.TestCase): + def test_process_stop_acknowledges_candidate_identity_is_empty(self): + boundary = process_boundary_module.CandidateProcessBoundary( + _LocalBoundaryHost(), + process_boundary_module.ProcessBoundaryConfig( + uid=2_000_000_000, + gid=2_000_000_000, + evaluator_root="/tmp/unused-evaluator", + control_root="/tmp/unused-control", + home="/tmp/unused-home", + temporary="/tmp/unused-tmp", + ), + ) + asyncio.run(boundary.kill_and_wait(_LocalEnvironment())) + + def test_executes_exact_instruction_and_excludes_profile_from_patch(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + self.assertEqual( + (fixture["task_root"] / "AGENTS.md").read_text(), + "fixture-profile\n", + ) + asyncio.run(agent.run("make the task ready", environment, context)) + + self.assertEqual( + (fixture["task_root"] / "src/status.txt").read_text(), "ready\n" + ) + self.assertFalse((fixture["task_root"] / "AGENTS.md").exists()) + changed = _git( + fixture["task_root"], + "diff", + "--name-only", + f"{fixture['base_commit']}..HEAD", + ).splitlines() + self.assertEqual(changed, ["src/status.txt"]) + self.assertEqual(context.n_input_tokens, None) + self.assertEqual(context.cost_usd, None) + self.assertEqual(context.metadata["executionId"], "pier-fixture-execution") + self.assertEqual( + context.metadata["termination"], {"kind": "exit", "exitCode": 0} + ) + self.assertEqual( + context.metadata["usageSource"], "protected-agent-eval-trace" + ) + + def test_rejects_instruction_substitution_before_launch(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + agent = _agent(fixture) + environment = _LocalEnvironment() + asyncio.run(agent.setup(environment)) + with self.assertRaisesRegex( + candidate.PierCandidateError, "instruction bytes do not match" + ): + asyncio.run(agent.run("different task", environment, _AgentContext())) + + def test_candidate_target_profile_still_captures_task_solution(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["profile"]["targetWorkspace"] = "candidate" + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + asyncio.run(agent.run("make the task ready", environment, context)) + + changed = _git( + fixture["task_root"], + "diff", + "--name-only", + f"{fixture['base_commit']}..HEAD", + ).splitlines() + self.assertEqual(changed, ["src/status.txt"]) + self.assertEqual( + (fixture["task_root"] / "src/status.txt").read_text(), "ready\n" + ) + + def test_enforces_exact_signed_timeout(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["launch"]["env"] = { + "FIXTURE_SLEEP": {"kind": "public", "value": "0.25"} + } + fixture["plan"]["limits"]["timeoutMs"] = 10 + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + started = time.monotonic() + with self.assertRaises(asyncio.TimeoutError): + asyncio.run(agent.run("make the task ready", environment, context)) + self.assertLess(time.monotonic() - started, 0.2) + self.assertEqual(context.metadata["termination"]["kind"], "timeout") + self.assertEqual(context.metadata["termination"]["timeoutMs"], 10) + + def test_profile_cleanup_does_not_follow_candidate_links(self): + for attack in ("symlink", "hardlink"): + with ( + self.subTest(attack=attack), + tempfile.TemporaryDirectory() as directory, + ): + fixture = _fixture(Path(directory)) + fixture["plan"]["launch"]["env"] = { + "FIXTURE_ATTACK": {"kind": "public", "value": attack} + } + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + environment = _LocalEnvironment() + context = _AgentContext() + asyncio.run(agent.setup(environment)) + asyncio.run(agent.run("make the task ready", environment, context)) + + self.assertFalse((fixture["task_root"] / "AGENTS.md").exists()) + self.assertEqual( + (fixture["task_root"] / "src/status.txt").read_text(), + "ready\n", + ) + changed = _git( + fixture["task_root"], + "diff", + "--name-only", + f"{fixture['base_commit']}..HEAD", + ).splitlines() + self.assertEqual(changed, ["src/status.txt"]) + + def test_rejects_symlinked_candidate_root_ancestor(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + alias = Path(directory) / "aliased-parent" + alias.symlink_to(fixture["task_root"], target_is_directory=True) + candidate_root = alias / "candidate" + fixture["plan"]["workspaces"]["candidateRoot"] = str(candidate_root) + fixture["plan"]["launch"]["args"] = [ + {"kind": "public", "value": str(candidate_root / "runner.py")} + ] + _rewrite_signed_plan(fixture) + agent = _agent(fixture) + with self.assertRaises(candidate.PierCandidateError): + asyncio.run(agent.setup(_LocalEnvironment())) + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/pier_agents/workspace_boundary.py b/bench/pier_agents/workspace_boundary.py new file mode 100644 index 00000000..52b4fddb --- /dev/null +++ b/bench/pier_agents/workspace_boundary.py @@ -0,0 +1,264 @@ +"""Verify and materialize runtime-signed workspaces inside a Pier container.""" + +from __future__ import annotations + +import shlex +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Protocol + +from pier.environments.base import BaseEnvironment + +from .candidate_contract import PreparedCandidateContract, WorkspaceFile + + +class WorkspaceBoundaryHost(Protocol): + async def _exec( + self, environment: BaseEnvironment, command: str, **kwargs: Any + ) -> Any: ... + + async def _ensure_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + anchor: str, + mode: int = 0o755, + ) -> None: ... + + async def _assert_real_directory( + self, + environment: BaseEnvironment, + path: str, + *, + label: str, + user: str | int | None = "root", + ) -> None: ... + + +@dataclass(frozen=True) +class WorkspaceBoundaryConfig: + contract: PreparedCandidateContract + task_snapshot: Path + protected_roots: tuple[str, ...] + error_type: type[Exception] + + +def candidate_git_env(task_root: str) -> dict[str, str]: + return { + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "safe.directory", + "GIT_CONFIG_VALUE_0": task_root, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_CONFIG_SYSTEM": "/dev/null", + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_TERMINAL_PROMPT": "0", + "LC_ALL": "C", + } + + +class CandidateWorkspaceBoundary: + def __init__(self, host: WorkspaceBoundaryHost, config: WorkspaceBoundaryConfig): + self._host = host + self._config = config + + @property + def _contract(self) -> PreparedCandidateContract: + return self._config.contract + + def _error(self, message: str) -> Exception: + return self._config.error_type(message) + + async def verify_repository(self, environment: BaseEnvironment) -> None: + root = shlex.quote(self._contract.task_root) + git = f"git -c core.hooksPath=/dev/null -C {root}" + env = candidate_git_env(self._contract.task_root) + head = await self._host._exec(environment, f"{git} rev-parse HEAD", env=env) + tree = await self._host._exec( + environment, f"{git} rev-parse 'HEAD^{{tree}}'", env=env + ) + replacements = await self._host._exec( + environment, + f"{git} for-each-ref --format='%(refname)' refs/replace", + env=env, + ) + if (head.stdout or "").strip() != self._contract.repository_base_commit: + raise self._error("task checkout HEAD does not match the signed base commit") + if (tree.stdout or "").strip() != self._contract.repository_base_tree: + raise self._error("task checkout tree does not match the signed base tree") + if (replacements.stdout or "").strip(): + raise self._error("task checkout contains forbidden Git replace refs") + + async def materialize_task_workspace(self, environment: BaseEnvironment) -> None: + root = self._contract.task_root + quoted = shlex.quote(root) + exists = await self._host._exec( + environment, + f"test -e {quoted} || test -L {quoted}", + user="root", + check=False, + ) + if exists.return_code == 0: + return + if exists.return_code != 1: + raise self._error("could not inspect the signed task root") + + parent = str(PurePosixPath(root).parent) + await self._host._ensure_real_directory(environment, parent, anchor="/") + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"test ! -e {quoted}", + f"test ! -L {quoted}", + f"mkdir -- {quoted}", + f'test "$(realpath -e -- {quoted})" = {quoted}', + ] + ), + user="root", + ) + await environment.upload_dir(self._config.task_snapshot, root) + git = f"git -c core.hooksPath=/dev/null -C {quoted}" + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"{git} init -b main", + f"{git} config user.email fixture@tangle.tools", + f'{git} config user.name "Tangle Fixture"', + f"{git} add -A", + f"GIT_AUTHOR_DATE=2000-01-01T00:00:00Z " + f"GIT_COMMITTER_DATE=2000-01-01T00:00:00Z " + f"{git} commit -m baseline", + f"{git} tag benchmark-base", + ] + ), + env=candidate_git_env(root), + user="root", + ) + + async def verify_pier_execution_identity( + self, environment: BaseEnvironment + ) -> None: + expected = self._contract.container + configured_image = getattr(environment.task_env_config, "docker_image", None) + pinned_image = f"{expected['image']}@{expected['indexDigest']}" + if configured_image != pinned_image: + raise self._error( + "Pier task image does not match the signed immutable image: " + f"expected={pinned_image} observed={configured_image}" + ) + configured_os = getattr(environment.task_env_config, "os", None) + configured_os = getattr(configured_os, "value", configured_os) + if configured_os is not None and str(configured_os).lower() != "linux": + raise self._error( + f"Pier task OS does not match signed Linux platform: {configured_os}" + ) + platform = expected["platform"] + observed = await self._host._exec( + environment, + 'printf \'%s %s\' "$(uname -s)" "$(uname -m)"', + user="root", + ) + kernel, _, machine = (observed.stdout or "").strip().partition(" ") + architectures = {"x86_64": "amd64", "aarch64": "arm64"} + if kernel != "Linux" or architectures.get(machine) != platform["architecture"]: + raise self._error( + "running Pier platform does not match the signed container platform: " + f"observed={kernel}/{machine} expected=linux/{platform['architecture']}" + ) + + async def verify_resolved_root_isolation( + self, environment: BaseEnvironment + ) -> None: + roots = [self._contract.task_root] + if self._contract.candidate_root is not None: + roots.append(self._contract.candidate_root) + for root in roots: + await self._host._assert_real_directory( + environment, root, label="execution workspace" + ) + if self._contract.candidate_root is not None: + task = shlex.quote(self._contract.task_root) + candidate = shlex.quote(self._contract.candidate_root) + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"task=$(realpath -e -- {task})", + f"candidate=$(realpath -e -- {candidate})", + 'test "$task" != "$candidate"', + 'case "$task" in "$candidate"/*) exit 1;; esac', + 'case "$candidate" in "$task"/*) exit 1;; esac', + ] + ), + user="root", + ) + for protected in self._config.protected_roots: + quoted_protected = shlex.quote(protected) + for root in roots: + quoted_root = shlex.quote(root) + await self._host._exec( + environment, + "\n".join( + [ + "set -eu", + f"if test -e {quoted_protected}; then", + f" protected=$(realpath -e -- {quoted_protected})", + f" root=$(realpath -e -- {quoted_root})", + ' test "$root" != "$protected"', + ' case "$root" in "$protected"/*) exit 1;; esac', + ' case "$protected" in "$root"/*) exit 1;; esac', + "fi", + ] + ), + user="root", + ) + + async def verify_container_workspace( + self, + environment: BaseEnvironment, + root: str, + files: tuple[WorkspaceFile, ...], + *, + allow_task_metadata: bool, + ) -> None: + quoted_root = shlex.quote(root) + prune = ( + f"\\( -path {shlex.quote(root + '/.git')} -o " + f"-path {shlex.quote(root + '/.sidecar')} \\) -prune -o " + if allow_task_metadata + else "" + ) + expected_args = " ".join(shlex.quote(file.path) for file in files) + expected = ( + f"$(printf '%s\\n' {expected_args} | LC_ALL=C sort)" if files else "''" + ) + checks = [ + "set -eu", + f"test -d {quoted_root}", + f"test ! -L {quoted_root}", + f'test "$(realpath -e -- {quoted_root})" = {quoted_root}', + f'test -z "$(find {quoted_root} {prune}-type l -print -quit)"', + f'test -z "$(find {quoted_root} {prune}! -type d ! -type f -print -quit)"', + f"observed=$(find {quoted_root} {prune}-type f -printf '%P\\n' | LC_ALL=C sort)", + f"expected={expected}", + 'test "$observed" = "$expected"', + ] + for file in files: + path = shlex.quote(f"{root}/{file.path}") + checks.extend( + [ + f"test -f {path}", + f"test ! -L {path}", + f'test "$(stat -c %h {path})" = 1', + f'test "$(stat -c %a {path})" = {file.mode:o}', + f'test "$(wc -c < {path})" = {file.byte_length}', + f"test \"$(sha256sum {path} | cut -d' ' -f1)\" = {file.sha256.removeprefix('sha256:')}", + ] + ) + await self._host._exec(environment, "\n".join(checks)) diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 78f1db81..caf019de 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -11,9 +11,12 @@ importers: '@tangle-network/agent-eval': specifier: ^0.108.1 version: 0.108.1(typescript@6.0.3) + '@tangle-network/agent-interface': + specifier: ^0.23.0 + version: 0.23.0 '@tangle-network/agent-runtime': specifier: ^0.90.1 - version: 0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) + version: 0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.23.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) '@tangle-network/sandbox': specifier: ^0.9.7 version: 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) @@ -268,6 +271,9 @@ packages: '@tangle-network/agent-interface@0.14.0': resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} + '@tangle-network/agent-interface@0.23.0': + resolution: {integrity: sha512-545pWgTQn9JixXeFJ7KIii1Y86i+26QXBvvsNvXKFDIuyzITUY73/DJt10JxUpcJZwFxzbgwUBKubOE+uiBz/Q==} + '@tangle-network/agent-knowledge@1.11.1': resolution: {integrity: sha512-2vvNHHsb4TQFnY+SrL7rYaMhwTxTK8R/lD9IAteonmgwcE5amhpPQZqftw+UDMl1d6OBe4QX+Qv1pNmWQDXzqQ==} engines: {node: '>=20'} @@ -589,6 +595,10 @@ snapshots: dependencies: zod: 4.4.3 + '@tangle-network/agent-interface@0.23.0': + dependencies: + zod: 4.4.3 + '@tangle-network/agent-knowledge@1.11.1(typescript@6.0.3)': dependencies: '@tangle-network/agent-eval': 0.108.1(typescript@6.0.3) @@ -602,12 +612,12 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-runtime@0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': + '@tangle-network/agent-runtime@0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.23.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': dependencies: '@tangle-network/agent-eval': 0.108.1(typescript@6.0.3) '@tangle-network/agent-knowledge': 1.11.1(typescript@6.0.3) optionalDependencies: - '@tangle-network/agent-interface': 0.14.0 + '@tangle-network/agent-interface': 0.23.0 '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) transitivePeerDependencies: - '@mastra/core' diff --git a/bench/scripts/run-package-tests.mjs b/bench/scripts/run-package-tests.mjs index ced391d7..2a3514da 100644 --- a/bench/scripts/run-package-tests.mjs +++ b/bench/scripts/run-package-tests.mjs @@ -51,4 +51,6 @@ await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], { TSX_TSCONFIG_PATH: 'tsconfig.public.json', }) -console.log(`package tests passed: ${tests.length}/${tests.length} files`) +await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py']) + +console.log(`package tests passed: ${tests.length}/${tests.length} TypeScript files + Pier bridge`) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 1542aafe..5af41e4e 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -8,10 +8,18 @@ import { promisify } from 'node:util' const execFileAsync = promisify(execFile) const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-consumer-')) +const runtimePackage = process.env.AGENT_RUNTIME_PACKAGE + ? path.resolve(process.env.AGENT_RUNTIME_PACKAGE) + : undefined -async function run(command, args, cwd) { +async function run(command, args, cwd, env = process.env) { try { - return await execFileAsync(command, args, { cwd, maxBuffer: 10 * 1024 * 1024, timeout: 120_000 }) + return await execFileAsync(command, args, { + cwd, + env, + maxBuffer: 10 * 1024 * 1024, + timeout: 120_000, + }) } catch (error) { if (error?.stdout) process.stdout.write(error.stdout) if (error?.stderr) process.stderr.write(error.stderr) @@ -40,10 +48,15 @@ try { !devDependencies.typescript || !devDependencies.tsx ) { - throw new Error('package verification requires @types/node, typescript, and tsx devDependencies') + throw new Error( + 'package verification requires @types/node, typescript, and tsx devDependencies', + ) } - const publicTsconfig = JSON.parse(await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8')) - if (!publicTsconfig.compilerOptions) throw new Error('tsconfig.public.json must define compilerOptions') + const publicTsconfig = JSON.parse( + await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8'), + ) + if (!publicTsconfig.compilerOptions) + throw new Error('tsconfig.public.json must define compilerOptions') await writeFile( path.join(consumerDir, 'package.json'), @@ -52,7 +65,12 @@ try { name: 'agent-bench-package-verifier', private: true, type: 'module', - dependencies: { '@tangle-network/agent-bench': `file:${tarball}` }, + dependencies: { + '@tangle-network/agent-bench': `file:${tarball}`, + ...(runtimePackage + ? { '@tangle-network/agent-runtime': `file:${runtimePackage}` } + : {}), + }, devDependencies: { '@types/node': devDependencies['@types/node'], typescript: devDependencies.typescript, @@ -65,7 +83,7 @@ try { ) await writeFile( path.join(consumerDir, 'index.ts'), - "import { resolveAdapter, runBenchmarks, type BenchmarkAdapter } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nvoid adapter\nvoid runBenchmarks\n", + "import { executePreparedPierCandidate, resolveAdapter, runBenchmarks, type BenchmarkAdapter, type PierCandidateTrialHandle, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nvoid adapter\nvoid staged\nvoid trial\nvoid executePreparedPierCandidate\nvoid runBenchmarks\n", ) await writeFile( path.join(consumerDir, 'tsconfig.json'), @@ -78,19 +96,68 @@ try { 2, )}\n`, ) + await writeFile( + path.join(consumerDir, 'verify_pier_import.py'), + `import sys +import types + +modules = { + name: types.ModuleType(name) + for name in ( + "pier", + "pier.agents", + "pier.agents.base", + "pier.agents.installed", + "pier.agents.installed.base", + "pier.environments", + "pier.environments.base", + "pier.models", + "pier.models.agent", + "pier.models.agent.context", + "pier.models.agent.network", + ) +} +modules["pier.agents.base"].BaseAgent = type("BaseAgent", (), {}) +modules["pier.agents.installed.base"].NonZeroAgentExitCodeError = type( + "NonZeroAgentExitCodeError", (RuntimeError,), {} +) +modules["pier.environments.base"].BaseEnvironment = type("BaseEnvironment", (), {}) +modules["pier.models.agent.context"].AgentContext = type("AgentContext", (), {}) +modules["pier.models.agent.network"].NetworkAllowlist = type("NetworkAllowlist", (), {}) +sys.modules.update(modules) + +from pier_agents.tangle_candidate import TangleCandidateAgent + +assert TangleCandidateAgent.__module__ == "pier_agents.tangle_candidate" +`, + ) // Intentionally resolve the declared ranges like a brand-new registry consumer. // The preceding frozen install + public typecheck cover the exact bench lockfile. - await run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'], consumerDir) + await run( + 'npm', + ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'], + consumerDir, + ) await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir) + const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench') + await run('python3', ['verify_pier_import.py'], consumerDir, { + ...process.env, + PYTHONPATH: installedPackage, + }) let runtimeManifest try { runtimeManifest = JSON.parse( - await readFile(path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'), 'utf8'), + await readFile( + path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'), + 'utf8', + ), ) } catch (error) { - throw new Error('packed consumer did not install @tangle-network/agent-runtime', { cause: error }) + throw new Error('packed consumer did not install @tangle-network/agent-runtime', { + cause: error, + }) } console.log( `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}`, diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts new file mode 100644 index 00000000..d5f54167 --- /dev/null +++ b/bench/scripts/verify-pier-agent.mts @@ -0,0 +1,744 @@ +import { execFileSync, spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + chmodSync, + cpSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { canonicalJson, InMemoryTraceStore } from '@tangle-network/agent-eval' +import type { + AgentCandidateArtifactRef, + AgentCandidateBundle, + AgentCandidateWorkspaceSnapshotEvidence, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + FileAgentCandidateExecutionClaimStore, + prepareAgentCandidateExecution, + type AgentCandidateExecutionPorts, + type AgentCandidateOutputArtifactPort, + type AgentCandidateTaskExecution, + type ResolvedAgentCandidateContainer, + verifyAgentCandidateBundle, +} from '@tangle-network/agent-runtime' + +import { executePreparedPierCandidate } from '../src/pier-agent' +import { createPierResultGrader } from '../src/pier-result-grader' +import { materializePierWorkspaceArchive } from '../src/pier-workspace-archive' + +const pinnedPierCommit = 'e69a20e4e0ac073ec71fde0274bab3d9f40bac87' +const pinnedPierVersion = '0.3.0' +const modelRequest = 'openai/gpt-5.4' +const fixtureImage = 'ghcr.io/tangle-network/devcontainers/universal:latest' +const proofArm = process.env.PIER_PROOF_ARM +if (proofArm !== 'failure' && proofArm !== 'success') { + throw new Error('PIER_PROOF_ARM must be failure or success') +} +const expectedReward = proofArm === 'success' ? 1 : 0 +const expectedPatchApplied = proofArm === 'success' ? 1 : 0 +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const pierRepo = path.resolve(process.env.PIER_REPO ?? path.join(benchDir, '..', '..', 'pier')) +const fixtureSource = path.join(benchDir, 'fixtures', 'pier-agent', 'no-model-task') +const scratch = mkdtempSync(path.join(tmpdir(), 'agent-bench-pier-runtime-')) +const taskDir = path.join(scratch, 'task') +const taskRoot = path.join(scratch, 'task-workspace') +const candidateRoot = path.join(scratch, 'candidate-workspace') +const profileRoot = path.join(scratch, 'profile-workspace') +const jobsDir = path.join(scratch, 'jobs') + +function output( + command: string, + args: string[], + cwd = benchDir, + env: NodeJS.ProcessEnv = process.env, +): string { + return execFileSync(command, args, { + cwd, + env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 10 * 60_000, + maxBuffer: 20 * 1024 * 1024, + }).trim() +} + +function containerIdsForImage(image: string): Set { + const ids = output('docker', ['ps', '-aq', '--filter', `ancestor=${image}`]) + return new Set(ids ? ids.split('\n').filter(Boolean) : []) +} + +function newContainerIds(image: string, before: ReadonlySet): string[] { + return [...containerIdsForImage(image)].filter((id) => !before.has(id)) +} + +function killProcessGroup(pid: number, signal: NodeJS.Signals): void { + try { + process.kill(-pid, signal) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } +} + +function startPierProcess( + args: readonly string[], + env: NodeJS.ProcessEnv, + taskImage: string, + protectedValues: readonly string[], +) { + const containersBefore = containerIdsForImage(taskImage) + const child = spawn('uv', [...args], { + cwd: pierRepo, + env, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + }) + if (child.pid === undefined) throw new Error('Pier process started without a pid') + const pid = child.pid + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + child.stdout.on('data', (chunk: Buffer) => stdout.push(Buffer.from(chunk))) + child.stderr.on('data', (chunk: Buffer) => stderr.push(Buffer.from(chunk))) + const exited = new Promise<{ + code: number | null + signal: NodeJS.Signals | null + error?: Error + }>((resolveExit) => { + let settled = false + child.once('error', (error) => { + if (settled) return + settled = true + resolveExit({ code: null, signal: null, error }) + }) + child.once('close', (code, signal) => { + if (settled) return + settled = true + resolveExit({ code, signal }) + }) + }) + + const removeTaskContainers = (): void => { + const containers = newContainerIds(taskImage, containersBefore) + if (containers.length > 0) output('docker', ['rm', '-f', ...containers]) + const remaining = newContainerIds(taskImage, containersBefore) + if (remaining.length > 0) { + throw new Error(`Pier task containers survived removal: ${remaining.join(', ')}`) + } + } + + const result = exited.then((exit) => { + removeTaskContainers() + if (exit.error) throw new Error(`Pier process failed to start: ${exit.error.message}`) + if (exit.code !== 0) { + let safeStderr = Buffer.concat(stderr).toString('utf8') + for (const value of protectedValues) safeStderr = safeStderr.replaceAll(value, '[redacted]') + throw new Error(`Pier process failed (${exit.signal ?? exit.code}): ${safeStderr}`) + } + return { + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8'), + } + }) + + let terminating: Promise<{ processExited: true; containersRemoved: true }> | undefined + return { + result, + terminateAndWait: () => { + terminating ??= (async () => { + killProcessGroup(pid, 'SIGTERM') + const graceful = await Promise.race([ + exited.then(() => true), + new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), 2_000)), + ]) + if (!graceful) { + killProcessGroup(pid, 'SIGKILL') + const killed = await Promise.race([ + exited.then(() => true), + new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), 5_000)), + ]) + if (!killed) throw new Error(`Pier process group ${pid} survived SIGKILL`) + } + + removeTaskContainers() + return { processExited: true as const, containersRemoved: true as const } + })() + return terminating + }, + } +} + +function sha256(bytes: Uint8Array): Sha256Digest { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +function canonicalBytes(value: unknown): Buffer { + return Buffer.from(canonicalJson(value), 'utf8') +} + +function embedded(bytes: Uint8Array) { + return { + encoding: 'base64' as const, + content: Buffer.from(bytes).toString('base64'), + sha256: sha256(bytes), + byteLength: bytes.byteLength, + } +} + +function workspaceSnapshot( + root: string, + paths: readonly string[], +): AgentCandidateWorkspaceSnapshotEvidence { + const files = paths + .map((relative) => { + const absolute = path.join(root, relative) + const bytes = readFileSync(absolute) + const mode = lstatSync(absolute).mode & 0o777 + if (mode !== 0o644 && mode !== 0o755) { + throw new Error(`unsupported fixture mode ${mode.toString(8)}: ${relative}`) + } + return { + path: relative, + mode: mode as 0o644 | 0o755, + sha256: sha256(bytes), + byteLength: bytes.byteLength, + } + }) + .sort((left, right) => left.path.localeCompare(right.path)) + const material = { + schemaVersion: 1 as const, + kind: 'agent-candidate-workspace-manifest' as const, + files, + } + const manifest = canonicalBytes(material) + return { + schemaVersion: 1, + kind: 'agent-candidate-workspace-snapshot', + digest: sha256(manifest), + material, + manifest: embedded(manifest), + archive: embedded(Buffer.from(`pre-materialized:${sha256(manifest)}`, 'utf8')), + } +} + +function deterministicRepository(root: string, seed: string): { commit: string; tree: string } { + mkdirSync(path.join(root, 'src'), { recursive: true }) + cpSync(seed, path.join(root, 'src', 'status.txt')) + chmodSync(path.join(root, 'src', 'status.txt'), 0o644) + output('git', ['init', '-b', 'main', root]) + output('git', ['-C', root, 'config', 'user.email', 'fixture@tangle.tools']) + output('git', ['-C', root, 'config', 'user.name', 'Tangle Fixture']) + output('git', [ + '-C', + root, + 'remote', + 'add', + 'origin', + 'git@github.com:tangle-network/agent-bench-pier-fixture.git', + ]) + output('git', ['-c', 'core.hooksPath=/dev/null', '-C', root, 'add', '-A']) + output( + 'git', + ['-c', 'core.hooksPath=/dev/null', '-C', root, 'commit', '-m', 'baseline'], + benchDir, + { + ...process.env, + GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', + GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z', + }, + ) + return { + commit: output('git', ['-C', root, 'rev-parse', 'HEAD']), + tree: output('git', ['-C', root, 'rev-parse', 'HEAD^{tree}']), + } +} + +function publicOciIdentity( + image: string, +): { indexDigest: Sha256Digest; manifestDigest: Sha256Digest } { + const material = JSON.parse( + output('docker', ['buildx', 'imagetools', 'inspect', image, '--format', '{{json .Manifest}}']), + ) as { + digest?: string + manifests?: Array<{ + digest?: string + platform?: { os?: string; architecture?: string } + }> + } + const topDigest = material.digest + if (!topDigest?.match(/^sha256:[a-f0-9]{64}$/)) { + throw new Error(`public image omitted a valid index digest: ${topDigest}`) + } + const selected = material.manifests?.find( + (entry) => entry.platform?.os === 'linux' && entry.platform.architecture === 'amd64', + ) + const manifestDigest = selected?.digest + if (!manifestDigest) throw new Error('public image has no linux/amd64 manifest') + if (!manifestDigest.match(/^sha256:[a-f0-9]{64}$/)) { + throw new Error(`public image returned an invalid platform manifest: ${manifestDigest}`) + } + return { + indexDigest: topDigest as Sha256Digest, + manifestDigest: manifestDigest as Sha256Digest, + } +} + +function findTrialResult(root: string): { path: string; value: Record } | undefined { + for (const entry of readdirSync(root, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + const child = path.join(root, entry.name) + const candidate = path.join(child, 'result.json') + try { + const value = JSON.parse(readFileSync(candidate, 'utf8')) as Record + if (typeof value.trial_name === 'string') return { path: candidate, value } + } catch {} + const nested = findTrialResult(child) + if (nested) return nested + } + return undefined +} + +function assertTreeOmits(root: string, forbidden: string): void { + for (const entry of readdirSync(root, { withFileTypes: true })) { + const absolute = path.join(root, entry.name) + if (entry.isDirectory()) { + assertTreeOmits(absolute, forbidden) + } else if (entry.isFile() && readFileSync(absolute).includes(Buffer.from(forbidden))) { + throw new Error(`protected value persisted in ${path.relative(root, absolute)}`) + } + } +} + +function outputArtifactStore(graderBytes: Uint8Array): { + outputArtifacts: AgentCandidateOutputArtifactPort + graderArtifact: AgentCandidateArtifactRef +} { + const stored = new Map() + const put = (bytes: Uint8Array, purpose: string): AgentCandidateArtifactRef => { + const detached = Uint8Array.from(bytes) + const digest = sha256(detached) + const artifact: AgentCandidateArtifactRef = { + locator: { + kind: 's3', + bucket: 'agent-bench-pier-proof', + key: `${purpose}/${digest.slice('sha256:'.length)}`, + }, + sha256: digest, + byteLength: detached.byteLength, + } + stored.set(digest, detached) + return artifact + } + const graderArtifact = put(graderBytes, 'graders') + return { + graderArtifact, + outputArtifacts: { + put: async ({ bytes, purpose, signal }) => { + signal?.throwIfAborted() + return put(bytes, purpose) + }, + read: async (artifact) => { + const bytes = stored.get(artifact.sha256) + if (!bytes) throw new Error(`missing Pier proof artifact ${artifact.sha256}`) + return Uint8Array.from(bytes) + }, + }, + } +} + +try { + const pierHead = output('git', ['rev-parse', 'HEAD'], pierRepo) + if (pierHead !== pinnedPierCommit) { + throw new Error(`Pier checkout mismatch: expected ${pinnedPierCommit}, got ${pierHead}`) + } + const pierStatus = output('git', ['status', '--porcelain'], pierRepo) + if (pierStatus !== '') throw new Error(`Pier checkout must be clean: ${pierStatus}`) + const pierVersion = output('uv', ['run', 'pier', '--version'], pierRepo) + if (pierVersion !== pinnedPierVersion) { + throw new Error(`Pier version mismatch: expected ${pinnedPierVersion}, got ${pierVersion}`) + } + + cpSync(fixtureSource, taskDir, { recursive: true }) + for (const relative of ['environment/seed/src/status.txt', 'tests/seed/src/status.txt']) { + chmodSync(path.join(taskDir, relative), 0o644) + } + for (const relative of ['pre_artifacts.sh', 'tests/test.sh']) { + chmodSync(path.join(taskDir, relative), 0o755) + } + + const contextDigest = sha256( + Buffer.concat([ + readFileSync(path.join(taskDir, 'environment', 'Dockerfile')), + readFileSync(path.join(taskDir, 'environment', 'seed', 'src', 'status.txt')), + ]), + ).slice(7, 23) + const identity = publicOciIdentity(fixtureImage) + const pinnedImage = `${fixtureImage}@${identity.indexDigest}` + output('docker', ['pull', '--platform', 'linux/amd64', pinnedImage]) + const platform = output('docker', [ + 'image', + 'inspect', + '--format', + '{{.Os}}/{{.Architecture}}', + pinnedImage, + ]) + if (platform !== 'linux/amd64') throw new Error(`fixture image platform drifted: ${platform}`) + + const configPath = path.join(taskDir, 'task.toml') + const config = readFileSync(configPath, 'utf8') + writeFileSync( + configPath, + config.replace( + '[environment]\n', + `[environment]\ndocker_image = "${pinnedImage}"\nos = "linux"\n`, + ), + ) + + mkdirSync(candidateRoot) + mkdirSync(profileRoot) + const instruction = readFileSync(path.join(taskDir, 'instruction.md'), 'utf8') + const runner = `import pathlib, sys +task = pathlib.Path.cwd() +profile = task / 'AGENTS.md' +assert profile.is_file() and profile.read_text().strip() +assert sys.argv[-1] == ${JSON.stringify(instruction)} +${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=tangle\\n')" : ''} +` + writeFileSync(path.join(candidateRoot, 'runner.py'), runner) + chmodSync(path.join(candidateRoot, 'runner.py'), 0o755) + const repositoryState = deterministicRepository( + taskRoot, + path.join(taskDir, 'environment', 'seed', 'src', 'status.txt'), + ) + + const taskWorkspace = workspaceSnapshot(taskRoot, ['src/status.txt']) + const candidateWorkspace = workspaceSnapshot(candidateRoot, ['runner.py']) + const bundleWithoutDigest = { + schemaVersion: 1 as const, + kind: 'agent-candidate-bundle' as const, + digestAlgorithm: 'rfc8785-sha256' as const, + profile: { + name: `pier-no-model-runtime-${proofArm}`, + prompt: { + instructions: [ + 'Edit only the task repository. Follow the exact user instruction and verify the result.', + ], + }, + model: { default: modelRequest, reasoningEffort: 'xhigh' as const }, + harness: 'codex' as const, + resources: { failOnError: true as const }, + }, + code: { + kind: 'no-op' as const, + reason: 'proposer-no-change' as const, + repository: { + kind: 'github' as const, + owner: 'tangle-network', + repo: 'agent-bench-pier-fixture', + }, + baseCommit: repositoryState.commit, + baseTree: repositoryState.tree, + }, + execution: { + harness: 'codex' as const, + harnessVersion: 'agent-bench-pier/2.0.0', + launch: { + kind: 'candidate-entrypoint' as const, + entrypoint: 'runner.py', + interpreter: 'python3' as const, + }, + instructionDelivery: { kind: 'argv-append' as const }, + cwd: { workspace: 'task' as const, path: '.' }, + environment: { kind: 'evaluator-task-container' as const }, + workspace: candidateWorkspace, + isolation: { + network: 'disabled' as const, + remoteIntegrations: 'disabled' as const, + candidateSecrets: 'disabled' as const, + }, + }, + memory: { mode: 'disabled' as const }, + lineage: { + source: 'optimizer' as const, + parentDigests: [`sha256:${'a'.repeat(64)}` as Sha256Digest], + runIds: [`pier-no-model-proposer-${proofArm}`], + benchmark: { + name: 'pier-runtime-proof', + version: '1', + splitDigest: `sha256:${'b'.repeat(64)}` as Sha256Digest, + }, + spend: { + proposal: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + evaluation: { costUsd: 0, inputTokens: 0, outputTokens: 0, modelCalls: 0 }, + }, + }, + } + const bundle = { + ...bundleWithoutDigest, + digest: sha256(canonicalBytes(bundleWithoutDigest)), + } as AgentCandidateBundle + const container: ResolvedAgentCandidateContainer = { + source: 'evaluator-task-container', + image: fixtureImage, + indexDigest: identity.indexDigest, + manifestDigest: identity.manifestDigest, + platform: { os: 'linux', architecture: 'amd64' }, + } + const executionId = `pier-no-model-${proofArm}-${contextDigest}` + const task: AgentCandidateTaskExecution = { + executionId, + benchmark: 'pier-runtime-proof', + benchmarkVersion: '1', + taskId: `agent-bench/pier-candidate-no-model-${proofArm}`, + splitDigest: `sha256:${'b'.repeat(64)}`, + instruction, + repository: { + identity: 'github.com/tangle-network/agent-bench-pier-fixture', + rootIdentity: 'tangle-network/agent-bench-pier-fixture', + baseCommit: repositoryState.commit, + baseTree: repositoryState.tree, + }, + attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, + model: { requested: modelRequest, reasoningEffort: 'xhigh' }, + executionRoots: { taskRoot: '/app', candidateRoot: '/opt/tangle-candidate' }, + stagingRoots: { taskRoot, candidateRoot, profileRoot }, + workspace: taskWorkspace, + evaluatorTaskContainer: container, + limits: { + timeoutMs: 60_000, + maxSteps: 8, + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }, + } + const ports: AgentCandidateExecutionPorts = { + artifacts: { + read: async () => { + throw new Error('the proof uses only embedded artifacts') + }, + }, + repositories: { resolve: async () => taskRoot }, + workspaces: { + materialize: async ({ snapshot, archive, destination }) => { + if (destination === taskRoot || destination === candidateRoot) return + await materializePierWorkspaceArchive({ archive, expected: snapshot.material, destination }) + }, + }, + containers: { resolve: async () => container }, + models: { + resolve: async ({ requested, reasoningEffort }) => { + if (!reasoningEffort) throw new Error('resolved model requires reasoning effort') + return { + requested, + provider: 'openai', + model: 'gpt-5.4', + snapshot: 'gpt-5.4-no-model-proof', + reasoningEffort, + } + }, + reserveGrant: async ({ limits, preparationId, expiresAtMs }) => ({ + preparationId, + digest: `sha256:${'c'.repeat(64)}`, + expiresAtMs, + enforcedLimits: limits, + network: { mode: 'disabled' }, + }), + activateGrant: async () => ({ + env: { MODEL_GATEWAY_TOKEN: 'zero-model-proof' }, + }), + settleGrant: async ({ preparationId }) => ({ + preparationId, + grantDigest: `sha256:${'c'.repeat(64)}`, + closed: true, + calls: [], + }), + }, + memory: { + reset: async () => { + throw new Error('disabled memory must not be reset') + }, + activate: async () => { + throw new Error('disabled memory must not be activated') + }, + close: async () => { + throw new Error('disabled memory must not be closed') + }, + }, + } + + const verified = await verifyAgentCandidateBundle(bundle, ports) + const prepared = await prepareAgentCandidateExecution(verified, task, ports) + const traceStore = new InMemoryTraceStore() + const claimStore = new FileAgentCandidateExecutionClaimStore({ + directory: path.join(scratch, 'claims'), + }) + const graderBytes = readFileSync(new URL('../src/pier-result-grader.mjs', import.meta.url)) + const { outputArtifacts, graderArtifact } = outputArtifactStore(graderBytes) + const grader = createPierResultGrader({ + name: 'pier-official-result', + version: '1.0.0', + artifact: graderArtifact, + }) + let acceptedRewards: { reward: number; patch_applied: number } | undefined + let acceptedTrialPath: string | undefined + const finalized = await executePreparedPierCandidate({ + prepared, + directory: path.join(scratch, 'sealed'), + pierVersion: pinnedPierVersion, + traceStore, + claimStore, + outputArtifacts, + grader, + start: (staged, { request }) => { + const evaluatorArgs = Object.keys(staged.evaluatorEnv).flatMap((name) => [ + '--agent-env', + `${name}=\${${name}}`, + ]) + const trial = startPierProcess( + [ + 'run', + 'pier', + 'run', + '--path', + taskDir, + ...staged.agentArgs, + ...staged.attemptArgs, + ...evaluatorArgs, + '--env', + 'docker', + '--job-name', + `tangle-runtime-candidate-no-model-${proofArm}`, + '--jobs-dir', + jobsDir, + '--n-concurrent', + '1', + '--agent-timeout-multiplier', + '2', + '--quiet', + ], + { ...process.env, PYTHONPATH: benchDir, ...staged.evaluatorEnv }, + pinnedImage, + Object.values(staged.evaluatorEnv), + ) + + return { + ...trial, + result: trial.result.then(async () => { + const trialResult = findTrialResult(jobsDir) + if (!trialResult) throw new Error(`Pier emitted no trial result under ${jobsDir}`) + const result = trialResult.value + if (result.exception_info !== null) { + throw new Error( + `Pier trial captured an exception: ${JSON.stringify(result.exception_info)}`, + ) + } + const rewards = result.verifier_result?.rewards + if ( + rewards?.reward !== expectedReward || + rewards?.patch_applied !== expectedPatchApplied + ) { + throw new Error( + `Pier ${proofArm} control returned unexpected rewards: ${JSON.stringify(rewards)}`, + ) + } + const agentResult = result.agent_result + for (const name of ['n_input_tokens', 'n_cache_tokens', 'n_output_tokens', 'cost_usd']) { + if (agentResult?.[name] !== null) { + throw new Error( + `Pier must not author protected usage ${name}: ${agentResult?.[name]}`, + ) + } + } + const observedElapsedMs = agentResult?.metadata?.observedElapsedMs + if (!Number.isInteger(observedElapsedMs) || observedElapsedMs < 0) { + throw new Error(`Pier omitted protected elapsed time: ${observedElapsedMs}`) + } + const endedAt = Date.now() + await traceStore.appendRun({ + runId: request.trace.runId, + scenarioId: task.taskId, + startedAt: endedAt - observedElapsedMs, + endedAt, + status: 'completed', + tags: { ...request.trace.tags }, + }) + acceptedRewards = rewards + acceptedTrialPath = path.relative(jobsDir, trialResult.path) + return { + value: result, + resultBytes: readFileSync(trialResult.path), + taskPatch: readFileSync( + path.join(path.dirname(trialResult.path), 'artifacts', 'model.patch'), + ), + } + }), + } + }, + }) + if (!finalized.succeeded) { + throw new Error(`runtime rejected the protected Pier capture: ${finalized.reason}`) + } + if (!acceptedRewards || !acceptedTrialPath) { + throw new Error('atomic Pier executor returned without accepted verifier evidence') + } + if ( + finalized.receipt.value.benchmarkResult.material.score !== expectedReward || + finalized.receipt.value.benchmarkResult.material.passed !== (expectedReward === 1) + ) { + throw new Error( + `runtime receipt rejected the official Pier result: ${JSON.stringify(finalized.receipt.value.benchmarkResult.material)}`, + ) + } + assertTreeOmits(scratch, 'zero-model-proof') + const usage = finalized.receipt.value.usage + if ( + usage.modelCalls !== 0 || + usage.inputTokens !== 0 || + usage.outputTokens !== 0 || + usage.costUsd !== 0 || + finalized.receipt.value.trace.modelCallCount !== 0 + ) { + throw new Error(`runtime minted nonzero usage for a zero-model run: ${JSON.stringify(usage)}`) + } + + console.log( + JSON.stringify( + { + arm: proofArm, + reward: acceptedRewards.reward, + patchApplied: acceptedRewards.patch_applied, + modelCalls: usage.modelCalls, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + costUsd: usage.costUsd, + pierContextUsage: null, + profileExcludedByVerifier: true, + container: { + image: fixtureImage, + indexDigest: identity.indexDigest, + manifestDigest: identity.manifestDigest, + platform, + }, + executionPlanDigest: prepared.executionPlan.value.digest, + materializationReceiptDigest: prepared.materializationReceipt.digest, + runReceiptDigest: finalized.receipt.digest, + trial: acceptedTrialPath, + }, + null, + 2, + ), + ) +} finally { + if (process.env.KEEP_PIER_FIXTURE !== '1') rmSync(scratch, { recursive: true, force: true }) + else console.error(`Pier runtime fixture retained at ${scratch}`) +} diff --git a/bench/scripts/verify-pier-pair.mts b/bench/scripts/verify-pier-pair.mts new file mode 100644 index 00000000..9d6e0e25 --- /dev/null +++ b/bench/scripts/verify-pier-pair.mts @@ -0,0 +1,57 @@ +import { execFileSync } from 'node:child_process' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +interface PierProofResult { + readonly arm: 'failure' | 'success' + readonly reward: number + readonly patchApplied: number + readonly modelCalls: number + readonly inputTokens: number + readonly outputTokens: number + readonly costUsd: number + readonly runReceiptDigest: string +} + +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const script = path.join(benchDir, 'scripts', 'verify-pier-agent.mts') + +function runArm(arm: PierProofResult['arm']): PierProofResult { + const stdout = execFileSync(process.execPath, ['--import', 'tsx', script], { + cwd: benchDir, + env: { ...process.env, PIER_PROOF_ARM: arm }, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + timeout: 20 * 60_000, + maxBuffer: 20 * 1024 * 1024, + }) + return JSON.parse(stdout) as PierProofResult +} + +const failure = runArm('failure') +const success = runArm('success') +if ( + failure.arm !== 'failure' || + failure.reward !== 0 || + failure.patchApplied !== 0 || + success.arm !== 'success' || + success.reward !== 1 || + success.patchApplied !== 1 +) { + throw new Error(`Pier controls did not separate: ${JSON.stringify({ failure, success })}`) +} +for (const result of [failure, success]) { + if ( + result.modelCalls !== 0 || + result.inputTokens !== 0 || + result.outputTokens !== 0 || + result.costUsd !== 0 + ) { + throw new Error(`Pier ${result.arm} control spent model budget: ${JSON.stringify(result)}`) + } + if (!/^sha256:[a-f0-9]{64}$/.test(result.runReceiptDigest)) { + throw new Error(`Pier ${result.arm} control omitted its run receipt digest`) + } +} + +console.log(JSON.stringify({ controls: [failure, success] }, null, 2)) diff --git a/bench/src/index.ts b/bench/src/index.ts index a3d522d2..412a3db0 100644 --- a/bench/src/index.ts +++ b/bench/src/index.ts @@ -53,3 +53,21 @@ export { type RunBenchmarksOptions, type RunBenchmarksReport, } from './run-benchmarks' +export { + executePreparedPierCandidate, + type ExecutePreparedPierCandidateOptions, + type PierCandidateGraderPort, + type PierCandidateOfficialResult, + type PierCandidateTerminationAcknowledgement, + type PierCandidateTrialHandle, + type PierCandidateTrialResult, + type StagedPierCandidateExecution, +} from './pier-agent' +export { createPierResultGrader } from './pier-result-grader' +export { + createPierWorkspaceArchive, + materializePierWorkspaceArchive, + type PierWorkspaceArchiveFile, + type PierWorkspaceArchiveV1, + type PierWorkspaceFile, +} from './pier-workspace-archive' diff --git a/bench/src/pier-agent.test.mts b/bench/src/pier-agent.test.mts new file mode 100644 index 00000000..0128dee2 --- /dev/null +++ b/bench/src/pier-agent.test.mts @@ -0,0 +1,342 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { canonicalJson } from '@tangle-network/agent-eval' +import type { + AgentCandidateExecutorRequest, + PreparedAgentCandidateExecution, +} from '@tangle-network/agent-runtime' + +import { + awaitAbortableTrial, + protectedCaptureFromPierResult, + stagePreparedPierCandidateExecution, +} from './pier-agent' + +const digest = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +function prepared(root: string): PreparedAgentCandidateExecution { + const bundleDigest = `sha256:${'b'.repeat(64)}` as const + const executionId = 'pier-test-execution' + const material = { + schemaVersion: 1, + kind: 'agent-candidate-execution-plan-material', + bundleDigest, + executionId, + limits: { + timeoutMs: 60_000, + maxSteps: 8, + maxModelCalls: 0, + maxInputTokens: 0, + maxOutputTokens: 0, + maxCostUsd: 0, + }, + } + const planBytes = Buffer.from(canonicalJson(material)) + const planDigest = digest(planBytes) + const receiptMaterial = { + schemaVersion: 1, + kind: 'agent-candidate-materialization', + digestAlgorithm: 'rfc8785-sha256', + bundleDigest, + executionPlan: { digest: planDigest, material }, + } + const receiptBytes = Buffer.from(canonicalJson(receiptMaterial)) + const receiptDigest = digest(receiptBytes) + const profileBytes = Buffer.from('fixture profile\n') + return { + bundle: { digest: bundleDigest }, + executionId, + roots: { + execution: { taskRoot: '/app' }, + staging: { + taskRoot: join(root, 'task'), + profileRoot: join(root, 'profile'), + }, + }, + executionPlan: { + value: { digest: planDigest, material }, + bytes: planBytes, + }, + profilePlan: { + value: { + material: { + files: [ + { + relPath: 'AGENTS.md', + mode: 0o644, + contentSha256: digest(profileBytes), + }, + ], + }, + }, + bytes: Buffer.from('{}'), + written: ['AGENTS.md'], + }, + materializationReceipt: { + value: { ...receiptMaterial, digest: receiptDigest }, + bytes: receiptBytes, + digest: receiptDigest, + }, + resolvedModel: { + requested: 'openai/gpt-5.4', + provider: 'openai', + model: 'gpt-5.4', + snapshot: 'gpt-5.4-2026-06-01', + reasoningEffort: 'xhigh', + }, + launch: { + executable: 'python3', + args: ['/opt/tangle-candidate/runner.py'], + env: { PUBLIC_MODE: 'fixture' }, + flags: [], + cwd: '/app', + }, + memory: { mode: 'disabled' }, + trace: { + runId: executionId, + tags: {}, + env: { + TANGLE_CANDIDATE_EXECUTION_ID: executionId, + TANGLE_CANDIDATE_BUNDLE_DIGEST: bundleDigest, + TANGLE_CANDIDATE_EXECUTION_PLAN_DIGEST: planDigest, + TANGLE_CANDIDATE_MATERIALIZATION_RECEIPT_DIGEST: receiptDigest, + TANGLE_TRACE_RUN_ID: executionId, + }, + }, + } as unknown as PreparedAgentCandidateExecution +} + +function request(execution: PreparedAgentCandidateExecution): AgentCandidateExecutorRequest { + const taskBytes = Buffer.from('not ready\n') + const profileBytes = Buffer.from('fixture profile\n') + return { + executionId: execution.executionId, + inputs: { + task: { + snapshot: { + material: { + files: [ + { + path: 'src/status.txt', + mode: 0o644, + sha256: digest(taskBytes), + byteLength: taskBytes.byteLength, + }, + ], + }, + }, + files: [{ path: 'src/status.txt', mode: 0o644, bytes: taskBytes }], + }, + profile: { + files: [{ path: 'AGENTS.md', mode: 0o644, bytes: profileBytes }], + }, + }, + roots: execution.roots.execution, + profilePlan: execution.profilePlan, + executionPlan: execution.executionPlan, + materializationReceipt: execution.materializationReceipt, + launch: { + ...execution.launch, + env: { + ...execution.launch.env, + OPENAI_API_KEY: 'evaluator-only', + ...execution.trace.env, + }, + }, + instruction: execution.instruction, + resolvedModel: execution.resolvedModel, + hardLimits: { timeoutMs: execution.executionPlan.value.material.limits.timeoutMs }, + observedLimits: { maxSteps: execution.executionPlan.value.material.limits.maxSteps }, + trace: execution.trace, + memory: execution.memory, + } as unknown as AgentCandidateExecutorRequest +} + +test('stages the runtime exact bytes without defining another execution plan', async () => { + const root = await mkdtemp(join(tmpdir(), 'pier-stage-')) + try { + const execution = prepared(root) + const staged = await stagePreparedPierCandidateExecution( + { + prepared: execution, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + request(execution), + ) + assert.deepEqual(await readFile(staged.planPath), Buffer.from(execution.executionPlan.bytes)) + assert.deepEqual( + await readFile(staged.receiptPath), + Buffer.from(execution.materializationReceipt.bytes), + ) + assert.equal( + await readFile(join(staged.taskDirectory, 'src/status.txt'), 'utf8'), + 'not ready\n', + ) + assert.equal( + await readFile(join(staged.profileDirectory, 'AGENTS.md'), 'utf8'), + 'fixture profile\n', + ) + assert.deepEqual(staged.evaluatorEnv, { + ...execution.trace.env, + OPENAI_API_KEY: 'evaluator-only', + }) + assert.doesNotMatch(JSON.stringify(execution), /evaluator-only|OPENAI_API_KEY/) + assert.doesNotMatch(staged.agentArgs.join(' '), /evaluator-only|OPENAI_API_KEY/) + assert.deepEqual(staged.attemptArgs, ['--n-attempts', '1', '--max-retries', '0']) + assert.ok(staged.agentArgs.includes('pier_agents.tangle_candidate:TangleCandidateAgent')) + assert.ok(staged.agentArgs.includes('openai/gpt-5.4')) + assert.ok( + staged.agentArgs.includes( + `expected_receipt_digest=${execution.materializationReceipt.digest}`, + ), + ) + assert.ok(staged.agentArgs.includes(`task_dir=${staged.taskDirectory}`)) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('reads only identity and termination from Pier, never candidate-authored usage', () => { + const root = '/tmp/pier-capture-test' + const execution = prepared(root) + const metadata = { + executionId: execution.executionId, + bundleDigest: execution.bundle.digest, + executionPlanDigest: execution.executionPlan.value.digest, + materializationReceiptDigest: execution.materializationReceipt.digest, + termination: { kind: 'exit', exitCode: 0 }, + } + const capture = protectedCaptureFromPierResult(request(execution), { + exception_info: null, + agent_result: { + n_input_tokens: 999_999, + cost_usd: 999_999, + metadata, + }, + }) + assert.deepEqual(capture, { + executionId: execution.executionId, + termination: { kind: 'exit', exitCode: 0 }, + }) + assert.deepEqual( + protectedCaptureFromPierResult(request(execution), { + exception_info: { exception_type: 'AgentTimeoutError' }, + agent_result: { metadata: { ...metadata, termination: { kind: 'cancelled' } } }, + }), + { + executionId: execution.executionId, + termination: { kind: 'timeout', timeoutMs: 60_000 }, + }, + ) +}) + +test('rejects substituted canonical bytes before writing Pier artifacts', async () => { + const root = await mkdtemp(join(tmpdir(), 'pier-stage-tamper-')) + try { + const execution = prepared(root) + const tampered = { + ...execution, + executionPlan: { ...execution.executionPlan, bytes: Buffer.from('{}') }, + } as PreparedAgentCandidateExecution + await assert.rejects( + stagePreparedPierCandidateExecution( + { + prepared: tampered, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + request(tampered), + ), + /do not match/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('rejects an executor request that changes signed public environment', async () => { + const root = await mkdtemp(join(tmpdir(), 'pier-stage-env-')) + try { + const execution = prepared(root) + const changed = request(execution) + ;(changed.launch as { env: Record }).env.PUBLIC_MODE = 'changed' + await assert.rejects( + stagePreparedPierCandidateExecution( + { + prepared: execution, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + changed, + ), + /changed signed public environment/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('rejects changed or escaping executor files before launch', async () => { + for (const attack of ['bytes', 'path'] as const) { + const root = await mkdtemp(join(tmpdir(), `pier-stage-${attack}-`)) + try { + const execution = prepared(root) + const changed = request(execution) + const file = changed.inputs.task.files[0] as { path: string; bytes: Uint8Array } + if (attack === 'bytes') file.bytes = Buffer.from('changed\n') + else file.path = '../escape' + await assert.rejects( + stagePreparedPierCandidateExecution( + { + prepared: execution, + directory: join(root, 'sealed'), + pierVersion: '0.3.0', + }, + changed, + ), + attack === 'bytes' ? /differs from signed evidence/ : /canonical relative POSIX path/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } + } +}) + +test('waits for process and container death acknowledgement after abort', async () => { + const controller = new AbortController() + let terminated = false + const waiting = awaitAbortableTrial( + { + result: new Promise(() => undefined), + terminateAndWait: async () => { + await new Promise((resolve) => setTimeout(resolve, 5)) + terminated = true + return { processExited: true, containersRemoved: true } + }, + }, + controller.signal, + ) + controller.abort(new Error('signed deadline elapsed')) + await assert.rejects(waiting, /signed deadline elapsed/) + assert.equal(terminated, true) +}) + +test('fails closed when termination does not acknowledge container removal', async () => { + const controller = new AbortController() + const waiting = awaitAbortableTrial( + { + result: new Promise(() => undefined), + terminateAndWait: async () => ({ processExited: true, containersRemoved: false }) as never, + }, + controller.signal, + ) + controller.abort() + await assert.rejects(waiting, /did not acknowledge/) +}) diff --git a/bench/src/pier-agent.ts b/bench/src/pier-agent.ts new file mode 100644 index 00000000..367981f3 --- /dev/null +++ b/bench/src/pier-agent.ts @@ -0,0 +1,579 @@ +/** Pier transport for a runtime-prepared immutable candidate execution. */ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { chmod, lstat, mkdir, realpath, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute, join, resolve } from 'node:path' + +import type { + AgentCandidateBenchmarkGraderPort, + AgentCandidateExecutionClaimStore, + AgentCandidateExecutorRequest, + AgentCandidateExecutorPort, + AgentCandidateOutputArtifactPort, + AgentCandidateProtectedRunCapture, + AgentCandidateRunFinalization, + PreparedAgentCandidateExecution, +} from '@tangle-network/agent-runtime' +import { executePreparedAgentCandidate } from '@tangle-network/agent-runtime' +import type { TraceStore } from '@tangle-network/agent-eval' + +import { capturePierTaskOutcome } from './pier-task-outcome' + +const adapterImportPath = 'pier_agents.tangle_candidate:TangleCandidateAgent' +const sha256Pattern = /^sha256:[a-f0-9]{64}$/ + +interface StagePreparedPierCandidateOptions { + readonly prepared: PreparedAgentCandidateExecution + /** Evaluator-owned directory persisted with the Pier trial. */ + readonly directory: string + /** Exact Pier package version pinned by the experiment contract. */ + readonly pierVersion: string +} + +export interface StagedPierCandidateExecution { + readonly executionId: string + readonly directory: string + readonly taskDirectory: string + readonly candidateDirectory?: string + readonly profileDirectory: string + readonly planPath: string + readonly receiptPath: string + readonly agentArgs: readonly string[] + /** Executor-only model and trace bindings; never present on the prepared object or disk. */ + readonly evaluatorEnv: Readonly> + /** One prepared execution is exactly one Pier trial attempt. */ + readonly attemptArgs: readonly ['--n-attempts', '1', '--max-retries', '0'] +} + +export interface PierCandidateTerminationAcknowledgement { + /** The Pier process has exited and has been reaped. */ + readonly processExited: true + /** Every task container created for this one trial has been removed. */ + readonly containersRemoved: true +} + +export interface PierCandidateTrialHandle { + /** Resolves only after the Pier process exits and its task container is gone. */ + readonly result: Promise + /** Idempotently kill/reap Pier and remove its task container, then acknowledge their death. */ + readonly terminateAndWait: () => Promise +} + +/** Evaluator-owned bytes captured from one completed official Pier trial. */ +export interface PierCandidateTrialResult { + /** Parsed value of the exact `result.json` bytes. */ + readonly value: unknown + /** Exact official `result.json` bytes used as grader evidence. */ + readonly resultBytes: Uint8Array + /** Exact `/logs/artifacts/model.patch` bytes emitted before official tests. */ + readonly taskPatch: Uint8Array +} + +export interface PierCandidateOfficialResult { + readonly value: unknown + readonly bytes: Uint8Array +} + +type RuntimeGraderInput = Parameters[0] +type RuntimeGraderResult = Awaited> + +/** Executes the exact admitted grader bytes against the official Pier result. */ +export interface PierCandidateGraderPort { + readonly name: string + readonly version: string + readonly artifact: AgentCandidateBenchmarkGraderPort['artifact'] + run(input: RuntimeGraderInput & { + readonly officialResult: PierCandidateOfficialResult + }): Promise +} + +export interface ExecutePreparedPierCandidateOptions extends StagePreparedPierCandidateOptions { + readonly traceStore: TraceStore + /** Durable one-shot store shared by every process capable of running this benchmark. */ + readonly claimStore: AgentCandidateExecutionClaimStore + readonly outputArtifacts: AgentCandidateOutputArtifactPort + readonly grader: PierCandidateGraderPort + /** + * Start exactly one Pier trial synchronously so an abort cannot race an unowned process. + * The returned handle owns process/container termination for the whole trial. + */ + readonly start: ( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ) => PierCandidateTrialHandle +} + +interface PierResultLike { + exception_info?: unknown + agent_result?: unknown +} + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +function nonEmpty(value: string, label: string): string { + if (!value || value.includes('\0')) throw new Error(`${label} must be non-empty without NUL`) + return value +} + +function absoluteHostPath(value: string, label: string): string { + if (!isAbsolute(value)) throw new Error(`${label} must be an absolute host path`) + return resolve(value) +} + +function digest(value: string, label: string): string { + if (!sha256Pattern.test(value)) throw new Error(`${label} is not a SHA-256 digest`) + return value +} + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + return value as Record +} + +function parseExactJson(bytes: Uint8Array, label: string): unknown { + try { + return JSON.parse(Buffer.from(bytes).toString('utf8')) + } catch (error) { + throw new Error(`${label} bytes are not UTF-8 JSON`, { cause: error }) + } +} + +function safeRelativePath(value: string, label: string): string { + const parts = value.split('/') + if ( + !value || + value.includes('\0') || + value.includes('\\') || + isAbsolute(value) || + parts.some((part) => !part || part === '.' || part === '..') + ) { + throw new Error(`${label} must be a canonical relative POSIX path`) + } + return value +} + +async function assertRealDirectory(path: string, label: string): Promise { + const stats = await lstat(path) + if (!stats.isDirectory() || stats.isSymbolicLink()) { + throw new Error(`${label} must be a real directory`) + } + if ((await realpath(path)) !== path) { + throw new Error(`${label} has a symlinked path component`) + } +} + +interface ExpectedExecutorFile { + readonly path: string + readonly mode: number + readonly sha256: string + readonly byteLength?: number +} + +async function materializeExecutorFiles( + root: string, + files: AgentCandidateExecutorRequest['inputs']['profile']['files'], + expected: readonly ExpectedExecutorFile[], + label: string, +): Promise { + const expectedByPath = new Map(expected.map((file) => [file.path, file])) + if (expectedByPath.size !== expected.length || files.length !== expected.length) { + throw new Error(`${label} file set differs from signed evidence`) + } + + await mkdir(root, { mode: 0o700 }) + await assertRealDirectory(root, label) + for (const file of files) { + const relative = safeRelativePath(file.path, `${label} file path`) + const identity = expectedByPath.get(relative) + if (!identity) throw new Error(`${label} contains unsigned file ${relative}`) + if ( + file.mode !== identity.mode || + sha256(file.bytes) !== identity.sha256 || + (identity.byteLength !== undefined && file.bytes.byteLength !== identity.byteLength) + ) { + throw new Error(`${label} file ${relative} differs from signed evidence`) + } + + const target = join(root, relative) + const parent = dirname(target) + if (parent !== root) await mkdir(parent, { recursive: true, mode: 0o700 }) + await writeFile(target, file.bytes, { flag: 'wx', mode: 0o600 }) + await chmod(target, file.mode) + const stats = await lstat(target) + if ( + !stats.isFile() || + stats.isSymbolicLink() || + stats.nlink !== 1 || + (stats.mode & 0o777) !== file.mode + ) { + throw new Error(`${label} file ${relative} is not a singly-linked regular file`) + } + } +} + +function protectedEnvironment( + prepared: PreparedAgentCandidateExecution, + request: AgentCandidateExecutorRequest, +): Readonly> { + const protectedEntries: Array = [] + for (const [name, value] of Object.entries(request.launch.env)) { + const publicValue = prepared.launch.env[name] + if (publicValue === undefined) { + protectedEntries.push([name, value]) + } else if (publicValue !== value) { + throw new Error(`executor request changed signed public environment ${name}`) + } + } + for (const name of Object.keys(prepared.launch.env)) { + if (!(name in request.launch.env)) { + throw new Error(`executor request omitted signed public environment ${name}`) + } + } + return Object.freeze(Object.fromEntries(protectedEntries)) +} + +/** + * Persist the runtime's exact canonical bytes and return only Pier transport + * arguments. No benchmark-specific plan or candidate-authored telemetry exists. + */ +/** @internal Package-private transport seam exercised by adapter tests. */ +export async function stagePreparedPierCandidateExecution( + options: StagePreparedPierCandidateOptions, + request: AgentCandidateExecutorRequest, +): Promise { + const { prepared } = options + nonEmpty(options.pierVersion, 'pierVersion') + const directory = absoluteHostPath(options.directory, 'directory') + await mkdir(directory, { mode: 0o700 }) + await assertRealDirectory(directory, 'directory') + if (request.memory.mode !== 'disabled') { + throw new Error('Pier transport does not yet implement protected isolated memory') + } + if (request.knowledge !== undefined) { + throw new Error('Pier transport does not yet implement immutable knowledge mounts') + } + const taskDirectory = join(directory, 'task') + const candidateDirectory = request.inputs.candidate ? join(directory, 'candidate') : undefined + const profileDirectory = join(directory, 'profile') + await materializeExecutorFiles( + taskDirectory, + request.inputs.task.files, + request.inputs.task.snapshot.material.files, + 'task executor input', + ) + if (request.inputs.candidate && candidateDirectory) { + await materializeExecutorFiles( + candidateDirectory, + request.inputs.candidate.files, + request.inputs.candidate.snapshot.material.files, + 'candidate executor input', + ) + } + await materializeExecutorFiles( + profileDirectory, + request.inputs.profile.files, + request.profilePlan.value.material.files.map((file) => ({ + path: file.relPath, + mode: file.mode, + sha256: file.contentSha256, + })), + 'profile executor input', + ) + + const planDigest = digest(request.executionPlan.value.digest, 'execution plan digest') + if (sha256(request.executionPlan.bytes) !== planDigest) { + throw new Error('prepared execution-plan bytes do not match their runtime digest') + } + assert.deepEqual( + parseExactJson(request.executionPlan.bytes, 'execution plan'), + request.executionPlan.value.material, + 'prepared execution-plan bytes differ from runtime material', + ) + + const receiptDigest = digest( + request.materializationReceipt.digest, + 'materialization receipt digest', + ) + if (sha256(request.materializationReceipt.bytes) !== receiptDigest) { + throw new Error('prepared materialization-receipt bytes do not match their runtime digest') + } + const { digest: _receiptDigest, ...receiptMaterial } = request.materializationReceipt.value + assert.deepEqual( + parseExactJson(request.materializationReceipt.bytes, 'materialization receipt'), + receiptMaterial, + 'prepared receipt bytes differ from runtime material', + ) + + const planPath = join(directory, 'execution-plan.json') + const receiptPath = join(directory, 'materialization-receipt.json') + await Promise.all([ + writeFile(planPath, request.executionPlan.bytes, { mode: 0o600, flag: 'wx' }), + writeFile(receiptPath, request.materializationReceipt.bytes, { mode: 0o600, flag: 'wx' }), + ]) + + const agentArgs = [ + '--agent-import-path', + adapterImportPath, + '--model', + nonEmpty(request.resolvedModel.requested, 'resolved requested model'), + '--agent-kwarg', + `plan_path=${planPath}`, + '--agent-kwarg', + `receipt_path=${receiptPath}`, + '--agent-kwarg', + `expected_receipt_digest=${receiptDigest}`, + '--agent-kwarg', + `trace_run_id=${nonEmpty(request.trace.runId, 'trace run id')}`, + '--agent-kwarg', + `task_dir=${taskDirectory}`, + '--agent-kwarg', + `profile_dir=${profileDirectory}`, + '--agent-kwarg', + `pier_version=${options.pierVersion}`, + ] + if (candidateDirectory !== undefined) { + agentArgs.push('--agent-kwarg', `candidate_dir=${candidateDirectory}`) + } + + return { + executionId: request.executionId, + directory, + taskDirectory, + ...(candidateDirectory ? { candidateDirectory } : {}), + profileDirectory, + planPath, + receiptPath, + agentArgs, + evaluatorEnv: protectedEnvironment(prepared, request), + attemptArgs: ['--n-attempts', '1', '--max-retries', '0'], + } +} + +function exceptionType(value: unknown): string | undefined { + const info = value && typeof value === 'object' ? (value as Record) : undefined + for (const key of ['exception_type', 'type', 'name']) { + if (typeof info?.[key] === 'string') return info[key] + } + return undefined +} + +/** Read only identity and termination from Pier; usage always comes from agent-eval. */ +/** @internal Package-private result parser exercised by adapter tests. */ +export function protectedCaptureFromPierResult( + request: AgentCandidateExecutorRequest, + value: unknown, +): AgentCandidateProtectedRunCapture { + const result = record(value, 'Pier trial result') as PierResultLike + const agentResult = record(result.agent_result, 'Pier agent_result') + const metadata = record(agentResult.metadata, 'Pier agent_result.metadata') + const expected = { + executionId: request.executionId, + bundleDigest: request.executionPlan.value.material.bundleDigest, + executionPlanDigest: request.executionPlan.value.digest, + materializationReceiptDigest: request.materializationReceipt.digest, + } + for (const [name, identity] of Object.entries(expected)) { + if (metadata[name] !== identity) { + throw new Error(`Pier result ${name} does not match the prepared execution`) + } + } + + const reported = record(metadata.termination, 'Pier termination') + const type = exceptionType(result.exception_info) + if (type?.includes('AgentTimeout')) { + return { + executionId: request.executionId, + termination: { + kind: 'timeout', + timeoutMs: request.hardLimits.timeoutMs, + }, + } + } + if (type?.includes('Cancelled') || reported.kind === 'cancelled') { + return { executionId: request.executionId, termination: { kind: 'cancelled' } } + } + if (reported.kind === 'exit' && Number.isInteger(reported.exitCode)) { + return { + executionId: request.executionId, + termination: { kind: 'exit', exitCode: reported.exitCode as number }, + } + } + throw new Error(`Pier result has no recognized truthful termination${type ? ` (${type})` : ''}`) +} + +function sealPierTrialResult(value: PierCandidateTrialResult): PierCandidateTrialResult { + if (!(value.resultBytes instanceof Uint8Array) || !(value.taskPatch instanceof Uint8Array)) { + throw new Error('Pier trial result must contain raw result and patch bytes') + } + const parsed = parseExactJson(value.resultBytes, 'Pier result.json') + assert.deepEqual(parsed, value.value, 'Pier parsed result differs from result.json bytes') + const resultBytes = Uint8Array.from(value.resultBytes) + const taskPatch = Uint8Array.from(value.taskPatch) + return Object.freeze({ + value: parsed, + get resultBytes(): Uint8Array { + return Uint8Array.from(resultBytes) + }, + get taskPatch(): Uint8Array { + return Uint8Array.from(taskPatch) + }, + }) +} + +function abortReason(signal: AbortSignal): Error { + const reason = signal.reason + if (reason instanceof Error) return reason + return new Error(`Pier candidate execution aborted${reason ? `: ${String(reason)}` : ''}`) +} + +function assertTerminationAcknowledged( + acknowledgement: PierCandidateTerminationAcknowledgement, +): void { + if (acknowledgement.processExited !== true || acknowledgement.containersRemoved !== true) { + throw new Error('Pier termination did not acknowledge process and container death') + } +} + +/** @internal Package-private cancellation seam exercised by adapter tests. */ +export async function awaitAbortableTrial( + trial: Omit & { readonly result: Promise }, + signal: AbortSignal, +): Promise { + if (signal.aborted) { + assertTerminationAcknowledged(await trial.terminateAndWait()) + throw abortReason(signal) + } + + return await new Promise((resolveResult, reject) => { + let settled = false + let aborting = false + const settle = (callback: () => void): void => { + if (settled) return + settled = true + signal.removeEventListener('abort', onAbort) + callback() + } + const onAbort = (): void => { + aborting = true + void trial.terminateAndWait().then( + (acknowledgement) => { + try { + assertTerminationAcknowledged(acknowledgement) + settle(() => reject(abortReason(signal))) + } catch (error) { + settle(() => reject(error)) + } + }, + (error) => settle(() => reject(new Error('Pier termination failed', { cause: error }))), + ) + } + signal.addEventListener('abort', onAbort, { once: true }) + trial.result.then( + (result) => { + if (!aborting && !signal.aborted) settle(() => resolveResult(result)) + }, + (error) => { + if (!aborting && !signal.aborted) settle(() => reject(error)) + }, + ) + }) +} + +/** Execute and finalize through the runtime's only gradable candidate path. */ +export async function executePreparedPierCandidate( + options: ExecutePreparedPierCandidateOptions, +): Promise { + const trials = new Map< + string, + { readonly handle: PierCandidateTrialHandle; result?: PierCandidateTrialResult } + >() + const officialResults = new Map() + const trialIdentity = (executionId: string, executionPlanDigest: string): string => + JSON.stringify([executionId, executionPlanDigest]) + const executor: AgentCandidateExecutorPort = { + execute: async (request, context) => { + context.signal.throwIfAborted() + const staged = await stagePreparedPierCandidateExecution(options, request) + context.signal.throwIfAborted() + const identity = trialIdentity(request.executionId, request.executionPlan.value.digest) + if (trials.has(identity)) throw new Error('Pier trial identity is already active') + const trial = options.start(staged, { + request, + traceStore: context.traceStore, + signal: context.signal, + deadlineAtMs: context.deadlineAtMs, + }) + const active = { handle: trial } as { + readonly handle: PierCandidateTrialHandle + result?: PierCandidateTrialResult + } + trials.set(identity, active) + const result = sealPierTrialResult(await awaitAbortableTrial(trial, context.signal)) + active.result = result + officialResults.set(request.executionId, result) + return protectedCaptureFromPierResult(request, result.value) + }, + stopAndCapture: async (request) => { + const identity = trialIdentity(request.executionId, request.executionPlanDigest) + const active = trials.get(identity) + if (!active) return { stopped: true } + assertTerminationAcknowledged(await active.handle.terminateAndWait()) + let result = active.result + if (!result) { + try { + result = sealPierTrialResult(await active.handle.result) + } catch { + result = undefined + } + } + trials.delete(identity) + if (!result) return { stopped: true } + officialResults.set(request.executionId, result) + const repository = options.prepared.executionPlan.value.material.task.repository + const taskOutcome = await capturePierTaskOutcome({ + repositoryRoot: options.prepared.roots.staging.taskRoot, + baseCommit: repository.baseCommit, + baseTree: repository.baseTree, + patch: result.taskPatch, + }) + return { stopped: true, taskOutcome } + }, + } + const grader: AgentCandidateBenchmarkGraderPort = { + name: options.grader.name, + version: options.grader.version, + artifact: options.grader.artifact, + run: async (input) => { + const official = officialResults.get(input.executionId) + if (!official) throw new Error('Pier official result is missing for executable grading') + return await options.grader.run({ + ...input, + officialResult: { + value: parseExactJson(official.resultBytes, 'Pier result.json'), + bytes: Uint8Array.from(official.resultBytes), + }, + }) + }, + } + try { + return await executePreparedAgentCandidate(options.prepared, { + executor, + grader, + outputArtifacts: options.outputArtifacts, + traceStore: options.traceStore, + claimStore: options.claimStore, + }) + } finally { + officialResults.clear() + trials.clear() + } +} diff --git a/bench/src/pier-result-grader.mjs b/bench/src/pier-result-grader.mjs new file mode 100644 index 00000000..388cab59 --- /dev/null +++ b/bench/src/pier-result-grader.mjs @@ -0,0 +1,30 @@ +/** Executable parser for official Pier `result.json` evidence. Reads JSON on stdin. */ +const chunks = [] +for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk)) +const result = JSON.parse(Buffer.concat(chunks).toString('utf8')) +const rewards = result?.verifier_result?.rewards +if (!rewards || typeof rewards !== 'object' || Array.isArray(rewards)) { + throw new Error('official Pier result omitted verifier_result.rewards') +} +const dimensions = {} +for (const name of Object.keys(rewards).sort()) { + const value = rewards[name] + if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/.test(name)) { + throw new Error(`official Pier reward name is not normalized: ${name}`) + } + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`official Pier reward ${name} is outside [0, 1]`) + } + dimensions[name] = value +} +if (typeof dimensions.reward !== 'number') { + throw new Error('official Pier result omitted the reward dimension') +} +process.stdout.write( + JSON.stringify({ + score: dimensions.reward, + passed: dimensions.reward === 1, + dimensions, + raw: {}, + }), +) diff --git a/bench/src/pier-result-grader.test.mts b/bench/src/pier-result-grader.test.mts new file mode 100644 index 00000000..cd40afd4 --- /dev/null +++ b/bench/src/pier-result-grader.test.mts @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import test from 'node:test' + +import type { AgentCandidateArtifactRef } from '@tangle-network/agent-interface' + +import { createPierResultGrader } from './pier-result-grader' + +const digest = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +const artifact: AgentCandidateArtifactRef = { + locator: { kind: 's3', bucket: 'pier-graders', key: 'pier-result-grader.mjs' }, + sha256: `sha256:${'a'.repeat(64)}`, + byteLength: 1, +} + +test('executes admitted Pier grader bytes over exact official result evidence', async () => { + const implementation = await readFile(new URL('./pier-result-grader.mjs', import.meta.url)) + const resultBytes = Buffer.from( + JSON.stringify({ verifier_result: { rewards: { reward: 1, patch_applied: 1 } } }), + ) + const grader = createPierResultGrader({ name: 'pier-result', version: '1', artifact }) + const output = await grader.run({ + executionId: 'execution-1', + termination: { kind: 'exit', exitCode: 0 }, + outcome: { evidence: { digest: `sha256:${'b'.repeat(64)}` } } as never, + implementation: { byteLength: implementation.byteLength, bytes: implementation }, + signal: new AbortController().signal, + officialResult: { value: JSON.parse(resultBytes.toString('utf8')), bytes: resultBytes }, + }) + + assert.deepEqual(output.evaluation, { + score: 1, + passed: true, + dimensions: { patch_applied: 1, reward: 1 }, + raw: {}, + }) + assert.deepEqual(Buffer.from(output.evidence), resultBytes) + assert.equal(output.binding.implementationDigest, digest(implementation)) + assert.equal(output.binding.outputDigest, digest(resultBytes)) +}) + +test('fails closed for missing official rewards and pre-aborted execution', async () => { + const implementation = await readFile(new URL('./pier-result-grader.mjs', import.meta.url)) + const grader = createPierResultGrader({ name: 'pier-result', version: '1', artifact }) + const base = { + executionId: 'execution-1', + termination: { kind: 'exit', exitCode: 0 } as const, + outcome: { evidence: { digest: `sha256:${'b'.repeat(64)}` } } as never, + implementation: { byteLength: implementation.byteLength, bytes: implementation }, + officialResult: { value: {}, bytes: Buffer.from('{}') }, + } + await assert.rejects( + grader.run({ ...base, signal: new AbortController().signal }), + /omitted verifier_result.rewards/, + ) + const controller = new AbortController() + controller.abort(new Error('deadline')) + await assert.rejects(grader.run({ ...base, signal: controller.signal }), /deadline/) +}) diff --git a/bench/src/pier-result-grader.ts b/bench/src/pier-result-grader.ts new file mode 100644 index 00000000..2ce52154 --- /dev/null +++ b/bench/src/pier-result-grader.ts @@ -0,0 +1,108 @@ +/** Execute the exact admitted Pier result parser bytes in a fresh Node process. */ +import { spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { BenchmarkEvaluation } from '@tangle-network/agent-eval' + +import type { PierCandidateGraderPort } from './pier-agent' + +const sha256 = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +export function createPierResultGrader( + descriptor: Pick, +): PierCandidateGraderPort { + return { + ...descriptor, + run: async (input) => { + input.signal.throwIfAborted() + const implementation = Uint8Array.from(input.implementation.bytes) + if (implementation.byteLength !== input.implementation.byteLength) { + throw new Error('Pier grader implementation length changed before execution') + } + const evidence = Uint8Array.from(input.officialResult.bytes) + const evaluation = await executeGrader(implementation, evidence, input.signal) + input.signal.throwIfAborted() + return { + evaluation, + evidence, + binding: { + implementationDigest: sha256(implementation), + taskOutcomeDigest: input.outcome.evidence.digest, + outputDigest: sha256(evidence), + }, + } + }, + } +} + +async function executeGrader( + implementation: Uint8Array, + resultBytes: Uint8Array, + signal: AbortSignal, +): Promise { + const directory = await mkdtemp(join(tmpdir(), 'pier-result-grader-')) + const script = join(directory, 'grader.mjs') + try { + await writeFile(script, implementation, { flag: 'wx', mode: 0o600 }) + await chmod(script, 0o500) + const stdout = await runNode(script, resultBytes, signal) + const parsed = JSON.parse(stdout.toString('utf8')) as BenchmarkEvaluation + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Pier result grader returned a non-object evaluation') + } + return parsed + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function runNode( + script: string, + input: Uint8Array, + signal: AbortSignal, +): Promise { + signal.throwIfAborted() + return await new Promise((resolveResult, reject) => { + const child = spawn(process.execPath, ['--no-warnings', script], { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + LANG: 'C', + LC_ALL: 'C', + }, + stdio: ['pipe', 'pipe', 'pipe'], + signal, + }) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + let bytes = 0 + const collect = (target: Buffer[]) => (chunk: Buffer) => { + bytes += chunk.byteLength + if (bytes > 1024 * 1024) { + child.kill('SIGKILL') + reject(new Error('Pier result grader exceeded its output limit')) + return + } + target.push(Buffer.from(chunk)) + } + child.stdout.on('data', collect(stdout)) + child.stderr.on('data', collect(stderr)) + child.once('error', reject) + child.once('close', (code, childSignal) => { + if (code !== 0) { + reject( + new Error( + `Pier result grader failed (${childSignal ?? code}): ${Buffer.concat(stderr).toString('utf8').trim()}`, + ), + ) + return + } + resolveResult(Buffer.concat(stdout)) + }) + child.stdin.end(input) + }) +} diff --git a/bench/src/pier-task-outcome.test.mts b/bench/src/pier-task-outcome.test.mts new file mode 100644 index 00000000..55980ac7 --- /dev/null +++ b/bench/src/pier-task-outcome.test.mts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { capturePierTaskOutcome } from './pier-task-outcome' +import { materializePierWorkspaceArchive } from './pier-workspace-archive' + +test('Pier task outcome reconstructs the exact tree and deterministic archive from a patch', async () => { + const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) + try { + mkdirSync(join(root, 'src')) + writeFileSync(join(root, 'src/status.txt'), 'not-ready\n') + chmodSync(join(root, 'src/status.txt'), 0o644) + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'fixture@example.com']) + git(root, ['config', 'user.name', 'Fixture']) + git(root, ['add', '-A']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + writeFileSync(join(root, 'src/status.txt'), 'ready\n') + const patch = execFileSync('git', ['-C', root, 'diff', '--binary']) + git(root, ['restore', '.']) + + const captured = await capturePierTaskOutcome({ repositoryRoot: root, baseCommit, baseTree, patch }) + assert.notEqual(captured.resultTree, baseTree) + assert.deepEqual(Buffer.from(captured.gitDiff), patch) + const destination = join(root, 'materialized') + await materializePierWorkspaceArchive({ + archive: captured.archive, + expected: captured.afterState, + destination, + }) + assert.equal(readFileSync(join(destination, 'src/status.txt'), 'utf8'), 'ready\n') + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +test('Pier task outcome preserves the signed base for an empty patch', async () => { + const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) + try { + writeFileSync(join(root, 'README.md'), 'base\n') + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'fixture@example.com']) + git(root, ['config', 'user.name', 'Fixture']) + git(root, ['add', '-A']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + + const captured = await capturePierTaskOutcome({ + repositoryRoot: root, + baseCommit, + baseTree, + patch: Buffer.alloc(0), + }) + assert.equal(captured.resultTree, baseTree) + assert.equal(captured.gitDiff.byteLength, 0) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +function git(root: string, args: string[]): string { + return execFileSync('git', ['-C', root, ...args], { + encoding: 'utf8', + env: { + ...process.env, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_NO_REPLACE_OBJECTS: '1', + LC_ALL: 'C', + }, + }).trim() +} diff --git a/bench/src/pier-task-outcome.ts b/bench/src/pier-task-outcome.ts new file mode 100644 index 00000000..ced6c50c --- /dev/null +++ b/bench/src/pier-task-outcome.ts @@ -0,0 +1,234 @@ +/** Reconstruct a runtime task outcome from Pier's evaluator-captured binary patch. */ +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import type { AgentCandidateExecutorTaskOutcomeCapture } from '@tangle-network/agent-runtime' + +import { createPierWorkspaceArchive } from './pier-workspace-archive' + +interface GitResult { + readonly stdout: Buffer + readonly stderr: Buffer +} + +/** Apply the official patch in an isolated Git object directory and capture its exact tree. */ +export async function capturePierTaskOutcome(input: { + readonly repositoryRoot: string + readonly baseCommit: string + readonly baseTree: string + readonly patch: Uint8Array + readonly signal?: AbortSignal +}): Promise { + input.signal?.throwIfAborted() + const repositoryRoot = resolve(input.repositoryRoot) + const head = (await git(repositoryRoot, ['rev-parse', 'HEAD'], undefined, {}, input.signal)).stdout + .toString('utf8') + .trim() + const headTree = ( + await git(repositoryRoot, ['rev-parse', 'HEAD^{tree}'], undefined, {}, input.signal) + ).stdout + .toString('utf8') + .trim() + if (head !== input.baseCommit || headTree !== input.baseTree) { + throw new Error('Pier task outcome repository drifted from its signed base') + } + const gitDir = resolve( + ( + await git(repositoryRoot, ['rev-parse', '--absolute-git-dir'], undefined, {}, input.signal) + ).stdout + .toString('utf8') + .trim(), + ) + if ((await realpath(gitDir)) !== gitDir || gitDir.includes(':')) { + throw new Error('Pier task outcome Git directory is unsupported') + } + const replacements = ( + await git( + repositoryRoot, + ['for-each-ref', '--format=%(refname)', 'refs/replace'], + undefined, + {}, + input.signal, + ) + ).stdout + .toString('utf8') + .trim() + if (replacements) throw new Error('Pier task outcome repository contains Git replace refs') + + const temporary = await mkdtemp(join(tmpdir(), 'pier-task-outcome-')) + try { + const objectDirectory = join(temporary, 'objects') + const indexFile = join(temporary, 'index') + await mkdir(objectDirectory) + const environment = { + GIT_INDEX_FILE: indexFile, + GIT_OBJECT_DIRECTORY: objectDirectory, + GIT_ALTERNATE_OBJECT_DIRECTORIES: join(gitDir, 'objects'), + } + await git(repositoryRoot, ['read-tree', input.baseTree], undefined, environment, input.signal) + const patch = Uint8Array.from(input.patch) + if (patch.byteLength > 0) { + await git( + repositoryRoot, + ['apply', '--cached', '--binary', '--whitespace=nowarn', '-'], + patch, + environment, + input.signal, + ) + } + const resultTree = ( + await git(repositoryRoot, ['write-tree'], undefined, environment, input.signal) + ).stdout + .toString('utf8') + .trim() + const entries = parseTreeEntries( + ( + await git( + repositoryRoot, + ['ls-tree', '-rz', '--full-tree', resultTree], + undefined, + environment, + input.signal, + ) + ).stdout, + ) + if (entries.length === 0) throw new Error('Pier task outcome tree cannot be empty') + const files = await Promise.all( + entries.map(async (entry) => { + if (entry.type !== 'blob' || (entry.mode !== '100644' && entry.mode !== '100755')) { + throw new Error(`Pier task outcome contains a non-regular entry: ${entry.path}`) + } + return { + path: safeGitPath(entry.path), + mode: entry.mode === '100755' ? (0o755 as const) : (0o644 as const), + bytes: ( + await git( + repositoryRoot, + ['cat-file', 'blob', entry.object], + undefined, + environment, + input.signal, + ) + ).stdout, + } + }), + ) + const captured = createPierWorkspaceArchive(files) + return { + resultTree, + afterState: captured.manifest, + archive: captured.archive, + gitDiff: patch, + } + } finally { + await rm(temporary, { recursive: true, force: true }) + } +} + +function parseTreeEntries(bytes: Uint8Array): Array<{ + readonly mode: string + readonly type: string + readonly object: string + readonly path: string +}> { + const raw = Buffer.from(bytes) + const decoded = raw.toString('utf8') + if (!Buffer.from(decoded, 'utf8').equals(raw)) { + throw new Error('Pier task outcome contains a non-UTF-8 path') + } + return decoded + .split('\0') + .filter(Boolean) + .map((row) => { + const tab = row.indexOf('\t') + const [mode, type, object] = row.slice(0, tab).split(' ') + const path = row.slice(tab + 1) + if (tab < 1 || !mode || !type || !object || !path) { + throw new Error('Pier task outcome contains a malformed Git entry') + } + return { mode, type, object, path } + }) +} + +function safeGitPath(path: string): string { + const parts = path.split('/') + if ( + !path || + path.startsWith('/') || + path.includes('\\') || + path.includes('\0') || + parts.some((part) => !part || part === '.' || part === '..') || + parts[0]?.toLowerCase() === '.git' + ) { + throw new Error(`Pier task outcome contains an unsafe Git path: ${path}`) + } + return path +} + +async function git( + repositoryRoot: string, + args: readonly string[], + input: Uint8Array | undefined, + extraEnvironment: Readonly>, + signal: AbortSignal | undefined, +): Promise { + signal?.throwIfAborted() + const environment = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')), + ) as Record + Object.assign(environment, { + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_NO_REPLACE_OBJECTS: '1', + LC_ALL: 'C', + ...extraEnvironment, + }) + return await new Promise((resolveResult, reject) => { + const child = spawn( + 'git', + [ + '-c', + 'core.hooksPath=/dev/null', + '-c', + 'protocol.file.allow=never', + '-C', + repositoryRoot, + ...args, + ], + { env: environment, stdio: ['pipe', 'pipe', 'pipe'], signal }, + ) + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + let outputBytes = 0 + const collect = (target: Buffer[]) => (chunk: Buffer) => { + outputBytes += chunk.byteLength + if (outputBytes > 64 * 1024 * 1024) { + child.kill('SIGKILL') + reject(new Error(`git ${args[0] ?? ''} exceeded the output limit`)) + return + } + target.push(Buffer.from(chunk)) + } + child.stdout.on('data', collect(stdout)) + child.stderr.on('data', collect(stderr)) + child.once('error', reject) + child.once('close', (code, childSignal) => { + const out = Buffer.concat(stdout) + const err = Buffer.concat(stderr) + if (code !== 0) { + reject( + new Error( + `git ${args[0] ?? ''} failed (${childSignal ?? code}): ${err.toString('utf8').trim()}`, + ), + ) + return + } + resolveResult({ stdout: out, stderr: err }) + }) + child.stdin.end(input) + }) +} diff --git a/bench/src/pier-workspace-archive.test.mts b/bench/src/pier-workspace-archive.test.mts new file mode 100644 index 00000000..7fdb694f --- /dev/null +++ b/bench/src/pier-workspace-archive.test.mts @@ -0,0 +1,58 @@ +import assert from 'node:assert/strict' +import { chmod, lstat, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' + +import { + createPierWorkspaceArchive, + materializePierWorkspaceArchive, +} from './pier-workspace-archive' + +test('Pier workspace archive round-trips exact bytes, paths, and modes', async () => { + const encoded = createPierWorkspaceArchive([ + { path: 'src/run.sh', mode: 0o755, bytes: Buffer.from('#!/bin/sh\n') }, + { path: 'README.md', mode: 0o644, bytes: Buffer.from('fixture\n') }, + ]) + const root = await mkdtemp(join(tmpdir(), 'pier-archive-test-')) + const destination = join(root, 'workspace') + try { + await materializePierWorkspaceArchive({ ...encoded, expected: encoded.manifest, destination }) + assert.equal(await readFile(join(destination, 'README.md'), 'utf8'), 'fixture\n') + assert.equal((await lstat(join(destination, 'src/run.sh'))).mode & 0o777, 0o755) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('Pier workspace archive rejects unsafe, noncanonical, and nonempty materialization', async () => { + assert.throws( + () => createPierWorkspaceArchive([{ path: '../escape', mode: 0o644, bytes: Buffer.alloc(0) }]), + /unsafe/, + ) + const encoded = createPierWorkspaceArchive([ + { path: 'source.ts', mode: 0o644, bytes: Buffer.from('export {}\n') }, + ]) + const root = await mkdtemp(join(tmpdir(), 'pier-archive-test-')) + const destination = join(root, 'workspace') + try { + await writeFile(join(root, 'tampered.json'), Buffer.from(encoded.archive)) + const noncanonical = Buffer.from(`${Buffer.from(encoded.archive).toString('utf8')}\n`) + await assert.rejects( + materializePierWorkspaceArchive({ + archive: noncanonical, + expected: encoded.manifest, + destination, + }), + /not canonical/, + ) + await materializePierWorkspaceArchive({ ...encoded, expected: encoded.manifest, destination }) + await chmod(join(destination, 'source.ts'), 0o644) + await assert.rejects( + materializePierWorkspaceArchive({ ...encoded, expected: encoded.manifest, destination }), + /must be empty/, + ) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/bench/src/pier-workspace-archive.ts b/bench/src/pier-workspace-archive.ts new file mode 100644 index 00000000..8a6a7d8f --- /dev/null +++ b/bench/src/pier-workspace-archive.ts @@ -0,0 +1,182 @@ +/** Deterministic regular-file archive used for runtime/Pier task outcomes. */ +import { createHash } from 'node:crypto' +import { chmod, lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute, join, resolve } from 'node:path' + +import { canonicalJson } from '@tangle-network/agent-eval' +import type { AgentCandidateWorkspaceManifestMaterialV1 } from '@tangle-network/agent-interface' + +export interface PierWorkspaceArchiveFile { + readonly path: string + readonly mode: 0o644 | 0o755 + readonly content: string + readonly sha256: `sha256:${string}` + readonly byteLength: number +} + +export interface PierWorkspaceArchiveV1 { + readonly schemaVersion: 1 + readonly kind: 'pier-workspace-archive' + readonly files: readonly PierWorkspaceArchiveFile[] +} + +export interface PierWorkspaceFile { + readonly path: string + readonly mode: 0o644 | 0o755 + readonly bytes: Uint8Array +} + +const sha256 = (bytes: Uint8Array): `sha256:${string}` => + `sha256:${createHash('sha256').update(bytes).digest('hex')}` + +/** Encode sorted, detached file bytes and return the exact matching runtime manifest. */ +export function createPierWorkspaceArchive(files: readonly PierWorkspaceFile[]): { + readonly archive: Uint8Array + readonly manifest: AgentCandidateWorkspaceManifestMaterialV1 +} { + const seen = new Set() + const ordered = [...files] + .map((file) => { + const path = safeRelativePath(file.path) + if (seen.has(path)) throw new Error(`Pier workspace archive repeats path ${path}`) + seen.add(path) + if (file.mode !== 0o644 && file.mode !== 0o755) { + throw new Error(`Pier workspace archive has unsupported mode for ${path}`) + } + const bytes = Uint8Array.from(file.bytes) + return { + path, + mode: file.mode, + content: Buffer.from(bytes).toString('base64'), + sha256: sha256(bytes), + byteLength: bytes.byteLength, + } + }) + .sort((left, right) => left.path.localeCompare(right.path)) + const document: PierWorkspaceArchiveV1 = { + schemaVersion: 1, + kind: 'pier-workspace-archive', + files: ordered, + } + return { + archive: Buffer.from(canonicalJson(document), 'utf8'), + manifest: { + schemaVersion: 1, + kind: 'agent-candidate-workspace-manifest', + files: ordered.map(({ path, mode, sha256, byteLength }) => ({ + path, + mode, + sha256, + byteLength, + })), + }, + } +} + +/** Decode only the canonical archive format and materialize its exact declared file set. */ +export async function materializePierWorkspaceArchive(input: { + readonly archive: Uint8Array + readonly expected: AgentCandidateWorkspaceManifestMaterialV1 + readonly destination: string +}): Promise { + const parsed = parseArchive(input.archive) + const observed = createPierWorkspaceArchive( + parsed.files.map((file) => ({ + path: file.path, + mode: file.mode, + bytes: Buffer.from(file.content, 'base64'), + })), + ) + if (!Buffer.from(observed.archive).equals(Buffer.from(input.archive))) { + throw new Error('Pier workspace archive is not canonical') + } + if (canonicalJson(observed.manifest) !== canonicalJson(input.expected)) { + throw new Error('Pier workspace archive does not match the signed manifest') + } + + const destination = resolve(input.destination) + await mkdir(destination, { recursive: true, mode: 0o700 }) + const stats = await lstat(destination) + if (!stats.isDirectory() || stats.isSymbolicLink() || (await realpath(destination)) !== destination) { + throw new Error('Pier workspace archive destination must be a real directory') + } + if ((await readdir(destination)).length !== 0) { + throw new Error('Pier workspace archive destination must be empty') + } + for (const file of parsed.files) { + const bytes = Buffer.from(file.content, 'base64') + if (bytes.byteLength !== file.byteLength || sha256(bytes) !== file.sha256) { + throw new Error(`Pier workspace archive content differs at ${file.path}`) + } + const output = join(destination, file.path) + await mkdir(dirname(output), { recursive: true, mode: 0o700 }) + await writeFile(output, bytes, { flag: 'wx', mode: 0o600 }) + await chmod(output, file.mode) + const written = await readFile(output) + if (!written.equals(bytes)) throw new Error(`Pier workspace archive write drifted at ${file.path}`) + } +} + +function parseArchive(bytes: Uint8Array): PierWorkspaceArchiveV1 { + let value: unknown + try { + value = JSON.parse(Buffer.from(bytes).toString('utf8')) + } catch (error) { + throw new Error('Pier workspace archive is not UTF-8 JSON', { cause: error }) + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Pier workspace archive must be an object') + } + const record = value as Record + if ( + record.schemaVersion !== 1 || + record.kind !== 'pier-workspace-archive' || + !Array.isArray(record.files) || + Object.keys(record).sort().join(',') !== 'files,kind,schemaVersion' + ) { + throw new Error('Pier workspace archive has an invalid envelope') + } + const files = record.files.map((value, index) => parseFile(value, index)) + return { schemaVersion: 1, kind: 'pier-workspace-archive', files } +} + +function parseFile(value: unknown, index: number): PierWorkspaceArchiveFile { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`Pier workspace archive file ${index} must be an object`) + } + const file = value as Record + if ( + Object.keys(file).sort().join(',') !== 'byteLength,content,mode,path,sha256' || + typeof file.path !== 'string' || + (file.mode !== 0o644 && file.mode !== 0o755) || + typeof file.content !== 'string' || + typeof file.sha256 !== 'string' || + !/^sha256:[a-f0-9]{64}$/.test(file.sha256) || + !Number.isSafeInteger(file.byteLength) || + (file.byteLength as number) < 0 + ) { + throw new Error(`Pier workspace archive file ${index} is invalid`) + } + return { + path: safeRelativePath(file.path), + mode: file.mode, + content: file.content, + sha256: file.sha256 as `sha256:${string}`, + byteLength: file.byteLength as number, + } +} + +function safeRelativePath(value: string): string { + const parts = value.split('/') + if ( + !value || + isAbsolute(value) || + value.includes('\\') || + value.includes('\0') || + parts.some((part) => !part || part === '.' || part === '..') || + parts[0]?.toLowerCase() === '.git' + ) { + throw new Error(`Pier workspace archive path is unsafe: ${value}`) + } + return value +} From 23bf499184058ba36b5d922ca288187b63e88c79 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 09:53:08 -0600 Subject: [PATCH 2/3] feat(bench): recover Pier candidate trials across restarts --- bench/HARNESS.md | 28 +- bench/README.md | 3 +- bench/package.json | 10 +- bench/pnpm-lock.yaml | 142 +++-- bench/scripts/terminate-pier-trial.mts | 37 ++ bench/scripts/verify-packed-consumer.mjs | 24 +- bench/scripts/verify-pier-agent.mts | 233 +++----- bench/scripts/verify-pier-pair.mts | 19 +- bench/scripts/verify-pier-recovery.mts | 139 +++++ bench/src/index.ts | 8 + bench/src/pier-agent.test.mts | 8 + bench/src/pier-agent.ts | 72 ++- bench/src/pier-trial-controller.test.mts | 181 ++++++ bench/src/pier-trial-controller.ts | 702 +++++++++++++++++++++++ bench/src/pier-trial-supervisor.mjs | 352 ++++++++++++ 15 files changed, 1747 insertions(+), 211 deletions(-) create mode 100644 bench/scripts/terminate-pier-trial.mts create mode 100644 bench/scripts/verify-pier-recovery.mts create mode 100644 bench/src/pier-trial-controller.test.mts create mode 100644 bench/src/pier-trial-controller.ts create mode 100644 bench/src/pier-trial-supervisor.mjs diff --git a/bench/HARNESS.md b/bench/HARNESS.md index 0ce1f4af..848d64b4 100644 --- a/bench/HARNESS.md +++ b/bench/HARNESS.md @@ -3,7 +3,7 @@ If you're an agent picking this up: read this page, then run `pnpm help` + `pnpm gate` — do NOT re-derive the harness from source. This map is SHORT on purpose; if it disagrees with the code, the code wins — fix this page in the same turn (the anti-rediscovery law). -Verified against source 2026-07-09 · agent-eval pinned `^0.108.1`. The CANONICAL surface is now +Verified against source 2026-07-11 · agent-eval pinned `^0.114.0`. The CANONICAL surface is now the published optimization suite (`@tangle-network/agent-runtime/loops`): `Environment` + `Strategy`/`defineStrategy` + `runBenchmark` — see the section below FIRST. The recursive diverse-vs-blind gate runs through the keystone (`gate-cli.mts` → `runGate`); @@ -234,11 +234,27 @@ Isolated memory and knowledge-bearing candidates currently fail closed until Pie The signed wall deadline is a hard stop: the runtime aborts, Pier kills the process tree, and the executor acknowledges process and container death. The signed tool-step count is a post-run validity check over protected traces, not a pre-tool stop; generic black-box Pier processes cannot honestly prevent step N+1. The executable zero-model fixture is `fixtures/pier-agent/`; run it against the R360 Pier checkout with `PIER_REPO=/path/to/pier pnpm verify:pier`. -That command runs both a no-change candidate that must score 0/1 and a known-good candidate that must score 1/1, then checks that each official result and exact task patch is bound into its own runtime receipt with zero model usage. +That command runs a no-change candidate that must score 0/1, proves a fresh evaluator process can kill a persisted child and remove its real Docker container, and runs a known-good candidate that must score 1/1. +It then checks that each official result and exact task patch is bound into its own runtime receipt with zero model usage. -For a real frozen candidate, start Pier synchronously inside the atomic callback, append `agentArgs` and `attemptArgs`, pass each executor-only `evaluatorEnv` entry through Pier's evaluator-owned environment mechanism, and return a handle that can kill and reap the process and remove its task container: +For a real frozen candidate, use `FilePierCandidateTrialController`, append `agentArgs` and `attemptArgs`, and pass each executor-only `evaluatorEnv` entry through Pier's evaluator-owned environment mechanism. +The controller sends secrets to its supervisor over a pipe, while its durable files contain only process and Docker-project identities: +`jobName` must be unique per prepared execution; the controller atomically reserves that job directory so recovery can remove only containers owned by that execution. ```ts +const controller = new FilePierCandidateTrialController({ + directory: '/var/lib/tangle/pier-control', + launch: (staged) => ({ + command: 'uv', + args: ['run', 'pier', 'run', ...staged.agentArgs, ...staged.attemptArgs], + cwd: pierCheckout, + env: { ...evaluatorEnvironment, ...staged.evaluatorEnv }, + jobsDirectory, + jobName, + readResult: () => readOfficialPierResult(jobsDirectory, jobName), + }), +}) + const result = await executePreparedPierCandidate({ prepared, directory: '/sealed/candidate', @@ -247,12 +263,12 @@ const result = await executePreparedPierCandidate({ claimStore, outputArtifacts, grader, - start: (staged, context) => startOnePierTrial(staged, context), + controller, }) ``` -The handle's result resolves only after normal process/container cleanup. -On a signed deadline or external abort, the adapter waits for `terminateAndWait()` to acknowledge both process exit and container removal before it returns control to the runtime. +The controller's result resolves only after normal process/container cleanup. +On a signed deadline, external abort, or recovery by another evaluator process, the adapter waits for `terminateAndWait()` to acknowledge both process exit and container removal before it returns control to the runtime. One prepared execution always maps to one Pier attempt (`--n-attempts 1 --max-retries 0`). Production callers pass a long-lived `FileAgentCandidateExecutionClaimStore`; an in-memory claim store is test-only and cannot prevent a second process from replaying the same attempt. diff --git a/bench/README.md b/bench/README.md index 1175b010..0491602b 100644 --- a/bench/README.md +++ b/bench/README.md @@ -19,5 +19,6 @@ The judge needs only Docker; workers need a model key (Tangle router `TANGLE_API The package executes a branded `PreparedAgentCandidateExecution` from `@tangle-network/agent-runtime` through one atomic API and ships `pier_agents.tangle_candidate:TangleCandidateAgent` as its thin Pier transport. The executor recreates every input from runtime-verified file bytes and reveals model credentials only inside the claimed execution callback. Pier owns the task container and verifier; protected model usage and traces stay in `@tangle-network/agent-eval` and are finalized by the shared runtime. -Run `PIER_REPO=/path/to/pier pnpm verify:pier` for the zero-model failure/pass proof, and see `HARNESS.md` for the exact invocation and failure contract. +`FilePierCandidateTrialController` atomically reserves a unique Pier job, then persists the supervisor PID, process-session identity, and that job's exact Docker projects so a fresh evaluator process can stop and remove an abandoned trial. +Run `PIER_REPO=/path/to/pier pnpm verify:pier` for the zero-model failure/pass and fresh-process recovery proof, and see `HARNESS.md` for the exact invocation and failure contract. From an installed npm package, expose the shipped Python module with `export PYTHONPATH="$(npm root)/@tangle-network/agent-bench${PYTHONPATH:+:$PYTHONPATH}"` before invoking Pier. diff --git a/bench/package.json b/bench/package.json index b3db33e5..413cde88 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.1.1", + "version": "0.2.0", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "main": "src/index.ts", @@ -22,10 +22,10 @@ "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "^0.108.1", - "@tangle-network/agent-interface": "^0.23.0", - "@tangle-network/agent-runtime": "^0.90.1", - "@tangle-network/sandbox": "^0.9.7" + "@tangle-network/agent-eval": "^0.114.0", + "@tangle-network/agent-interface": "^0.25.0", + "@tangle-network/agent-runtime": "^0.92.0", + "@tangle-network/sandbox": "^0.10.3" }, "devDependencies": { "@types/node": "^25.0.0", diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index caf019de..45557ff1 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -9,17 +9,17 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.108.1 - version: 0.108.1(typescript@6.0.3) + specifier: ^0.114.0 + version: 0.114.0(typescript@6.0.3) '@tangle-network/agent-interface': - specifier: ^0.23.0 - version: 0.23.0 + specifier: ^0.25.0 + version: 0.25.0 '@tangle-network/agent-runtime': - specifier: ^0.90.1 - version: 0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.23.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) + specifier: ^0.92.0 + version: 0.92.0(@tangle-network/agent-eval@0.114.0(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) '@tangle-network/sandbox': - specifier: ^0.9.7 - version: 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + specifier: ^0.10.3 + version: 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) devDependencies: '@types/node': specifier: ^25.0.0 @@ -257,13 +257,18 @@ packages: '@tangle-network/agent-core@0.3.4': resolution: {integrity: sha512-Hvz3ABRouNtBmRvGqPxifAO2yuILneJMylWH5jW/jeS2F03RvqkGYuXyGXWWLqosYbb3hVAvSEe4Ykm2FMGEDQ==} - '@tangle-network/agent-eval@0.108.1': - resolution: {integrity: sha512-5T6XKtqcxB8OrR1c8V4EGZk+gQizxmVVQKLCkxTlnMh2lApV+e0EIbEUpKZzD9HmCQSgRY8a2tW3hCTFBuZAyg==} + '@tangle-network/agent-core@0.4.9': + resolution: {integrity: sha512-BFU4WV12Y6D28PX+o8LIVkIMD+cXwrL1uyV182l12Bn9zEu7VHt2oVMEEzbsN8L+X0xM9baEfMCHTFKFNJv7Aw==} + + '@tangle-network/agent-eval@0.113.0': + resolution: {integrity: sha512-1KhW6l69arpanK8vamnvJueD+ZwEO4A7BMF5qpkPLXUVVNcrUail0GWp3M5SpAjQdACa3HTuVdexXuEpZt5sQg==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-interface@0.10.1': - resolution: {integrity: sha512-yehY/0EgKvu8lG6jIVoZCtMPLkj8VEWwasuAtuph2RaB9MKE5wuxRF647O6jw8KufNZ3aQ2UVVWpZ19dGCbs6w==} + '@tangle-network/agent-eval@0.114.0': + resolution: {integrity: sha512-pIZCLkKsHeKDpLOpRXRhnc2HaREcHduHm4u3jw0nN73B1BWe6Jaqh2wlNHTPnPUA1qQl5sCl1K6XF1p9R7eTZw==} + engines: {node: '>=20'} + hasBin: true '@tangle-network/agent-interface@0.13.0': resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} @@ -271,31 +276,61 @@ packages: '@tangle-network/agent-interface@0.14.0': resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} - '@tangle-network/agent-interface@0.23.0': - resolution: {integrity: sha512-545pWgTQn9JixXeFJ7KIii1Y86i+26QXBvvsNvXKFDIuyzITUY73/DJt10JxUpcJZwFxzbgwUBKubOE+uiBz/Q==} + '@tangle-network/agent-interface@0.21.0': + resolution: {integrity: sha512-jDxhVJgxymrvU1RLWxWKueuaWQIpBAfrW8BuVTB5m2Y4eFMLo1SawDBEMDLdZN4/Gf34xrFrsRk+PAj9brGKMQ==} + + '@tangle-network/agent-interface@0.22.0': + resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} - '@tangle-network/agent-knowledge@1.11.1': - resolution: {integrity: sha512-2vvNHHsb4TQFnY+SrL7rYaMhwTxTK8R/lD9IAteonmgwcE5amhpPQZqftw+UDMl1d6OBe4QX+Qv1pNmWQDXzqQ==} + '@tangle-network/agent-interface@0.24.0': + resolution: {integrity: sha512-iaHWNTYne49cBYXwb72NjGDw5bjT2KVJlfawXygbvkqnfUsQG57BS++Yjf/XYvwJJdDeEaGs+Xvy8SWYkvu0qA==} + + '@tangle-network/agent-interface@0.25.0': + resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} + + '@tangle-network/agent-knowledge@1.11.2': + resolution: {integrity: sha512-sJLLYai35JIHou0YgcPg5I4WCz90vhIUmyF+DQ5GtMBDB+KT92y+rwvLQ4MojIZCTq3p/xv8C3SEhkuqCfzVsw==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-runtime@0.90.1': - resolution: {integrity: sha512-doX0KtA5QAJOYifCAAd8EmjOoMWwlpsVfvXDRdaZJIakAzrznstSllK9NeaENpZBj19yb6XTN3jPKuZwbcl6rQ==} + '@tangle-network/agent-profile-materialize@0.3.1': + resolution: {integrity: sha512-yA/DaxmC+DsHdPl00iovR3tg80lQY1ukkHgGViLtzFYY7O46yZR9Ykhw/tMGTFYWFi5bzqjmr0NShSxdNmSzcQ==} + + '@tangle-network/agent-runtime@0.92.0': + resolution: {integrity: sha512-R4G7JTgpQJ/Xgzf0+YQ50VWewBoAnfJnF3nwQPCE3VzvDJzQ/kBVcVDaRTEtteP/ygS30PQi90tnbBSy2x3wzg==} engines: {node: '>=20'} hasBin: true peerDependencies: '@tangle-network/agent-eval': '>=0.101.0 <1.0.0' - '@tangle-network/agent-interface': '>=0.14.0 <1.0.0' + '@tangle-network/agent-interface': '>=0.25.0 <1.0.0' '@tangle-network/sandbox': '>=0.8.0 <1.0.0' playwright: ^1.40.0 peerDependenciesMeta: - '@tangle-network/agent-interface': - optional: true '@tangle-network/sandbox': optional: true playwright: optional: true + '@tangle-network/sandbox@0.10.3': + resolution: {integrity: sha512-3nZnIaXc/vCH3Lb6SCZA3cYEJT4+ThGXLPkGo5z2kh434n3eJryk/PdnSXSwPoA6xuTQToJAPbGGIdMEHFU/Vw==} + peerDependencies: + '@mastra/core': ^1.36.0 + '@modelcontextprotocol/sdk': ^1.29.0 + ai: ^6.0.175 + openai: ^6.36.0 + viem: ^2.0.0 + peerDependenciesMeta: + '@mastra/core': + optional: true + '@modelcontextprotocol/sdk': + optional: true + ai: + optional: true + openai: + optional: true + viem: + optional: true + '@tangle-network/sandbox@0.9.7': resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: @@ -565,12 +600,17 @@ snapshots: '@tangle-network/agent-interface': 0.14.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.108.1(typescript@6.0.3)': + '@tangle-network/agent-core@0.4.9': + dependencies: + '@tangle-network/agent-interface': 0.24.0 + zod: 4.4.3 + + '@tangle-network/agent-eval@0.113.0(typescript@6.0.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) '@hono/node-server': 2.0.4(hono@4.12.23) - '@tangle-network/agent-interface': 0.10.1 + '@tangle-network/agent-interface': 0.22.0 '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) hono: 4.12.23 zod: 4.4.3 @@ -583,9 +623,23 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-interface@0.10.1': + '@tangle-network/agent-eval@0.114.0(typescript@6.0.3)': dependencies: + '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) + '@ax-llm/ax': 19.0.45(zod@4.4.3) + '@hono/node-server': 2.0.4(hono@4.12.23) + '@tangle-network/agent-interface': 0.22.0 + '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) + hono: 4.12.23 zod: 4.4.3 + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate '@tangle-network/agent-interface@0.13.0': dependencies: @@ -595,13 +649,25 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.23.0': + '@tangle-network/agent-interface@0.21.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-knowledge@1.11.1(typescript@6.0.3)': + '@tangle-network/agent-interface@0.22.0': dependencies: - '@tangle-network/agent-eval': 0.108.1(typescript@6.0.3) + zod: 4.4.3 + + '@tangle-network/agent-interface@0.24.0': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-interface@0.25.0': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-knowledge@1.11.2(typescript@6.0.3)': + dependencies: + '@tangle-network/agent-eval': 0.113.0(typescript@6.0.3) zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -612,13 +678,18 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-runtime@0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.23.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': + '@tangle-network/agent-profile-materialize@0.3.1': dependencies: - '@tangle-network/agent-eval': 0.108.1(typescript@6.0.3) - '@tangle-network/agent-knowledge': 1.11.1(typescript@6.0.3) + '@tangle-network/agent-interface': 0.22.0 + + '@tangle-network/agent-runtime@0.92.0(@tangle-network/agent-eval@0.114.0(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': + dependencies: + '@tangle-network/agent-eval': 0.114.0(typescript@6.0.3) + '@tangle-network/agent-interface': 0.25.0 + '@tangle-network/agent-knowledge': 1.11.2(typescript@6.0.3) + '@tangle-network/agent-profile-materialize': 0.3.1 optionalDependencies: - '@tangle-network/agent-interface': 0.23.0 - '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + '@tangle-network/sandbox': 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) transitivePeerDependencies: - '@mastra/core' - '@modelcontextprotocol/sdk' @@ -628,6 +699,13 @@ snapshots: - typescript - utf-8-validate + '@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': + dependencies: + '@tangle-network/agent-core': 0.4.9 + '@tangle-network/agent-interface': 0.21.0 + optionalDependencies: + viem: 2.52.0(typescript@6.0.3)(zod@4.4.3) + '@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.3.4 diff --git a/bench/scripts/terminate-pier-trial.mts b/bench/scripts/terminate-pier-trial.mts new file mode 100644 index 00000000..e9aec48d --- /dev/null +++ b/bench/scripts/terminate-pier-trial.mts @@ -0,0 +1,37 @@ +import path from 'node:path' + +import { InMemoryTraceStore } from '@tangle-network/agent-eval' + +import { createPierCandidateRecoveryExecutor } from '../src/pier-agent' +import { FilePierCandidateTrialController } from '../src/pier-trial-controller' + +const [directoryArg, executionId, executionPlanDigest, ...extra] = process.argv.slice(2) +if (!directoryArg || !executionId || !executionPlanDigest || extra.length > 0) { + throw new Error( + 'usage: terminate-pier-trial.mts ', + ) +} +if (!/^sha256:[a-f0-9]{64}$/.test(executionPlanDigest)) { + throw new Error('execution-plan-digest must be a SHA-256 digest') +} + +const controller = new FilePierCandidateTrialController({ + directory: path.resolve(directoryArg), +}) +const executor = createPierCandidateRecoveryExecutor(controller) +const recovered = await executor.stopAndCapture( + { + executionId, + executionPlanDigest: executionPlanDigest as `sha256:${string}`, + }, + { + traceStore: new InMemoryTraceStore(), + reason: 'failed', + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 30_000, + }, +) +if (recovered.stopped !== true) throw new Error('Pier recovery executor did not stop the trial') +process.stdout.write( + `${JSON.stringify({ processExited: true, containersRemoved: true })}\n`, +) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 5af41e4e..d42826cf 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -83,7 +83,7 @@ try { ) await writeFile( path.join(consumerDir, 'index.ts'), - "import { executePreparedPierCandidate, resolveAdapter, runBenchmarks, type BenchmarkAdapter, type PierCandidateTrialHandle, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nvoid adapter\nvoid staged\nvoid trial\nvoid executePreparedPierCandidate\nvoid runBenchmarks\n", + "import { executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, type BenchmarkAdapter, type PierCandidateTrialController, type PierCandidateTrialHandle, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nvoid adapter\nvoid staged\nvoid trial\nvoid controller\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\n", ) await writeFile( path.join(consumerDir, 'tsconfig.json'), @@ -142,6 +142,26 @@ assert TangleCandidateAgent.__module__ == "pier_agents.tangle_candidate" await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir) const installedPackage = path.join(consumerDir, 'node_modules', '@tangle-network', 'agent-bench') + const prepared = await run( + 'npm', + [ + 'exec', + '--', + 'tsx', + path.join(installedPackage, 'scripts', 'verify-pier-agent.mts'), + ], + consumerDir, + { ...process.env, PIER_PREPARE_ONLY: '1' }, + ) + const prepareProof = JSON.parse(prepared.stdout) + if ( + prepareProof.prepared !== true || + prepareProof.disposed !== true || + !/^sha256:[a-f0-9]{64}$/.test(prepareProof.executionPlanDigest) || + !/^sha256:[a-f0-9]{64}$/.test(prepareProof.graderDigest) + ) { + throw new Error(`packed consumer did not prepare a real candidate: ${prepared.stdout}`) + } await run('python3', ['verify_pier_import.py'], consumerDir, { ...process.env, PYTHONPATH: installedPackage, @@ -160,7 +180,7 @@ assert TangleCandidateAgent.__module__ == "pier_agents.tangle_candidate" }) } console.log( - `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}`, + `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}; prepared ${prepareProof.executionPlanDigest}`, ) } finally { await rm(scratch, { recursive: true, force: true }) diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts index d5f54167..b530c56e 100644 --- a/bench/scripts/verify-pier-agent.mts +++ b/bench/scripts/verify-pier-agent.mts @@ -1,4 +1,4 @@ -import { execFileSync, spawn } from 'node:child_process' +import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' import { chmodSync, @@ -23,6 +23,7 @@ import type { Sha256Digest, } from '@tangle-network/agent-interface' import { + disposePreparedAgentCandidateExecution, FileAgentCandidateExecutionClaimStore, prepareAgentCandidateExecution, type AgentCandidateExecutionPorts, @@ -34,13 +35,15 @@ import { import { executePreparedPierCandidate } from '../src/pier-agent' import { createPierResultGrader } from '../src/pier-result-grader' +import { FilePierCandidateTrialController } from '../src/pier-trial-controller' import { materializePierWorkspaceArchive } from '../src/pier-workspace-archive' const pinnedPierCommit = 'e69a20e4e0ac073ec71fde0274bab3d9f40bac87' const pinnedPierVersion = '0.3.0' const modelRequest = 'openai/gpt-5.4' const fixtureImage = 'ghcr.io/tangle-network/devcontainers/universal:latest' -const proofArm = process.env.PIER_PROOF_ARM +const prepareOnly = process.env.PIER_PREPARE_ONLY === '1' +const proofArm = process.env.PIER_PROOF_ARM ?? (prepareOnly ? 'failure' : undefined) if (proofArm !== 'failure' && proofArm !== 'success') { throw new Error('PIER_PROOF_ARM must be failure or success') } @@ -72,110 +75,6 @@ function output( }).trim() } -function containerIdsForImage(image: string): Set { - const ids = output('docker', ['ps', '-aq', '--filter', `ancestor=${image}`]) - return new Set(ids ? ids.split('\n').filter(Boolean) : []) -} - -function newContainerIds(image: string, before: ReadonlySet): string[] { - return [...containerIdsForImage(image)].filter((id) => !before.has(id)) -} - -function killProcessGroup(pid: number, signal: NodeJS.Signals): void { - try { - process.kill(-pid, signal) - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error - } -} - -function startPierProcess( - args: readonly string[], - env: NodeJS.ProcessEnv, - taskImage: string, - protectedValues: readonly string[], -) { - const containersBefore = containerIdsForImage(taskImage) - const child = spawn('uv', [...args], { - cwd: pierRepo, - env, - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - }) - if (child.pid === undefined) throw new Error('Pier process started without a pid') - const pid = child.pid - const stdout: Buffer[] = [] - const stderr: Buffer[] = [] - child.stdout.on('data', (chunk: Buffer) => stdout.push(Buffer.from(chunk))) - child.stderr.on('data', (chunk: Buffer) => stderr.push(Buffer.from(chunk))) - const exited = new Promise<{ - code: number | null - signal: NodeJS.Signals | null - error?: Error - }>((resolveExit) => { - let settled = false - child.once('error', (error) => { - if (settled) return - settled = true - resolveExit({ code: null, signal: null, error }) - }) - child.once('close', (code, signal) => { - if (settled) return - settled = true - resolveExit({ code, signal }) - }) - }) - - const removeTaskContainers = (): void => { - const containers = newContainerIds(taskImage, containersBefore) - if (containers.length > 0) output('docker', ['rm', '-f', ...containers]) - const remaining = newContainerIds(taskImage, containersBefore) - if (remaining.length > 0) { - throw new Error(`Pier task containers survived removal: ${remaining.join(', ')}`) - } - } - - const result = exited.then((exit) => { - removeTaskContainers() - if (exit.error) throw new Error(`Pier process failed to start: ${exit.error.message}`) - if (exit.code !== 0) { - let safeStderr = Buffer.concat(stderr).toString('utf8') - for (const value of protectedValues) safeStderr = safeStderr.replaceAll(value, '[redacted]') - throw new Error(`Pier process failed (${exit.signal ?? exit.code}): ${safeStderr}`) - } - return { - stdout: Buffer.concat(stdout).toString('utf8'), - stderr: Buffer.concat(stderr).toString('utf8'), - } - }) - - let terminating: Promise<{ processExited: true; containersRemoved: true }> | undefined - return { - result, - terminateAndWait: () => { - terminating ??= (async () => { - killProcessGroup(pid, 'SIGTERM') - const graceful = await Promise.race([ - exited.then(() => true), - new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), 2_000)), - ]) - if (!graceful) { - killProcessGroup(pid, 'SIGKILL') - const killed = await Promise.race([ - exited.then(() => true), - new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), 5_000)), - ]) - if (!killed) throw new Error(`Pier process group ${pid} survived SIGKILL`) - } - - removeTaskContainers() - return { processExited: true as const, containersRemoved: true as const } - })() - return terminating - }, - } -} - function sha256(bytes: Uint8Array): Sha256Digest { return `sha256:${createHash('sha256').update(bytes).digest('hex')}` } @@ -355,15 +254,17 @@ function outputArtifactStore(graderBytes: Uint8Array): { } try { - const pierHead = output('git', ['rev-parse', 'HEAD'], pierRepo) - if (pierHead !== pinnedPierCommit) { - throw new Error(`Pier checkout mismatch: expected ${pinnedPierCommit}, got ${pierHead}`) - } - const pierStatus = output('git', ['status', '--porcelain'], pierRepo) - if (pierStatus !== '') throw new Error(`Pier checkout must be clean: ${pierStatus}`) - const pierVersion = output('uv', ['run', 'pier', '--version'], pierRepo) - if (pierVersion !== pinnedPierVersion) { - throw new Error(`Pier version mismatch: expected ${pinnedPierVersion}, got ${pierVersion}`) + if (!prepareOnly) { + const pierHead = output('git', ['rev-parse', 'HEAD'], pierRepo) + if (pierHead !== pinnedPierCommit) { + throw new Error(`Pier checkout mismatch: expected ${pinnedPierCommit}, got ${pierHead}`) + } + const pierStatus = output('git', ['status', '--porcelain'], pierRepo) + if (pierStatus !== '') throw new Error(`Pier checkout must be clean: ${pierStatus}`) + const pierVersion = output('uv', ['run', 'pier', '--version'], pierRepo) + if (pierVersion !== pinnedPierVersion) { + throw new Error(`Pier version mismatch: expected ${pinnedPierVersion}, got ${pierVersion}`) + } } cpSync(fixtureSource, taskDir, { recursive: true }) @@ -380,16 +281,23 @@ try { readFileSync(path.join(taskDir, 'environment', 'seed', 'src', 'status.txt')), ]), ).slice(7, 23) - const identity = publicOciIdentity(fixtureImage) + const identity = prepareOnly + ? { + indexDigest: `sha256:${'1'.repeat(64)}` as Sha256Digest, + manifestDigest: `sha256:${'2'.repeat(64)}` as Sha256Digest, + } + : publicOciIdentity(fixtureImage) const pinnedImage = `${fixtureImage}@${identity.indexDigest}` - output('docker', ['pull', '--platform', 'linux/amd64', pinnedImage]) - const platform = output('docker', [ - 'image', - 'inspect', - '--format', - '{{.Os}}/{{.Architecture}}', - pinnedImage, - ]) + if (!prepareOnly) output('docker', ['pull', '--platform', 'linux/amd64', pinnedImage]) + const platform = prepareOnly + ? 'linux/amd64' + : output('docker', [ + 'image', + 'inspect', + '--format', + '{{.Os}}/{{.Architecture}}', + pinnedImage, + ]) if (platform !== 'linux/amd64') throw new Error(`fixture image platform drifted: ${platform}`) const configPath = path.join(taskDir, 'task.toml') @@ -492,6 +400,13 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= manifestDigest: identity.manifestDigest, platform: { os: 'linux', architecture: 'amd64' }, } + const graderBytes = readFileSync(new URL('../src/pier-result-grader.mjs', import.meta.url)) + const { outputArtifacts, graderArtifact } = outputArtifactStore(graderBytes) + const grader = createPierResultGrader({ + name: 'pier-official-result', + version: '1.0.0', + artifact: graderArtifact, + }) const executionId = `pier-no-model-${proofArm}-${contextDigest}` const task: AgentCandidateTaskExecution = { executionId, @@ -508,6 +423,11 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= }, attempt: { number: 1, maxAttempts: 1, retryPolicy: 'none' }, model: { requested: modelRequest, reasoningEffort: 'xhigh' }, + grader: { + name: grader.name, + version: grader.version, + artifact: grader.artifact, + }, executionRoots: { taskRoot: '/app', candidateRoot: '/opt/tangle-candidate' }, stagingRoots: { taskRoot, candidateRoot, profileRoot }, workspace: taskWorkspace, @@ -578,34 +498,35 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= const verified = await verifyAgentCandidateBundle(bundle, ports) const prepared = await prepareAgentCandidateExecution(verified, task, ports) + if (prepareOnly) { + const disposal = await disposePreparedAgentCandidateExecution(prepared) + if (disposal.disposed !== true) throw new Error('prepared candidate was not disposed') + process.stdout.write( + `${JSON.stringify({ + prepared: true, + disposed: true, + executionPlanDigest: prepared.executionPlan.value.digest, + graderDigest: task.grader.artifact.sha256, + })}\n`, + ) + } else { const traceStore = new InMemoryTraceStore() const claimStore = new FileAgentCandidateExecutionClaimStore({ directory: path.join(scratch, 'claims'), }) - const graderBytes = readFileSync(new URL('../src/pier-result-grader.mjs', import.meta.url)) - const { outputArtifacts, graderArtifact } = outputArtifactStore(graderBytes) - const grader = createPierResultGrader({ - name: 'pier-official-result', - version: '1.0.0', - artifact: graderArtifact, - }) let acceptedRewards: { reward: number; patch_applied: number } | undefined let acceptedTrialPath: string | undefined - const finalized = await executePreparedPierCandidate({ - prepared, - directory: path.join(scratch, 'sealed'), - pierVersion: pinnedPierVersion, - traceStore, - claimStore, - outputArtifacts, - grader, - start: (staged, { request }) => { + const jobName = `tangle-runtime-candidate-no-model-${proofArm}` + const controller = new FilePierCandidateTrialController({ + directory: path.join(scratch, 'trial-control'), + launch: (staged, { request }) => { const evaluatorArgs = Object.keys(staged.evaluatorEnv).flatMap((name) => [ '--agent-env', `${name}=\${${name}}`, ]) - const trial = startPierProcess( - [ + return { + command: 'uv', + args: [ 'run', 'pier', 'run', @@ -617,7 +538,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= '--env', 'docker', '--job-name', - `tangle-runtime-candidate-no-model-${proofArm}`, + jobName, '--jobs-dir', jobsDir, '--n-concurrent', @@ -626,14 +547,11 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= '2', '--quiet', ], - { ...process.env, PYTHONPATH: benchDir, ...staged.evaluatorEnv }, - pinnedImage, - Object.values(staged.evaluatorEnv), - ) - - return { - ...trial, - result: trial.result.then(async () => { + cwd: pierRepo, + env: { ...process.env, PYTHONPATH: benchDir, ...staged.evaluatorEnv }, + jobsDirectory: jobsDir, + jobName, + readResult: async () => { const trialResult = findTrialResult(jobsDir) if (!trialResult) throw new Error(`Pier emitted no trial result under ${jobsDir}`) const result = trialResult.value @@ -681,10 +599,20 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= path.join(path.dirname(trialResult.path), 'artifacts', 'model.patch'), ), } - }), + }, } }, }) + const finalized = await executePreparedPierCandidate({ + prepared, + directory: path.join(scratch, 'sealed'), + pierVersion: pinnedPierVersion, + traceStore, + claimStore, + outputArtifacts, + grader, + controller, + }) if (!finalized.succeeded) { throw new Error(`runtime rejected the protected Pier capture: ${finalized.reason}`) } @@ -738,6 +666,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= 2, ), ) + } } finally { if (process.env.KEEP_PIER_FIXTURE !== '1') rmSync(scratch, { recursive: true, force: true }) else console.error(`Pier runtime fixture retained at ${scratch}`) diff --git a/bench/scripts/verify-pier-pair.mts b/bench/scripts/verify-pier-pair.mts index 9d6e0e25..e5d60152 100644 --- a/bench/scripts/verify-pier-pair.mts +++ b/bench/scripts/verify-pier-pair.mts @@ -15,6 +15,7 @@ interface PierProofResult { const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const script = path.join(benchDir, 'scripts', 'verify-pier-agent.mts') +const recoveryScript = path.join(benchDir, 'scripts', 'verify-pier-recovery.mts') function runArm(arm: PierProofResult['arm']): PierProofResult { const stdout = execFileSync(process.execPath, ['--import', 'tsx', script], { @@ -29,6 +30,22 @@ function runArm(arm: PierProofResult['arm']): PierProofResult { } const failure = runArm('failure') +const recovery = JSON.parse( + execFileSync(process.execPath, ['--import', 'tsx', recoveryScript], { + cwd: benchDir, + env: { ...process.env }, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + timeout: 120_000, + }), +) as { freshProcess?: boolean; processExited?: boolean; containersRemoved?: boolean } +if ( + recovery.freshProcess !== true || + recovery.processExited !== true || + recovery.containersRemoved !== true +) { + throw new Error(`Pier fresh-process recovery failed: ${JSON.stringify(recovery)}`) +} const success = runArm('success') if ( failure.arm !== 'failure' || @@ -54,4 +71,4 @@ for (const result of [failure, success]) { } } -console.log(JSON.stringify({ controls: [failure, success] }, null, 2)) +console.log(JSON.stringify({ recovery, controls: [failure, success] }, null, 2)) diff --git a/bench/scripts/verify-pier-recovery.mts b/bench/scripts/verify-pier-recovery.mts new file mode 100644 index 00000000..c9779018 --- /dev/null +++ b/bench/scripts/verify-pier-recovery.mts @@ -0,0 +1,139 @@ +import { execFileSync } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' +import { InMemoryTraceStore } from '@tangle-network/agent-eval' + +import type { StagedPierCandidateExecution } from '../src/pier-agent' +import { FilePierCandidateTrialController } from '../src/pier-trial-controller' + +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const root = mkdtempSync(path.join(tmpdir(), 'pier-fresh-recovery-')) +const controlRoot = path.join(root, 'control') +const jobsDirectory = path.join(root, 'jobs') +const jobName = 'fresh-recovery' +const trialName = 'fresh-recovery-trial' +const executionId = 'pier-fresh-process-recovery' +const executionPlanDigest = `sha256:${'e'.repeat(64)}` as const +const image = 'ghcr.io/tangle-network/devcontainers/universal:latest' +let containerId: string | undefined +let handle: ReturnType | undefined +let resultSettled: Promise | undefined + +try { + containerId = execFileSync( + 'docker', + [ + 'run', + '-d', + '--label', + `com.docker.compose.project=${trialName}`, + image, + 'sleep', + 'infinity', + ], + { encoding: 'utf8', timeout: 120_000 }, + ).trim() + if (!/^[a-f0-9]{12,64}$/.test(containerId)) { + throw new Error(`Docker returned an invalid recovery container id: ${containerId}`) + } + + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + launch: () => ({ + command: process.execPath, + args: [ + '-e', + `require('node:fs').mkdirSync(${JSON.stringify(path.join(jobsDirectory, jobName, trialName))}, { recursive: true }); setInterval(() => undefined, 1_000)`, + ], + cwd: root, + env: { ...process.env }, + jobsDirectory, + jobName, + readResult: async () => { + throw new Error('recovery probe must not complete normally') + }, + }), + }) + handle = controller.start( + { executionId, evaluatorEnv: {} } as StagedPierCandidateExecution, + { + request: { + executionId, + executionPlan: { value: { digest: executionPlanDigest } }, + } as AgentCandidateExecutorRequest, + traceStore: new InMemoryTraceStore(), + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 30_000, + }, + ) + resultSettled = handle.result.catch(() => undefined) + + const [slot] = readdirSync(controlRoot) + if (!slot) throw new Error('Pier controller omitted durable state') + const identityPath = path.join(controlRoot, slot, 'identity.json') + let pierPid: number | undefined + for (let attempt = 0; attempt < 200; attempt++) { + const identity = JSON.parse(readFileSync(identityPath, 'utf8')) + if (identity.state === 'running' && Number.isSafeInteger(identity.pier?.pid)) { + pierPid = identity.pier.pid + break + } + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + if (!pierPid) throw new Error('Pier controller never persisted the child process identity') + const trialDirectory = path.join(jobsDirectory, jobName, trialName) + for (let attempt = 0; attempt < 200 && !existsSync(trialDirectory); attempt++) { + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + if (!existsSync(trialDirectory)) { + throw new Error('recovery probe never created its evaluator-owned trial directory') + } + + const recovery = JSON.parse( + execFileSync( + process.execPath, + [ + '--import', + 'tsx', + path.join(benchDir, 'scripts', 'terminate-pier-trial.mts'), + controlRoot, + executionId, + executionPlanDigest, + ], + { cwd: benchDir, encoding: 'utf8', timeout: 20_000 }, + ), + ) + if (recovery.processExited !== true || recovery.containersRemoved !== true) { + throw new Error(`fresh recovery returned an incomplete acknowledgement: ${JSON.stringify(recovery)}`) + } + try { + process.kill(pierPid, 0) + throw new Error(`recovered Pier process ${pierPid} is still alive`) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } + let containerPresent = true + try { + execFileSync('docker', ['inspect', containerId], { stdio: 'ignore', timeout: 30_000 }) + } catch { + containerPresent = false + } + if (containerPresent) throw new Error(`recovered Pier container ${containerId} is still present`) + await resultSettled + process.stdout.write( + `${JSON.stringify({ freshProcess: true, processExited: true, containersRemoved: true })}\n`, + ) +} finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + if (resultSettled) await resultSettled + if (containerId) { + try { + execFileSync('docker', ['rm', '-f', containerId], { stdio: 'ignore', timeout: 30_000 }) + } catch {} + } + rmSync(root, { recursive: true, force: true }) +} diff --git a/bench/src/index.ts b/bench/src/index.ts index 412a3db0..fc66a7ba 100644 --- a/bench/src/index.ts +++ b/bench/src/index.ts @@ -54,15 +54,23 @@ export { type RunBenchmarksReport, } from './run-benchmarks' export { + createPierCandidateRecoveryExecutor, executePreparedPierCandidate, type ExecutePreparedPierCandidateOptions, type PierCandidateGraderPort, type PierCandidateOfficialResult, type PierCandidateTerminationAcknowledgement, + type PierCandidateTrialController, type PierCandidateTrialHandle, + type PierCandidateTrialIdentity, type PierCandidateTrialResult, type StagedPierCandidateExecution, } from './pier-agent' +export { + FilePierCandidateTrialController, + type FilePierCandidateTrialControllerOptions, + type PierCandidateProcessSpec, +} from './pier-trial-controller' export { createPierResultGrader } from './pier-result-grader' export { createPierWorkspaceArchive, diff --git a/bench/src/pier-agent.test.mts b/bench/src/pier-agent.test.mts index 0128dee2..61d3bae6 100644 --- a/bench/src/pier-agent.test.mts +++ b/bench/src/pier-agent.test.mts @@ -314,6 +314,10 @@ test('waits for process and container death acknowledgement after abort', async let terminated = false const waiting = awaitAbortableTrial( { + identity: { + executionId: 'abort-test', + executionPlanDigest: `sha256:${'a'.repeat(64)}`, + }, result: new Promise(() => undefined), terminateAndWait: async () => { await new Promise((resolve) => setTimeout(resolve, 5)) @@ -332,6 +336,10 @@ test('fails closed when termination does not acknowledge container removal', asy const controller = new AbortController() const waiting = awaitAbortableTrial( { + identity: { + executionId: 'bad-ack-test', + executionPlanDigest: `sha256:${'b'.repeat(64)}`, + }, result: new Promise(() => undefined), terminateAndWait: async () => ({ processExited: true, containersRemoved: false }) as never, }, diff --git a/bench/src/pier-agent.ts b/bench/src/pier-agent.ts index 367981f3..7319b2b3 100644 --- a/bench/src/pier-agent.ts +++ b/bench/src/pier-agent.ts @@ -8,6 +8,7 @@ import type { AgentCandidateBenchmarkGraderPort, AgentCandidateExecutionClaimStore, AgentCandidateExecutorRequest, + AgentCandidateExecutorStopRequest, AgentCandidateExecutorPort, AgentCandidateOutputArtifactPort, AgentCandidateProtectedRunCapture, @@ -53,12 +54,35 @@ export interface PierCandidateTerminationAcknowledgement { } export interface PierCandidateTrialHandle { + /** Non-secret durable identity shared with a fresh evaluator process. */ + readonly identity: PierCandidateTrialIdentity /** Resolves only after the Pier process exits and its task container is gone. */ readonly result: Promise /** Idempotently kill/reap Pier and remove its task container, then acknowledge their death. */ readonly terminateAndWait: () => Promise } +export type PierCandidateTrialIdentity = Readonly + +/** + * Evaluator-owned lifecycle whose stop path works without the process-local + * handle returned by `start`. + */ +export interface PierCandidateTrialController { + start( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ): PierCandidateTrialHandle + terminateAndWait( + identity: PierCandidateTrialIdentity, + ): Promise +} + /** Evaluator-owned bytes captured from one completed official Pier trial. */ export interface PierCandidateTrialResult { /** Parsed value of the exact `result.json` bytes. */ @@ -94,18 +118,30 @@ export interface ExecutePreparedPierCandidateOptions extends StagePreparedPierCa readonly outputArtifacts: AgentCandidateOutputArtifactPort readonly grader: PierCandidateGraderPort /** - * Start exactly one Pier trial synchronously so an abort cannot race an unowned process. - * The returned handle owns process/container termination for the whole trial. + * Starts exactly one Pier trial synchronously and persists its non-secret + * process/container identity before returning. */ - readonly start: ( - staged: StagedPierCandidateExecution, - context: { - readonly request: AgentCandidateExecutorRequest - readonly traceStore: TraceStore - readonly signal: AbortSignal - readonly deadlineAtMs: number + readonly controller: PierCandidateTrialController +} + +/** Recovery-only runtime executor for an expired attempt owned by another process. */ +export function createPierCandidateRecoveryExecutor( + controller: PierCandidateTrialController, +): AgentCandidateExecutorPort { + return { + execute: async () => { + throw new Error('recovery-only Pier executor cannot start a candidate') }, - ) => PierCandidateTrialHandle + stopAndCapture: async (request) => { + assertTerminationAcknowledged( + await controller.terminateAndWait({ + executionId: request.executionId, + executionPlanDigest: request.executionPlanDigest, + }), + ) + return { stopped: true } + }, + } } interface PierResultLike { @@ -506,12 +542,19 @@ export async function executePreparedPierCandidate( context.signal.throwIfAborted() const identity = trialIdentity(request.executionId, request.executionPlan.value.digest) if (trials.has(identity)) throw new Error('Pier trial identity is already active') - const trial = options.start(staged, { + const trial = options.controller.start(staged, { request, traceStore: context.traceStore, signal: context.signal, deadlineAtMs: context.deadlineAtMs, }) + if ( + trial.identity.executionId !== request.executionId || + trial.identity.executionPlanDigest !== request.executionPlan.value.digest + ) { + assertTerminationAcknowledged(await trial.terminateAndWait()) + throw new Error('Pier controller returned a different durable trial identity') + } const active = { handle: trial } as { readonly handle: PierCandidateTrialHandle result?: PierCandidateTrialResult @@ -525,8 +568,13 @@ export async function executePreparedPierCandidate( stopAndCapture: async (request) => { const identity = trialIdentity(request.executionId, request.executionPlanDigest) const active = trials.get(identity) + assertTerminationAcknowledged( + await options.controller.terminateAndWait({ + executionId: request.executionId, + executionPlanDigest: request.executionPlanDigest, + }), + ) if (!active) return { stopped: true } - assertTerminationAcknowledged(await active.handle.terminateAndWait()) let result = active.result if (!result) { try { diff --git a/bench/src/pier-trial-controller.test.mts b/bench/src/pier-trial-controller.test.mts new file mode 100644 index 00000000..e3bf2525 --- /dev/null +++ b/bench/src/pier-trial-controller.test.mts @@ -0,0 +1,181 @@ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { chmod, mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { promisify } from 'node:util' + +import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' +import { InMemoryTraceStore } from '@tangle-network/agent-eval' + +import type { StagedPierCandidateExecution } from './pier-agent' +import { FilePierCandidateTrialController } from './pier-trial-controller' + +const execFileAsync = promisify(execFile) + +function testRequest(executionId: string, executionPlanDigest: `sha256:${string}`) { + return { + staged: { executionId, evaluatorEnv: {} } as StagedPierCandidateExecution, + context: { + request: { + executionId, + executionPlan: { value: { digest: executionPlanDigest } }, + } as AgentCandidateExecutorRequest, + traceStore: new InMemoryTraceStore(), + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 30_000, + }, + } +} + +test('an existing Pier job is rejected without deleting or starting it', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-scope-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const jobName = 'already-owned' + const marker = path.join(jobsDirectory, jobName, 'keep') + const fakeDocker = path.join(root, 'docker') + try { + await mkdir(path.dirname(marker), { recursive: true }) + await writeFile(marker, 'do not delete\n') + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: { ...process.env }, + jobsDirectory, + jobName, + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('existing job must never launch') + }, + }), + }) + const { staged, context } = testRequest( + 'pier-job-scope', + `sha256:${'c'.repeat(64)}`, + ) + assert.throws( + () => controller.start(staged, context), + /job name must be unique/, + ) + assert.equal(await readFile(marker, 'utf8'), 'do not delete\n') + assert.deepEqual(await readdir(controlRoot), []) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +test('a fresh evaluator process terminates the persisted process and container identity', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-recovery-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const jobName = 'recovery-job' + const trialName = 'trial-recovery' + const fakeContainerState = path.join(root, 'fake-container-live') + const fakeDocker = path.join(root, 'docker') + const childPidPath = path.join(root, 'child.pid') + const executionId = 'pier-fresh-process-recovery' + const executionPlanDigest = `sha256:${'d'.repeat(64)}` as const + let handle: ReturnType | undefined + let resultSettled: Promise | undefined + + try { + await writeFile(fakeContainerState, 'live\n') + await writeFile( + fakeDocker, + `#!/bin/sh +set -eu +state=${JSON.stringify(fakeContainerState)} +if test "\${1:-}" = ps; then + test -f "$state" && printf 'fake-container\\t${trialName}\\n' + exit 0 +fi +if test "\${1:-}" = rm; then + rm -f -- "$state" + exit 0 +fi +printf 'unexpected fake docker invocation: %s\\n' "$*" >&2 +exit 2 +`, + { mode: 0o700 }, + ) + await chmod(fakeDocker, 0o700) + + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + launch: () => ({ + command: process.execPath, + args: [ + '-e', + `const { spawn } = require('node:child_process'); const { mkdirSync, writeFileSync } = require('node:fs'); mkdirSync(${JSON.stringify(path.join(jobsDirectory, jobName, trialName))}, { recursive: true }); const child = spawn(process.execPath, ['-e', 'setInterval(() => undefined, 1_000)'], { stdio: 'ignore' }); writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid)); setInterval(() => undefined, 1_000)`, + ], + cwd: root, + env: { ...process.env }, + jobsDirectory, + jobName, + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('recovery trial must not complete normally') + }, + }), + }) + const { staged, context } = testRequest(executionId, executionPlanDigest) + handle = controller.start(staged, context) + resultSettled = handle.result.catch(() => undefined) + + const [slot] = await readdir(controlRoot) + assert.ok(slot) + const identityPath = path.join(controlRoot, slot, 'identity.json') + let persisted: { state?: string; pier?: { pid: number } } = {} + for (let attempt = 0; attempt < 200; attempt++) { + persisted = JSON.parse(await readFile(identityPath, 'utf8')) + if (persisted.state === 'running' && persisted.pier) break + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + assert.equal(persisted.state, 'running') + assert.ok(persisted.pier) + let childPid: number | undefined + for (let attempt = 0; attempt < 200; attempt++) { + try { + childPid = Number(await readFile(childPidPath, 'utf8')) + if (Number.isSafeInteger(childPid)) break + } catch {} + await new Promise((resolveWait) => setTimeout(resolveWait, 10)) + } + assert.ok(childPid) + + const recoveryScript = path.resolve('scripts/terminate-pier-trial.mts') + const recoveryArgs = [ + '--import', + 'tsx', + recoveryScript, + controlRoot, + executionId, + executionPlanDigest, + ] + const recovered = await Promise.all([ + execFileAsync(process.execPath, recoveryArgs, { cwd: path.resolve('.'), timeout: 20_000 }), + execFileAsync(process.execPath, recoveryArgs, { cwd: path.resolve('.'), timeout: 20_000 }), + ]) + for (const worker of recovered) { + assert.deepEqual(JSON.parse(worker.stdout), { + processExited: true, + containersRemoved: true, + }) + } + await assert.rejects(readFile(fakeContainerState), /ENOENT/) + assert.throws(() => process.kill(persisted.pier!.pid, 0), /ESRCH/) + assert.throws(() => process.kill(childPid!, 0), /ESRCH/) + await resultSettled + } finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + if (resultSettled) await resultSettled + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/bench/src/pier-trial-controller.ts b/bench/src/pier-trial-controller.ts new file mode 100644 index 00000000..d7696a9b --- /dev/null +++ b/bench/src/pier-trial-controller.ts @@ -0,0 +1,702 @@ +/** Restart-safe evaluator ownership for one Pier process and its Docker projects. */ +import { execFileSync, spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + accessSync, + closeSync, + constants, + existsSync, + fsyncSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { isAbsolute, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { Writable } from 'node:stream' + +import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' +import type { TraceStore } from '@tangle-network/agent-eval' + +import type { + PierCandidateTerminationAcknowledgement, + PierCandidateTrialController, + PierCandidateTrialHandle, + PierCandidateTrialIdentity, + PierCandidateTrialResult, + StagedPierCandidateExecution, +} from './pier-agent' + +const identityFile = 'identity.json' +const terminalFile = 'terminal.json' +const stopFile = 'stop-requested' +const sha256Pattern = /^sha256:[a-f0-9]{64}$/ + +interface ProcessIdentity { + readonly pid: number + readonly processGroupId: number + readonly sessionId: number + readonly startTicks: string +} + +interface PersistedTrialIdentity { + readonly schemaVersion: 1 + readonly kind: 'pier-trial-identity' + readonly state: 'allocating' | 'starting' | 'running' + readonly executionId: string + readonly executionPlanDigest: `sha256:${string}` + readonly supervisor?: ProcessIdentity + readonly pier?: ProcessIdentity + readonly jobsDirectory: string + readonly jobName: string + readonly dockerCommand: string +} + +interface PersistedTrialTerminal { + readonly schemaVersion: 1 + readonly kind: 'pier-trial-terminal' + readonly status: 'completed' | 'stopped' | 'failed' + readonly processExited: boolean + readonly containersRemoved: boolean + readonly removedContainers?: number + readonly exitCode?: number | null + readonly signal?: NodeJS.Signals | null + readonly error?: string +} + +export interface PierCandidateProcessSpec { + /** Returns launch data only. The controller, not this callback, starts the process. */ + readonly command: string + readonly args: readonly string[] + readonly cwd: string + /** Complete evaluator-owned environment. It is sent over a pipe and never persisted. */ + readonly env: Readonly> + readonly jobsDirectory: string + /** Must be unique: the controller atomically reserves this Pier job directory. */ + readonly jobName: string + readonly dockerCommand?: string + /** Called only by the originating process after the supervisor reports clean exit. */ + readonly readResult: () => Promise +} + +export interface FilePierCandidateTrialControllerOptions { + /** Shared evaluator-owned control root, available to crash-recovery workers. */ + readonly directory: string + readonly launch?: ( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ) => PierCandidateProcessSpec + readonly supervisorPath?: string + readonly pollIntervalMs?: number +} + +function nonEmpty(value: string, label: string): string { + if (!value || value.includes('\0')) throw new Error(`${label} must be non-empty without NUL`) + return value +} + +function absolutePath(value: string, label: string): string { + if (!isAbsolute(value)) throw new Error(`${label} must be an absolute path`) + return resolve(value) +} + +function resolveExecutable(command: string, searchPath: string | undefined, label: string): string { + nonEmpty(command, label) + const candidates = command.includes('/') + ? [absolutePath(command, label)] + : (searchPath ?? '') + .split(':') + .filter(Boolean) + .map((directory) => join(directory, command)) + for (const candidate of candidates) { + try { + accessSync(candidate, constants.X_OK) + return realpathSync(candidate) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT' && (error as NodeJS.ErrnoException).code !== 'EACCES') { + throw error + } + } + } + throw new Error(`${label} is not executable on the evaluator PATH`) +} + +function syncDirectory(directory: string): void { + const fd = openSync(directory, 'r') + try { + fsyncSync(fd) + } finally { + closeSync(fd) + } +} + +function writeJsonAtomic(directory: string, name: string, value: unknown): void { + const target = join(directory, name) + const temporary = join(directory, `.${name}.${process.pid}.${Date.now()}.tmp`) + const fd = openSync(temporary, 'wx', 0o600) + try { + writeFileSync(fd, `${JSON.stringify(value)}\n`, 'utf8') + fsyncSync(fd) + } finally { + closeSync(fd) + } + renameSync(temporary, target) + syncDirectory(directory) +} + +function touchDurable(directory: string, name: string): void { + try { + const fd = openSync(join(directory, name), 'wx', 0o600) + try { + fsyncSync(fd) + } finally { + closeSync(fd) + } + syncDirectory(directory) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + } +} + +function readJson(path: string, label: string): Record { + let value: unknown + try { + value = JSON.parse(readFileSync(path, 'utf8')) + } catch (error) { + throw new Error(`${label} is not valid JSON`, { cause: error }) + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + return value as Record +} + +function parseProcessIdentity(value: unknown, label: string): ProcessIdentity { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`) + } + const record = value as Record + if ( + !Number.isSafeInteger(record.pid) || + !Number.isSafeInteger(record.processGroupId) || + !Number.isSafeInteger(record.sessionId) || + typeof record.startTicks !== 'string' || + !/^\d+$/.test(record.startTicks) + ) { + throw new Error(`${label} is malformed`) + } + return { + pid: record.pid as number, + processGroupId: record.processGroupId as number, + sessionId: record.sessionId as number, + startTicks: record.startTicks, + } +} + +function parsePersistedIdentity(path: string): PersistedTrialIdentity { + const record = readJson(path, 'Pier trial identity') + if ( + record.schemaVersion !== 1 || + record.kind !== 'pier-trial-identity' || + !['allocating', 'starting', 'running'].includes(record.state as string) || + typeof record.executionId !== 'string' || + typeof record.executionPlanDigest !== 'string' || + !sha256Pattern.test(record.executionPlanDigest) || + typeof record.jobsDirectory !== 'string' || + typeof record.jobName !== 'string' || + typeof record.dockerCommand !== 'string' + ) { + throw new Error('Pier trial identity is malformed') + } + return { + schemaVersion: 1, + kind: 'pier-trial-identity', + state: record.state as PersistedTrialIdentity['state'], + executionId: record.executionId, + executionPlanDigest: record.executionPlanDigest as `sha256:${string}`, + ...(record.supervisor + ? { supervisor: parseProcessIdentity(record.supervisor, 'Pier supervisor identity') } + : {}), + ...(record.pier ? { pier: parseProcessIdentity(record.pier, 'Pier process identity') } : {}), + jobsDirectory: absolutePath(record.jobsDirectory, 'persisted jobs directory'), + jobName: nonEmpty(record.jobName, 'persisted job name'), + dockerCommand: nonEmpty(record.dockerCommand, 'persisted Docker command'), + } +} + +function parseTerminal(path: string): PersistedTrialTerminal { + const record = readJson(path, 'Pier trial terminal') + if ( + record.schemaVersion !== 1 || + record.kind !== 'pier-trial-terminal' || + !['completed', 'stopped', 'failed'].includes(record.status as string) || + typeof record.processExited !== 'boolean' || + typeof record.containersRemoved !== 'boolean' + ) { + throw new Error('Pier trial terminal is malformed') + } + return record as unknown as PersistedTrialTerminal +} + +function processIdentity(pid: number): ProcessIdentity { + const text = readFileSync(`/proc/${pid}/stat`, 'utf8') + const closing = text.lastIndexOf(')') + if (closing < 0) throw new Error(`cannot parse process identity for ${pid}`) + const fields = text.slice(closing + 2).trim().split(/\s+/) + const processGroupId = Number(fields[2]) + const sessionId = Number(fields[3]) + const startTicks = fields[19] + if ( + !Number.isSafeInteger(processGroupId) || + !Number.isSafeInteger(sessionId) || + !/^\d+$/.test(startTicks ?? '') + ) { + throw new Error(`cannot parse process identity for ${pid}`) + } + return { pid, processGroupId, sessionId, startTicks } +} + +function processMatches(identity: ProcessIdentity): boolean { + try { + const current = processIdentity(identity.pid) + return ( + current.startTicks === identity.startTicks && + current.processGroupId === identity.processGroupId && + current.sessionId === identity.sessionId + ) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false + throw error + } +} + +function signalProcess(identity: ProcessIdentity, signal: NodeJS.Signals, group: boolean): void { + if (!processMatches(identity)) return + try { + process.kill(group ? -identity.processGroupId : identity.pid, signal) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } +} + +function sessionMembers(sessionId: number): ProcessIdentity[] { + const members: ProcessIdentity[] = [] + for (const entry of readdirSync('/proc', { withFileTypes: true })) { + if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue + try { + const identity = processIdentity(Number(entry.name)) + if (identity.sessionId === sessionId) members.push(identity) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + } + return members +} + +async function signalSessionUntilEmpty( + identity: ProcessIdentity, + signal: NodeJS.Signals, + timeoutMs: number, + intervalMs: number, +): Promise { + const deadline = Date.now() + timeoutMs + while (true) { + const members = sessionMembers(identity.sessionId) + if (members.length === 0) return true + for (const member of members) signalProcess(member, signal, false) + if (Date.now() >= deadline) return false + await sleep(intervalMs) + } +} + +function sanitizeProject(value: string): string { + let project = value.toLowerCase() + if (!/^[a-z0-9]/.test(project)) project = `0${project}` + return project.replace(/[^a-z0-9_-]/g, '-') +} + +function trialProjects(identity: PersistedTrialIdentity): string[] { + const jobRoot = join(identity.jobsDirectory, identity.jobName) + if (!existsSync(jobRoot)) return [] + return readdirSync(jobRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => sanitizeProject(entry.name)) +} + +function matchingContainers(identity: PersistedTrialIdentity): string[] { + const projects = trialProjects(identity) + if (projects.length === 0) return [] + const output = execFileSync( + identity.dockerCommand, + ['ps', '-a', '--format', '{{.ID}}\t{{.Label "com.docker.compose.project"}}'], + { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }, + ) + return output + .split('\n') + .filter(Boolean) + .flatMap((line) => { + const tab = line.indexOf('\t') + if (tab < 1) return [] + const id = line.slice(0, tab) + const project = line.slice(tab + 1) + return projects.some( + (expected) => + project === expected || project.startsWith(`${expected}__verifier__`), + ) + ? [id] + : [] + }) +} + +function removeTrialContainers(identity: PersistedTrialIdentity): number { + const containers = matchingContainers(identity) + if (containers.length > 0) { + try { + execFileSync(identity.dockerCommand, ['rm', '-f', ...containers], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }) + } catch (error) { + if (matchingContainers(identity).length > 0) throw error + } + } + const remaining = matchingContainers(identity) + if (remaining.length > 0) { + throw new Error(`Pier task containers survived removal: ${remaining.join(', ')}`) + } + return containers.length +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, milliseconds)) +} + +async function waitUntil(predicate: () => boolean, timeoutMs: number, intervalMs: number) { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() >= deadline) return false + await sleep(intervalMs) + } + return true +} + +function trialKey(identity: PierCandidateTrialIdentity): string { + if (!sha256Pattern.test(identity.executionPlanDigest)) { + throw new Error('executionPlanDigest must be a SHA-256 digest') + } + return createHash('sha256') + .update(JSON.stringify([nonEmpty(identity.executionId, 'executionId'), identity.executionPlanDigest])) + .digest('hex') +} + +function assertRealDirectory(path: string, label: string): void { + const stats = lstatSync(path) + if (!stats.isDirectory() || stats.isSymbolicLink() || realpathSync(path) !== path) { + throw new Error(`${label} must be a real canonical directory`) + } +} + +export class FilePierCandidateTrialController implements PierCandidateTrialController { + private readonly directory: string + private readonly launch?: NonNullable + private readonly supervisorPath: string + private readonly pollIntervalMs: number + + constructor(options: FilePierCandidateTrialControllerOptions) { + this.directory = absolutePath(options.directory, 'Pier controller directory') + this.launch = options.launch + this.supervisorPath = absolutePath( + options.supervisorPath ?? + fileURLToPath(new URL('./pier-trial-supervisor.mjs', import.meta.url)), + 'Pier supervisor path', + ) + this.pollIntervalMs = options.pollIntervalMs ?? 25 + if (!Number.isSafeInteger(this.pollIntervalMs) || this.pollIntervalMs < 1) { + throw new Error('pollIntervalMs must be a positive integer') + } + mkdirSync(this.directory, { recursive: true, mode: 0o700 }) + assertRealDirectory(this.directory, 'Pier controller directory') + } + + start( + staged: StagedPierCandidateExecution, + context: { + readonly request: AgentCandidateExecutorRequest + readonly traceStore: TraceStore + readonly signal: AbortSignal + readonly deadlineAtMs: number + }, + ): PierCandidateTrialHandle { + if (!this.launch) throw new Error('this Pier controller is recovery-only') + const identity: PierCandidateTrialIdentity = { + executionId: context.request.executionId, + executionPlanDigest: context.request.executionPlan.value.digest, + } + if (staged.executionId !== identity.executionId) { + throw new Error('staged Pier execution does not match the runtime request') + } + + // The launch callback is a pure spec builder. Resolve every fallible input + // before reserving durable identities or starting a process. + const spec = this.launch(staged, context) + this.validateSpec(spec) + mkdirSync(spec.jobsDirectory, { recursive: true, mode: 0o700 }) + assertRealDirectory(spec.jobsDirectory, 'Pier jobs directory') + const dockerCommand = resolveExecutable( + spec.dockerCommand ?? 'docker', + process.env.PATH, + 'Docker command', + ) + const pierCommand = resolveExecutable(spec.command, spec.env.PATH, 'Pier command') + + const controlDirectory = this.controlDirectory(identity) + try { + mkdirSync(controlDirectory, { mode: 0o700 }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new Error('Pier trial identity already has durable control state') + } + throw error + } + assertRealDirectory(controlDirectory, 'Pier trial control directory') + + // Pier creates each trial directory before starting its environment. An + // atomically reserved, one-execution job directory therefore makes every + // Compose project discovered below exclusive to this execution. + const jobRoot = join(spec.jobsDirectory, spec.jobName) + let jobRootCreated = false + try { + mkdirSync(jobRoot, { mode: 0o700 }) + jobRootCreated = true + assertRealDirectory(jobRoot, 'Pier job directory') + } catch (error) { + // No identity or process exists yet; leave the execution reusable. + rmSync(controlDirectory, { recursive: true, force: true }) + if (jobRootCreated) rmSync(jobRoot, { recursive: true, force: true }) + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new Error('Pier job name must be unique; its job directory already exists') + } + throw error + } + const allocating: PersistedTrialIdentity = { + schemaVersion: 1, + kind: 'pier-trial-identity', + state: 'allocating', + ...identity, + jobsDirectory: absolutePath(spec.jobsDirectory, 'Pier jobs directory'), + jobName: nonEmpty(spec.jobName, 'Pier job name'), + dockerCommand: nonEmpty(dockerCommand, 'Docker command'), + } + try { + writeJsonAtomic(controlDirectory, identityFile, allocating) + } catch (error) { + // The supervisor can only start after this durable write succeeds. + rmSync(controlDirectory, { recursive: true, force: true }) + rmSync(jobRoot, { recursive: true, force: true }) + throw error + } + + const supervisor = spawn(process.execPath, [this.supervisorPath, controlDirectory], { + detached: true, + stdio: ['ignore', 'ignore', 'ignore', 'pipe'], + }) + if (supervisor.pid === undefined) throw new Error('Pier supervisor started without a pid') + const controlPipe = supervisor.stdio[3] as (Writable & { unref?: () => void }) | null + let supervisorIdentity: ProcessIdentity | undefined + try { + supervisorIdentity = processIdentity(supervisor.pid) + const persisted: PersistedTrialIdentity = { + ...allocating, + state: 'starting', + supervisor: supervisorIdentity, + } + writeJsonAtomic(controlDirectory, identityFile, persisted) + if (!controlPipe) throw new Error('Pier supervisor has no protected control pipe') + controlPipe.on('error', () => undefined) + controlPipe.end( + JSON.stringify({ + command: pierCommand, + args: spec.args, + cwd: spec.cwd, + env: spec.env, + jobsDirectory: spec.jobsDirectory, + jobName: spec.jobName, + dockerCommand, + }), + ) + } catch (error) { + controlPipe?.destroy() + if (supervisorIdentity) signalProcess(supervisorIdentity, 'SIGKILL', false) + else { + try { + process.kill(supervisor.pid, 'SIGKILL') + } catch (killError) { + if ((killError as NodeJS.ErrnoException).code !== 'ESRCH') throw killError + } + } + throw error + } + supervisor.unref() + controlPipe.unref?.() + + const result = this.waitForResult(controlDirectory, spec, context.deadlineAtMs) + return { + identity, + result, + terminateAndWait: async () => await this.terminateAndWait(identity), + } + } + + async terminateAndWait( + requested: PierCandidateTrialIdentity, + ): Promise { + const controlDirectory = this.controlDirectory(requested) + assertRealDirectory(controlDirectory, 'Pier trial control directory') + const persisted = parsePersistedIdentity(join(controlDirectory, identityFile)) + if ( + persisted.executionId !== requested.executionId || + persisted.executionPlanDigest !== requested.executionPlanDigest + ) { + throw new Error('persisted Pier trial identity differs from the stop request') + } + + touchDurable(controlDirectory, stopFile) + if (!existsSync(join(controlDirectory, terminalFile))) { + if (persisted.supervisor) signalProcess(persisted.supervisor, 'SIGTERM', false) + } + const terminalAppeared = await waitUntil( + () => existsSync(join(controlDirectory, terminalFile)), + persisted.supervisor ? 10_000 : 1, + this.pollIntervalMs, + ) + if (!terminalAppeared) await this.forceRecovery(controlDirectory) + + let terminal = parseTerminal(join(controlDirectory, terminalFile)) + const latest = parsePersistedIdentity(join(controlDirectory, identityFile)) + const liveProcesses = latest.pier ? sessionMembers(latest.pier.sessionId).length : 0 + const liveContainers = matchingContainers(latest).length + if (liveProcesses > 0 || liveContainers > 0) { + await this.forceRecovery(controlDirectory) + terminal = parseTerminal(join(controlDirectory, terminalFile)) + } + if (!terminal.processExited || !terminal.containersRemoved) { + throw new Error( + `Pier recovery did not prove process/container death${terminal.error ? `: ${terminal.error}` : ''}`, + ) + } + return { processExited: true, containersRemoved: true } + } + + private controlDirectory(identity: PierCandidateTrialIdentity): string { + return join(this.directory, trialKey(identity)) + } + + private validateSpec(spec: PierCandidateProcessSpec): void { + nonEmpty(spec.command, 'Pier command') + if (!Array.isArray(spec.args) || spec.args.some((arg) => typeof arg !== 'string' || arg.includes('\0'))) { + throw new Error('Pier arguments must be strings without NUL') + } + absolutePath(spec.cwd, 'Pier cwd') + absolutePath(spec.jobsDirectory, 'Pier jobs directory') + if (!/^[A-Za-z0-9._-]{1,200}$/.test(spec.jobName)) { + throw new Error('Pier job name must be filesystem-neutral') + } + for (const [name, value] of Object.entries(spec.env)) { + if (!name || name.includes('\0') || value?.includes('\0')) { + throw new Error('Pier environment contains an invalid name or value') + } + } + } + + private async waitForResult( + controlDirectory: string, + spec: PierCandidateProcessSpec, + deadlineAtMs: number, + ): Promise { + const waitMs = Math.max(1, deadlineAtMs - Date.now() + 15_000) + const appeared = await waitUntil( + () => existsSync(join(controlDirectory, terminalFile)), + waitMs, + this.pollIntervalMs, + ) + if (!appeared) throw new Error('Pier supervisor emitted no terminal acknowledgement') + const terminal = parseTerminal(join(controlDirectory, terminalFile)) + if (!terminal.processExited || !terminal.containersRemoved) { + throw new Error('Pier supervisor did not prove process and container death') + } + if (terminal.status !== 'completed') { + throw new Error(terminal.error ? `Pier process failed: ${terminal.error}` : 'Pier process stopped') + } + return await spec.readResult() + } + + private async forceRecovery( + controlDirectory: string, + ): Promise { + const latest = parsePersistedIdentity(join(controlDirectory, identityFile)) + const errors: string[] = [] + if (latest.pier) { + try { + await signalSessionUntilEmpty(latest.pier, 'SIGTERM', 2_000, this.pollIntervalMs) + } catch (error) { + errors.push(`could not signal Pier session: ${error instanceof Error ? error.message : String(error)}`) + } + } + if (latest.supervisor) { + try { + signalProcess(latest.supervisor, 'SIGKILL', false) + if (!(await waitUntil(() => !processMatches(latest.supervisor!), 5_000, this.pollIntervalMs))) { + errors.push(`Pier supervisor ${latest.supervisor.pid} survived SIGKILL`) + } + } catch (error) { + errors.push(`could not kill Pier supervisor: ${error instanceof Error ? error.message : String(error)}`) + } + } + if (latest.pier) { + try { + if ( + !(await signalSessionUntilEmpty( + latest.pier, + 'SIGKILL', + 5_000, + this.pollIntervalMs, + )) + ) { + errors.push(`Pier process session ${latest.pier.sessionId} survived SIGKILL`) + } + } catch (error) { + errors.push(`could not kill Pier session: ${error instanceof Error ? error.message : String(error)}`) + } + } + let removedContainers = 0 + try { + removedContainers = removeTrialContainers(latest) + } catch (error) { + errors.push(`could not remove Pier containers: ${error instanceof Error ? error.message : String(error)}`) + } + if (errors.length > 0) { + throw new Error(`Pier recovery incomplete: ${errors.join('; ')}`) + } + writeJsonAtomic(controlDirectory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'stopped', + processExited: true, + containersRemoved: true, + removedContainers, + recoveredByPid: process.pid, + }) + } +} diff --git a/bench/src/pier-trial-supervisor.mjs b/bench/src/pier-trial-supervisor.mjs new file mode 100644 index 00000000..349b15ed --- /dev/null +++ b/bench/src/pier-trial-supervisor.mjs @@ -0,0 +1,352 @@ +import { execFileSync, spawn } from 'node:child_process' +import { + closeSync, + createReadStream, + existsSync, + fsyncSync, + openSync, + readFileSync, + readdirSync, + renameSync, + writeFileSync, +} from 'node:fs' +import { basename, isAbsolute, join, resolve } from 'node:path' + +const identityFile = 'identity.json' +const terminalFile = 'terminal.json' +const stopFile = 'stop-requested' + +function fail(message) { + throw new Error(message) +} + +function readJson(path, label) { + let value + try { + value = JSON.parse(readFileSync(path, 'utf8')) + } catch (error) { + throw new Error(`${label} is not valid JSON`, { cause: error }) + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + fail(`${label} must be an object`) + } + return value +} + +function syncDirectory(directory) { + const fd = openSync(directory, 'r') + try { + fsyncSync(fd) + } finally { + closeSync(fd) + } +} + +function writeJsonAtomic(directory, name, value) { + const target = join(directory, name) + const temporary = join(directory, `.${name}.${process.pid}.tmp`) + const fd = openSync(temporary, 'wx', 0o600) + try { + writeFileSync(fd, `${JSON.stringify(value)}\n`, 'utf8') + fsyncSync(fd) + } finally { + closeSync(fd) + } + renameSync(temporary, target) + syncDirectory(directory) +} + +async function readControlInput() { + const chunks = [] + for await (const chunk of createReadStream('', { fd: 3 })) { + chunks.push(Buffer.from(chunk)) + } + const bytes = Buffer.concat(chunks) + if (bytes.byteLength === 0) fail('Pier supervisor received no launch request') + return JSON.parse(bytes.toString('utf8')) +} + +function processIdentity(pid) { + const text = readFileSync(`/proc/${pid}/stat`, 'utf8') + const closing = text.lastIndexOf(')') + if (closing < 0) fail(`cannot parse process identity for ${pid}`) + const fields = text.slice(closing + 2).trim().split(/\s+/) + const processGroupId = Number(fields[2]) + const sessionId = Number(fields[3]) + const startTicks = fields[19] + if ( + !Number.isSafeInteger(processGroupId) || + !Number.isSafeInteger(sessionId) || + !/^\d+$/.test(startTicks ?? '') + ) { + fail(`cannot parse process identity for ${pid}`) + } + return { pid, processGroupId, sessionId, startTicks } +} + +function processMatches(identity) { + try { + const current = processIdentity(identity.pid) + return ( + current.startTicks === identity.startTicks && + current.processGroupId === identity.processGroupId && + current.sessionId === identity.sessionId + ) + } catch (error) { + if (error?.code === 'ENOENT' || error?.code === 'ESRCH') return false + throw error + } +} + +function sessionMembers(sessionId) { + const members = [] + for (const entry of readdirSync('/proc', { withFileTypes: true })) { + if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue + try { + const identity = processIdentity(Number(entry.name)) + if (identity.sessionId === sessionId) members.push(identity) + } catch (error) { + if (error?.code !== 'ENOENT') throw error + } + } + return members +} + +async function signalSessionUntilEmpty(identity, signal, timeoutMs) { + const deadline = Date.now() + timeoutMs + while (true) { + const members = sessionMembers(identity.sessionId) + if (members.length === 0) return true + for (const member of members) { + try { + process.kill(member.pid, signal) + } catch (error) { + if (error?.code !== 'ESRCH') throw error + } + } + if (Date.now() >= deadline) return false + await new Promise((resolveWait) => setTimeout(resolveWait, 20)) + } +} + +function sanitizeProject(value) { + let project = value.toLowerCase() + if (!/^[a-z0-9]/.test(project)) project = `0${project}` + return project.replace(/[^a-z0-9_-]/g, '-') +} + +function trialProjects(jobsDirectory, jobName) { + const jobRoot = join(jobsDirectory, jobName) + if (!existsSync(jobRoot)) return [] + return readdirSync(jobRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => sanitizeProject(entry.name)) +} + +function matchingContainers(dockerCommand, projects) { + if (projects.length === 0) return [] + const output = execFileSync( + dockerCommand, + ['ps', '-a', '--format', '{{.ID}}\t{{.Label "com.docker.compose.project"}}'], + { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }, + ) + const matches = [] + for (const line of output.split('\n')) { + if (!line) continue + const tab = line.indexOf('\t') + if (tab < 1) continue + const id = line.slice(0, tab) + const project = line.slice(tab + 1) + if ( + projects.some( + (expected) => + project === expected || project.startsWith(`${expected}__verifier__`), + ) + ) { + matches.push(id) + } + } + return matches +} + +function removeTrialContainers(config) { + const projects = trialProjects(config.jobsDirectory, config.jobName) + const containers = matchingContainers(config.dockerCommand, projects) + if (containers.length > 0) { + try { + execFileSync(config.dockerCommand, ['rm', '-f', ...containers], { + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }) + } catch (error) { + if (matchingContainers(config.dockerCommand, projects).length > 0) throw error + } + } + const remaining = matchingContainers(config.dockerCommand, projects) + if (remaining.length > 0) { + fail(`Pier task containers survived removal: ${remaining.join(', ')}`) + } + return containers.length +} + +async function waitForExit(exited, timeoutMs) { + return await Promise.race([ + exited.then(() => true), + new Promise((resolveTimeout) => setTimeout(() => resolveTimeout(false), timeoutMs)), + ]) +} + +const directory = resolve(process.argv[2] ?? '') +if (!isAbsolute(directory) || basename(directory) === '') { + fail('Pier supervisor control directory must be absolute') +} + +let config +let child +let childIdentity +let stopRequested = existsSync(join(directory, stopFile)) +let stopping + +async function stopChild() { + if (!child) return + if (!childIdentity) { + try { + child.process.kill('SIGKILL') + } catch (error) { + if (error?.code !== 'ESRCH') throw error + } + if (!(await waitForExit(child.exited, 5_000))) { + fail('Pier process with unreadable identity survived SIGKILL') + } + return + } + if (!(await signalSessionUntilEmpty(childIdentity, 'SIGTERM', 2_000))) { + if (!(await signalSessionUntilEmpty(childIdentity, 'SIGKILL', 5_000))) { + fail(`Pier process session ${childIdentity.sessionId} survived SIGKILL`) + } + } + if (!(await waitForExit(child.exited, 5_000))) { + fail(`Pier process session ${childIdentity.sessionId} retained child processes`) + } +} + +function requestStop() { + stopRequested = true + stopping ??= stopChild() +} + +process.on('SIGTERM', requestStop) +process.on('SIGINT', requestStop) + +try { + config = await readControlInput() + if ( + !config || + typeof config !== 'object' || + typeof config.command !== 'string' || + !Array.isArray(config.args) || + typeof config.cwd !== 'string' || + !config.env || + typeof config.env !== 'object' || + typeof config.jobsDirectory !== 'string' || + typeof config.jobName !== 'string' || + typeof config.dockerCommand !== 'string' + ) { + fail('Pier supervisor launch request is malformed') + } + + const identity = readJson(join(directory, identityFile), 'Pier trial identity') + if (stopRequested || existsSync(join(directory, stopFile))) { + const removedContainers = removeTrialContainers(config) + writeJsonAtomic(directory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'stopped', + processExited: true, + containersRemoved: true, + removedContainers, + }) + process.exitCode = 0 + } else { + const spawned = spawn(config.command, config.args, { + cwd: config.cwd, + env: config.env, + detached: true, + stdio: 'ignore', + }) + if (spawned.pid === undefined) fail('Pier process started without a pid') + const exited = new Promise((resolveExit) => { + let settled = false + spawned.once('error', (error) => { + if (settled) return + settled = true + resolveExit({ code: null, signal: null, error }) + }) + spawned.once('close', (code, signal) => { + if (settled) return + settled = true + resolveExit({ code, signal }) + }) + }) + child = { process: spawned, exited } + childIdentity = processIdentity(spawned.pid) + writeJsonAtomic(directory, identityFile, { + ...identity, + state: 'running', + pier: childIdentity, + }) + + if (stopRequested || existsSync(join(directory, stopFile))) { + stopRequested = true + stopping = stopChild() + } + const exit = await exited + if (stopping) await stopping + else await stopChild() + const removedContainers = removeTrialContainers(config) + const status = stopRequested ? 'stopped' : exit.code === 0 ? 'completed' : 'failed' + writeJsonAtomic(directory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status, + processExited: true, + containersRemoved: true, + removedContainers, + exitCode: exit.code, + signal: exit.signal, + ...(exit.error ? { error: 'Pier process failed to start' } : {}), + ...(status === 'failed' && !exit.error + ? { error: `Pier process exited ${exit.signal ?? exit.code ?? 'without status'}` } + : {}), + }) + } +} catch (error) { + const cleanupErrors = [] + try { + requestStop() + if (stopping) await stopping + } catch (cleanup) { + cleanupErrors.push(cleanup) + } + if (config) { + try { + removeTrialContainers(config) + } catch (cleanup) { + cleanupErrors.push(cleanup) + } + } + writeJsonAtomic(directory, terminalFile, { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'failed', + processExited: !childIdentity || !processMatches(childIdentity), + containersRemoved: cleanupErrors.length === 0, + error: `${error instanceof Error ? error.message : String(error)}${ + cleanupErrors.length > 0 + ? `; cleanup failed: ${cleanupErrors.map((item) => item?.message ?? String(item)).join('; ')}` + : '' + }`, + }) + process.exitCode = 1 +} From 3cc6622e9087548c453fec3b8037c114a3578628 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 10:57:13 -0600 Subject: [PATCH 3/3] fix(bench): harden Pier trial controls --- bench/HARNESS.md | 24 +- bench/package.json | 5 + bench/pier_agents/__init__.py | 15 +- bench/pier_agents/candidate_contract.py | 5 +- bench/pier_agents/tangle_candidate_test.py | 26 +++ bench/pnpm-lock.yaml | 32 +-- bench/scripts/terminate-pier-trial.mts | 29 +++ bench/scripts/verify-packed-consumer.mjs | 54 ++--- bench/scripts/verify-pier-agent.mts | 4 +- bench/scripts/verify-pier-recovery.mts | 4 +- bench/src/index.ts | 1 + bench/src/pier-agent.test-fixtures.mts | 19 ++ bench/src/pier-trial-controller.test.mts | 249 ++++++++++++++++++++- bench/src/pier-trial-controller.ts | 163 ++++++++++++-- 14 files changed, 526 insertions(+), 104 deletions(-) create mode 100644 bench/src/pier-agent.test-fixtures.mts diff --git a/bench/HARNESS.md b/bench/HARNESS.md index 848d64b4..b14364a1 100644 --- a/bench/HARNESS.md +++ b/bench/HARNESS.md @@ -238,21 +238,27 @@ That command runs a no-change candidate that must score 0/1, proves a fresh eval It then checks that each official result and exact task patch is bound into its own runtime receipt with zero model usage. For a real frozen candidate, use `FilePierCandidateTrialController`, append `agentArgs` and `attemptArgs`, and pass each executor-only `evaluatorEnv` entry through Pier's evaluator-owned environment mechanism. +The launch callback's second argument carries the signed runtime `request`, protected `traceStore`, cancellation `signal`, and absolute `deadlineAtMs`; production launchers must use that context rather than reconstructing it. The controller sends secrets to its supervisor over a pipe, while its durable files contain only process and Docker-project identities: +The supervisor never inherits `process.env`; non-default Docker connections use a stable `dockerConnection.id` plus the exact environment injected into both Pier and cleanup. +Fresh recovery workers must reconstruct that same named connection; `terminate-pier-trial.mts` selects only the comma-delimited variables named by `PIER_DOCKER_ENV_NAMES` when `PIER_DOCKER_CONNECTION_ID` is set. `jobName` must be unique per prepared execution; the controller atomically reserves that job directory so recovery can remove only containers owned by that execution. ```ts const controller = new FilePierCandidateTrialController({ directory: '/var/lib/tangle/pier-control', - launch: (staged) => ({ - command: 'uv', - args: ['run', 'pier', 'run', ...staged.agentArgs, ...staged.attemptArgs], - cwd: pierCheckout, - env: { ...evaluatorEnvironment, ...staged.evaluatorEnv }, - jobsDirectory, - jobName, - readResult: () => readOfficialPierResult(jobsDirectory, jobName), - }), + launch: (staged, { request }) => { + const jobName = request.executionId + return { + command: 'uv', + args: ['run', 'pier', 'run', ...staged.agentArgs, ...staged.attemptArgs], + cwd: pierCheckout, + env: { ...evaluatorEnvironment, ...staged.evaluatorEnv }, + jobsDirectory, + jobName, + readResult: () => readOfficialPierResult(jobsDirectory, jobName), + } + }, }) const result = await executePreparedPierCandidate({ diff --git a/bench/package.json b/bench/package.json index 413cde88..47166c0d 100644 --- a/bench/package.json +++ b/bench/package.json @@ -32,6 +32,11 @@ "tsx": "^4.19.0", "typescript": "^6.0.3" }, + "pnpm": { + "overrides": { + "@tangle-network/agent-eval": "0.114.0" + } + }, "files": [ "src", "fixtures", diff --git a/bench/pier_agents/__init__.py b/bench/pier_agents/__init__.py index 6f65e392..a27997bc 100644 --- a/bench/pier_agents/__init__.py +++ b/bench/pier_agents/__init__.py @@ -1,5 +1,18 @@ """Pier custom agents shipped by ``@tangle-network/agent-bench``.""" -from .tangle_candidate import TangleCandidateAgent +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from .tangle_candidate import TangleCandidateAgent __all__ = ["TangleCandidateAgent"] + + +def __getattr__(name: str) -> Any: + """Load Pier only when callers request the custom agent.""" + if name != "TangleCandidateAgent": + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from .tangle_candidate import TangleCandidateAgent + + return TangleCandidateAgent diff --git a/bench/pier_agents/candidate_contract.py b/bench/pier_agents/candidate_contract.py index ce52275e..5d667163 100644 --- a/bench/pier_agents/candidate_contract.py +++ b/bench/pier_agents/candidate_contract.py @@ -323,11 +323,12 @@ def _instruction(value: Any) -> InstructionEvidence: env = path = None elif kind == "utf8-file": if ( - delivery.get("env") != "TANGLE_CANDIDATE_TASK_PATH" + set(delivery) != {"kind", "env", "path"} + or delivery.get("env") != "TANGLE_CANDIDATE_TASK_PATH" or delivery.get("path") != "/tangle/input/task.txt" ): raise CandidateContractError( - "utf8-file delivery must use the fixed env and path" + "utf8-file delivery has unexpected fields or does not use the fixed env and path" ) env = "TANGLE_CANDIDATE_TASK_PATH" path = "/tangle/input/task.txt" diff --git a/bench/pier_agents/tangle_candidate_test.py b/bench/pier_agents/tangle_candidate_test.py index 29af6759..0c8396e0 100644 --- a/bench/pier_agents/tangle_candidate_test.py +++ b/bench/pier_agents/tangle_candidate_test.py @@ -452,6 +452,32 @@ def test_loads_exact_runtime_artifacts_and_rejects_plan_substitution(self): fixture["receipt_digest"], ) + def test_rejects_extra_utf8_file_delivery_fields(self): + with tempfile.TemporaryDirectory() as directory: + fixture = _fixture(Path(directory)) + fixture["plan"]["task"]["instruction"]["delivery"] = { + "kind": "utf8-file", + "env": "TANGLE_CANDIDATE_TASK_PATH", + "path": "/tangle/input/task.txt", + "ignored": "unsigned-semantics", + } + fixture["plan"]["launch"]["env"] = { + "TANGLE_CANDIDATE_TASK_PATH": { + "kind": "public", + "value": "/tangle/input/task.txt", + } + } + _rewrite_signed_plan(fixture) + + with self.assertRaisesRegex( + contract_module.CandidateContractError, "unexpected fields" + ): + contract_module.load_prepared_candidate_contract( + fixture["plan_path"], + fixture["receipt_path"], + fixture["receipt_digest"], + ) + def test_accepts_only_frozen_public_model_gateway_domains(self): with tempfile.TemporaryDirectory() as directory: fixture = _fixture(Path(directory)) diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 45557ff1..2454aa2a 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -4,12 +4,15 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + '@tangle-network/agent-eval': 0.114.0 + importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.114.0 + specifier: 0.114.0 version: 0.114.0(typescript@6.0.3) '@tangle-network/agent-interface': specifier: ^0.25.0 @@ -260,11 +263,6 @@ packages: '@tangle-network/agent-core@0.4.9': resolution: {integrity: sha512-BFU4WV12Y6D28PX+o8LIVkIMD+cXwrL1uyV182l12Bn9zEu7VHt2oVMEEzbsN8L+X0xM9baEfMCHTFKFNJv7Aw==} - '@tangle-network/agent-eval@0.113.0': - resolution: {integrity: sha512-1KhW6l69arpanK8vamnvJueD+ZwEO4A7BMF5qpkPLXUVVNcrUail0GWp3M5SpAjQdACa3HTuVdexXuEpZt5sQg==} - engines: {node: '>=20'} - hasBin: true - '@tangle-network/agent-eval@0.114.0': resolution: {integrity: sha512-pIZCLkKsHeKDpLOpRXRhnc2HaREcHduHm4u3jw0nN73B1BWe6Jaqh2wlNHTPnPUA1qQl5sCl1K6XF1p9R7eTZw==} engines: {node: '>=20'} @@ -301,7 +299,7 @@ packages: engines: {node: '>=20'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.101.0 <1.0.0' + '@tangle-network/agent-eval': 0.114.0 '@tangle-network/agent-interface': '>=0.25.0 <1.0.0' '@tangle-network/sandbox': '>=0.8.0 <1.0.0' playwright: ^1.40.0 @@ -605,24 +603,6 @@ snapshots: '@tangle-network/agent-interface': 0.24.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.113.0(typescript@6.0.3)': - dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.4(hono@4.12.23) - '@tangle-network/agent-interface': 0.22.0 - '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) - hono: 4.12.23 - zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - '@tangle-network/agent-eval@0.114.0(typescript@6.0.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) @@ -667,7 +647,7 @@ snapshots: '@tangle-network/agent-knowledge@1.11.2(typescript@6.0.3)': dependencies: - '@tangle-network/agent-eval': 0.113.0(typescript@6.0.3) + '@tangle-network/agent-eval': 0.114.0(typescript@6.0.3) zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' diff --git a/bench/scripts/terminate-pier-trial.mts b/bench/scripts/terminate-pier-trial.mts index e9aec48d..4ea9d489 100644 --- a/bench/scripts/terminate-pier-trial.mts +++ b/bench/scripts/terminate-pier-trial.mts @@ -15,8 +15,37 @@ if (!/^sha256:[a-f0-9]{64}$/.test(executionPlanDigest)) { throw new Error('execution-plan-digest must be a SHA-256 digest') } +const dockerConnectionId = process.env.PIER_DOCKER_CONNECTION_ID +const dockerEnvironmentNames = process.env.PIER_DOCKER_ENV_NAMES +if ((dockerConnectionId === undefined) !== (dockerEnvironmentNames === undefined)) { + throw new Error( + 'PIER_DOCKER_CONNECTION_ID and PIER_DOCKER_ENV_NAMES must be set together', + ) +} +const dockerConnection = (() => { + if (dockerConnectionId === undefined || dockerEnvironmentNames === undefined) return undefined + const names = dockerEnvironmentNames + .split(',') + .map((name) => name.trim()) + .filter(Boolean) + if (new Set(names).size !== names.length) { + throw new Error('PIER_DOCKER_ENV_NAMES must not contain duplicates') + } + const env: Record = {} + for (const name of names) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + throw new Error('PIER_DOCKER_ENV_NAMES contains an invalid environment name') + } + const value = process.env[name] + if (value === undefined) throw new Error(`required Docker environment variable ${name} is unset`) + env[name] = value + } + return { id: dockerConnectionId, env } +})() + const controller = new FilePierCandidateTrialController({ directory: path.resolve(directoryArg), + ...(dockerConnection ? { dockerConnection } : {}), }) const executor = createPierCandidateRecoveryExecutor(controller) const recovered = await executor.stopAndCapture( diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index d42826cf..298b45d7 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -83,7 +83,7 @@ try { ) await writeFile( path.join(consumerDir, 'index.ts'), - "import { executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, type BenchmarkAdapter, type PierCandidateTrialController, type PierCandidateTrialHandle, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nvoid adapter\nvoid staged\nvoid trial\nvoid controller\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\n", + "import { executePreparedPierCandidate, FilePierCandidateTrialController, resolveAdapter, runBenchmarks, type BenchmarkAdapter, type PierCandidateTrialController, type PierCandidateTrialHandle, type PierDockerConnection, type StagedPierCandidateExecution } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nconst staged = undefined as StagedPierCandidateExecution | undefined\nconst trial = undefined as PierCandidateTrialHandle | undefined\nconst controller = undefined as PierCandidateTrialController | undefined\nconst dockerConnection = undefined as PierDockerConnection | undefined\nvoid adapter\nvoid staged\nvoid trial\nvoid controller\nvoid dockerConnection\nvoid executePreparedPierCandidate\nvoid FilePierCandidateTrialController\nvoid runBenchmarks\n", ) await writeFile( path.join(consumerDir, 'tsconfig.json'), @@ -97,38 +97,26 @@ try { )}\n`, ) await writeFile( - path.join(consumerDir, 'verify_pier_import.py'), - `import sys -import types + path.join(consumerDir, 'verify_pier_payload.py'), + `import py_compile +from pathlib import Path -modules = { - name: types.ModuleType(name) - for name in ( - "pier", - "pier.agents", - "pier.agents.base", - "pier.agents.installed", - "pier.agents.installed.base", - "pier.environments", - "pier.environments.base", - "pier.models", - "pier.models.agent", - "pier.models.agent.context", - "pier.models.agent.network", - ) -} -modules["pier.agents.base"].BaseAgent = type("BaseAgent", (), {}) -modules["pier.agents.installed.base"].NonZeroAgentExitCodeError = type( - "NonZeroAgentExitCodeError", (RuntimeError,), {} -) -modules["pier.environments.base"].BaseEnvironment = type("BaseEnvironment", (), {}) -modules["pier.models.agent.context"].AgentContext = type("AgentContext", (), {}) -modules["pier.models.agent.network"].NetworkAllowlist = type("NetworkAllowlist", (), {}) -sys.modules.update(modules) +import pier_agents +from pier_agents import candidate_contract -from pier_agents.tangle_candidate import TangleCandidateAgent - -assert TangleCandidateAgent.__module__ == "pier_agents.tangle_candidate" +root = Path(pier_agents.__file__).parent +expected = { + "__init__.py", + "candidate_contract.py", + "process_boundary.py", + "tangle_candidate.py", + "workspace_boundary.py", +} +observed = {path.name for path in root.glob("*.py")} +assert observed == expected, (observed, expected) +assert candidate_contract.PreparedCandidateContract.__module__ == "pier_agents.candidate_contract" +for name in sorted(expected): + py_compile.compile(root / name, doraise=True) `, ) @@ -151,7 +139,7 @@ assert TangleCandidateAgent.__module__ == "pier_agents.tangle_candidate" path.join(installedPackage, 'scripts', 'verify-pier-agent.mts'), ], consumerDir, - { ...process.env, PIER_PREPARE_ONLY: '1' }, + { ...process.env, PIER_PREPARE_ONLY: '1', PIER_PROOF_ARM: 'failure' }, ) const prepareProof = JSON.parse(prepared.stdout) if ( @@ -162,7 +150,7 @@ assert TangleCandidateAgent.__module__ == "pier_agents.tangle_candidate" ) { throw new Error(`packed consumer did not prepare a real candidate: ${prepared.stdout}`) } - await run('python3', ['verify_pier_import.py'], consumerDir, { + await run('python3', ['verify_pier_payload.py'], consumerDir, { ...process.env, PYTHONPATH: installedPackage, }) diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts index b530c56e..add31075 100644 --- a/bench/scripts/verify-pier-agent.mts +++ b/bench/scripts/verify-pier-agent.mts @@ -43,9 +43,9 @@ const pinnedPierVersion = '0.3.0' const modelRequest = 'openai/gpt-5.4' const fixtureImage = 'ghcr.io/tangle-network/devcontainers/universal:latest' const prepareOnly = process.env.PIER_PREPARE_ONLY === '1' -const proofArm = process.env.PIER_PROOF_ARM ?? (prepareOnly ? 'failure' : undefined) +const proofArm = process.env.PIER_PROOF_ARM if (proofArm !== 'failure' && proofArm !== 'success') { - throw new Error('PIER_PROOF_ARM must be failure or success') + throw new Error('PIER_PROOF_ARM must be explicitly set to failure or success') } const expectedReward = proofArm === 'success' ? 1 : 0 const expectedPatchApplied = proofArm === 'success' ? 1 : 0 diff --git a/bench/scripts/verify-pier-recovery.mts b/bench/scripts/verify-pier-recovery.mts index c9779018..0aa25645 100644 --- a/bench/scripts/verify-pier-recovery.mts +++ b/bench/scripts/verify-pier-recovery.mts @@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url' import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' import { InMemoryTraceStore } from '@tangle-network/agent-eval' -import type { StagedPierCandidateExecution } from '../src/pier-agent' +import { createStagedPierCandidateExecutionFixture } from '../src/pier-agent.test-fixtures.mts' import { FilePierCandidateTrialController } from '../src/pier-trial-controller' const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') @@ -59,7 +59,7 @@ try { }), }) handle = controller.start( - { executionId, evaluatorEnv: {} } as StagedPierCandidateExecution, + createStagedPierCandidateExecutionFixture(executionId), { request: { executionId, diff --git a/bench/src/index.ts b/bench/src/index.ts index fc66a7ba..84c23efc 100644 --- a/bench/src/index.ts +++ b/bench/src/index.ts @@ -70,6 +70,7 @@ export { FilePierCandidateTrialController, type FilePierCandidateTrialControllerOptions, type PierCandidateProcessSpec, + type PierDockerConnection, } from './pier-trial-controller' export { createPierResultGrader } from './pier-result-grader' export { diff --git a/bench/src/pier-agent.test-fixtures.mts b/bench/src/pier-agent.test-fixtures.mts new file mode 100644 index 00000000..d331e57a --- /dev/null +++ b/bench/src/pier-agent.test-fixtures.mts @@ -0,0 +1,19 @@ +import type { StagedPierCandidateExecution } from './pier-agent' + +export function createStagedPierCandidateExecutionFixture( + executionId: string, +): StagedPierCandidateExecution { + const directory = `/tmp/agent-bench-pier-test/${encodeURIComponent(executionId)}` + return { + executionId, + directory, + taskDirectory: `${directory}/task`, + candidateDirectory: `${directory}/candidate`, + profileDirectory: `${directory}/profile`, + planPath: `${directory}/execution-plan.json`, + receiptPath: `${directory}/materialization-receipt.json`, + agentArgs: ['--agent', 'pier_agents.tangle_candidate:TangleCandidateAgent'], + evaluatorEnv: {}, + attemptArgs: ['--n-attempts', '1', '--max-retries', '0'], + } +} diff --git a/bench/src/pier-trial-controller.test.mts b/bench/src/pier-trial-controller.test.mts index e3bf2525..8eeb036a 100644 --- a/bench/src/pier-trial-controller.test.mts +++ b/bench/src/pier-trial-controller.test.mts @@ -9,14 +9,14 @@ import { promisify } from 'node:util' import type { AgentCandidateExecutorRequest } from '@tangle-network/agent-runtime' import { InMemoryTraceStore } from '@tangle-network/agent-eval' -import type { StagedPierCandidateExecution } from './pier-agent' +import { createStagedPierCandidateExecutionFixture } from './pier-agent.test-fixtures.mts' import { FilePierCandidateTrialController } from './pier-trial-controller' const execFileAsync = promisify(execFile) function testRequest(executionId: string, executionPlanDigest: `sha256:${string}`) { return { - staged: { executionId, evaluatorEnv: {} } as StagedPierCandidateExecution, + staged: createStagedPierCandidateExecutionFixture(executionId), context: { request: { executionId, @@ -71,13 +71,209 @@ test('an existing Pier job is rejected without deleting or starting it', async ( } }) +test('the Pier result wait does not add time beyond an expired deadline', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-deadline-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const fakeDocker = path.join(root, 'docker') + const readyPath = path.join(root, 'supervisor-ready') + const supervisorPath = path.join(root, 'supervisor.mjs') + let handle: ReturnType | undefined + let resultSettled: Promise | undefined + try { + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + await writeFile( + supervisorPath, + `import { createReadStream } from 'node:fs' +import { writeFileSync } from 'node:fs' +import path from 'node:path' +const directory = process.argv[2] +process.on('SIGTERM', () => { + writeFileSync(path.join(directory, 'terminal.json'), JSON.stringify({ + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'stopped', + processExited: true, + containersRemoved: true, + })) + process.exit(0) +}) +for await (const _chunk of createReadStream('', { fd: 3 })) {} +writeFileSync(${JSON.stringify(readyPath)}, 'ready') +setInterval(() => undefined, 1_000) +`, + ) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + supervisorPath, + pollIntervalMs: 5, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: {}, + jobsDirectory, + jobName: 'expired-deadline', + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('an expired trial must not read a result') + }, + }), + }) + const { staged, context } = testRequest( + 'pier-expired-deadline', + `sha256:${'e'.repeat(64)}`, + ) + context.deadlineAtMs = Date.now() + handle = controller.start(staged, context) + resultSettled = handle.result.catch((error) => error) + + const outcome = await Promise.race([ + resultSettled, + new Promise<'pending'>((resolvePending) => setTimeout(() => resolvePending('pending'), 500)), + ]) + assert.notEqual(outcome, 'pending') + assert.match(String(outcome), /no terminal acknowledgement/) + } finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + if (resultSettled) await resultSettled + await rm(root, { recursive: true, force: true }) + } +}) + +test('the supervisor receives only launch data and no inherited evaluator environment', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-supervisor-env-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const fakeDocker = path.join(root, 'docker') + const supervisorPath = path.join(root, 'supervisor.mjs') + const previousSecret = process.env.PIER_PARENT_SECRET + let handle: ReturnType | undefined + try { + process.env.PIER_PARENT_SECRET = 'must-not-be-inherited' + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + await writeFile( + supervisorPath, + `import { createReadStream, writeFileSync } from 'node:fs' +import path from 'node:path' +const chunks = [] +for await (const chunk of createReadStream('', { fd: 3 })) chunks.push(Buffer.from(chunk)) +const launch = JSON.parse(Buffer.concat(chunks).toString('utf8')) +const isolated = process.env.PIER_SUPERVISOR_MODE === 'explicit' && + process.env.PIER_PARENT_SECRET === undefined && + launch.env.PIER_PARENT_SECRET === undefined && + launch.env.PIER_CHILD_SECRET === 'child-only' +writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({ + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: isolated ? 'completed' : 'failed', + processExited: true, + containersRemoved: true, + ...(isolated ? {} : { error: 'supervisor inherited evaluator environment' }), +})) +`, + ) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + supervisorPath, + pollIntervalMs: 5, + dockerConnection: { + id: 'explicit-supervisor-test', + env: { PIER_SUPERVISOR_MODE: 'explicit' }, + }, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: { PIER_CHILD_SECRET: 'child-only' }, + jobsDirectory, + jobName: 'isolated-supervisor', + dockerCommand: fakeDocker, + readResult: async () => ({ + value: { output: 'isolated' }, + resultBytes: Uint8Array.from(Buffer.from('{}')), + taskPatch: new Uint8Array(), + }), + }), + }) + const { staged, context } = testRequest( + 'pier-isolated-supervisor', + `sha256:${'f'.repeat(64)}`, + ) + handle = controller.start(staged, context) + assert.deepEqual((await handle.result).value, { output: 'isolated' }) + } finally { + if (previousSecret === undefined) delete process.env.PIER_PARENT_SECRET + else process.env.PIER_PARENT_SECRET = previousSecret + if (handle) await handle.terminateAndWait().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + } +}) + +test('terminal acknowledgements reject unknown fields', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-terminal-schema-')) + const controlRoot = path.join(root, 'control') + const jobsDirectory = path.join(root, 'jobs') + const fakeDocker = path.join(root, 'docker') + const supervisorPath = path.join(root, 'supervisor.mjs') + let handle: ReturnType | undefined + try { + await writeFile(fakeDocker, '#!/bin/sh\nexit 0\n', { mode: 0o700 }) + await chmod(fakeDocker, 0o700) + await writeFile( + supervisorPath, + `import { createReadStream, writeFileSync } from 'node:fs' +import path from 'node:path' +for await (const _chunk of createReadStream('', { fd: 3 })) {} +writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({ + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: 'completed', + processExited: true, + containersRemoved: true, + containerRemoved: true, +})) +`, + ) + const controller = new FilePierCandidateTrialController({ + directory: controlRoot, + supervisorPath, + pollIntervalMs: 5, + launch: () => ({ + command: process.execPath, + args: ['-e', 'process.exit(0)'], + cwd: root, + env: {}, + jobsDirectory, + jobName: 'strict-terminal', + dockerCommand: fakeDocker, + readResult: async () => { + throw new Error('a malformed terminal must not read a result') + }, + }), + }) + const { staged, context } = testRequest( + 'pier-strict-terminal', + `sha256:${'0'.repeat(64)}`, + ) + handle = controller.start(staged, context) + await assert.rejects(handle.result, /terminal is malformed/) + } finally { + if (handle) await handle.terminateAndWait().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + } +}) + test('a fresh evaluator process terminates the persisted process and container identity', async () => { const root = await mkdtemp(path.join(tmpdir(), 'pier-controller-recovery-')) const controlRoot = path.join(root, 'control') const jobsDirectory = path.join(root, 'jobs') const jobName = 'recovery-job' const trialName = 'trial-recovery' - const fakeContainerState = path.join(root, 'fake-container-live') + const primaryContainerState = path.join(root, 'primary-container-live') + const otherContainerState = path.join(root, 'other-container-live') const fakeDocker = path.join(root, 'docker') const childPidPath = path.join(root, 'child.pid') const executionId = 'pier-fresh-process-recovery' @@ -86,12 +282,13 @@ test('a fresh evaluator process terminates the persisted process and container i let resultSettled: Promise | undefined try { - await writeFile(fakeContainerState, 'live\n') + await writeFile(primaryContainerState, 'live\n') + await writeFile(otherContainerState, 'live\n') await writeFile( fakeDocker, `#!/bin/sh set -eu -state=${JSON.stringify(fakeContainerState)} +state="\${FAKE_DOCKER_STATE:?missing FAKE_DOCKER_STATE}" if test "\${1:-}" = ps; then test -f "$state" && printf 'fake-container\\t${trialName}\\n' exit 0 @@ -109,6 +306,10 @@ exit 2 const controller = new FilePierCandidateTrialController({ directory: controlRoot, + dockerConnection: { + id: 'fake-primary', + env: { FAKE_DOCKER_STATE: primaryContainerState }, + }, launch: () => ({ command: process.execPath, args: [ @@ -116,7 +317,7 @@ exit 2 `const { spawn } = require('node:child_process'); const { mkdirSync, writeFileSync } = require('node:fs'); mkdirSync(${JSON.stringify(path.join(jobsDirectory, jobName, trialName))}, { recursive: true }); const child = spawn(process.execPath, ['-e', 'setInterval(() => undefined, 1_000)'], { stdio: 'ignore' }); writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid)); setInterval(() => undefined, 1_000)`, ], cwd: root, - env: { ...process.env }, + env: {}, jobsDirectory, jobName, dockerCommand: fakeDocker, @@ -159,9 +360,38 @@ exit 2 executionId, executionPlanDigest, ] + const recoveryEnvironment = { + ...process.env, + PIER_DOCKER_CONNECTION_ID: 'fake-primary', + PIER_DOCKER_ENV_NAMES: 'FAKE_DOCKER_STATE', + FAKE_DOCKER_STATE: primaryContainerState, + } + await assert.rejects( + execFileAsync(process.execPath, recoveryArgs, { + cwd: path.resolve('.'), + env: { + ...recoveryEnvironment, + PIER_DOCKER_CONNECTION_ID: 'fake-other', + FAKE_DOCKER_STATE: otherContainerState, + }, + timeout: 20_000, + }), + (error: unknown) => + String((error as { stderr?: string }).stderr).includes( + 'requires Docker connection fake-primary', + ), + ) const recovered = await Promise.all([ - execFileAsync(process.execPath, recoveryArgs, { cwd: path.resolve('.'), timeout: 20_000 }), - execFileAsync(process.execPath, recoveryArgs, { cwd: path.resolve('.'), timeout: 20_000 }), + execFileAsync(process.execPath, recoveryArgs, { + cwd: path.resolve('.'), + env: recoveryEnvironment, + timeout: 20_000, + }), + execFileAsync(process.execPath, recoveryArgs, { + cwd: path.resolve('.'), + env: recoveryEnvironment, + timeout: 20_000, + }), ]) for (const worker of recovered) { assert.deepEqual(JSON.parse(worker.stdout), { @@ -169,7 +399,8 @@ exit 2 containersRemoved: true, }) } - await assert.rejects(readFile(fakeContainerState), /ENOENT/) + await assert.rejects(readFile(primaryContainerState), /ENOENT/) + assert.equal(await readFile(otherContainerState, 'utf8'), 'live\n') assert.throws(() => process.kill(persisted.pier!.pid, 0), /ESRCH/) assert.throws(() => process.kill(childPid!, 0), /ESRCH/) await resultSettled diff --git a/bench/src/pier-trial-controller.ts b/bench/src/pier-trial-controller.ts index d7696a9b..c818098a 100644 --- a/bench/src/pier-trial-controller.ts +++ b/bench/src/pier-trial-controller.ts @@ -37,6 +37,7 @@ const identityFile = 'identity.json' const terminalFile = 'terminal.json' const stopFile = 'stop-requested' const sha256Pattern = /^sha256:[a-f0-9]{64}$/ +const defaultDockerConnectionId = 'local-default' interface ProcessIdentity { readonly pid: number @@ -56,6 +57,7 @@ interface PersistedTrialIdentity { readonly jobsDirectory: string readonly jobName: string readonly dockerCommand: string + readonly dockerConnectionId: string } interface PersistedTrialTerminal { @@ -68,6 +70,7 @@ interface PersistedTrialTerminal { readonly exitCode?: number | null readonly signal?: NodeJS.Signals | null readonly error?: string + readonly recoveredByPid?: number } export interface PierCandidateProcessSpec { @@ -75,7 +78,7 @@ export interface PierCandidateProcessSpec { readonly command: string readonly args: readonly string[] readonly cwd: string - /** Complete evaluator-owned environment. It is sent over a pipe and never persisted. */ + /** Exact Pier child environment. The controller adds its Docker connection variables. */ readonly env: Readonly> readonly jobsDirectory: string /** Must be unique: the controller atomically reserves this Pier job directory. */ @@ -85,6 +88,13 @@ export interface PierCandidateProcessSpec { readonly readResult: () => Promise } +export interface PierDockerConnection { + /** Stable, non-secret name that every recovery worker maps to the same Docker endpoint. */ + readonly id: string + /** Exact variables needed by both Pier and Docker cleanup; values are never persisted. */ + readonly env: Readonly> +} + export interface FilePierCandidateTrialControllerOptions { /** Shared evaluator-owned control root, available to crash-recovery workers. */ readonly directory: string @@ -97,6 +107,8 @@ export interface FilePierCandidateTrialControllerOptions { readonly deadlineAtMs: number }, ) => PierCandidateProcessSpec + /** Omit only for the default local Docker socket with no environment variables. */ + readonly dockerConnection?: PierDockerConnection readonly supervisorPath?: string readonly pollIntervalMs?: number } @@ -106,11 +118,29 @@ function nonEmpty(value: string, label: string): string { return value } +function stableDockerConnectionId(value: string): string { + if (!/^[A-Za-z0-9._:-]{1,200}$/.test(value)) { + throw new Error('Docker connection id must be a non-secret stable name') + } + return value +} + function absolutePath(value: string, label: string): string { if (!isAbsolute(value)) throw new Error(`${label} must be an absolute path`) return resolve(value) } +function validateEnvironment( + environment: Readonly>, + label: string, +): void { + for (const [name, value] of Object.entries(environment)) { + if (!name || name.includes('\0') || value?.includes('\0')) { + throw new Error(`${label} contains an invalid name or value`) + } + } +} + function resolveExecutable(command: string, searchPath: string | undefined, label: string): string { nonEmpty(command, label) const candidates = command.includes('/') @@ -215,7 +245,8 @@ function parsePersistedIdentity(path: string): PersistedTrialIdentity { !sha256Pattern.test(record.executionPlanDigest) || typeof record.jobsDirectory !== 'string' || typeof record.jobName !== 'string' || - typeof record.dockerCommand !== 'string' + typeof record.dockerCommand !== 'string' || + typeof record.dockerConnectionId !== 'string' ) { throw new Error('Pier trial identity is malformed') } @@ -232,21 +263,63 @@ function parsePersistedIdentity(path: string): PersistedTrialIdentity { jobsDirectory: absolutePath(record.jobsDirectory, 'persisted jobs directory'), jobName: nonEmpty(record.jobName, 'persisted job name'), dockerCommand: nonEmpty(record.dockerCommand, 'persisted Docker command'), + dockerConnectionId: stableDockerConnectionId(record.dockerConnectionId), } } function parseTerminal(path: string): PersistedTrialTerminal { const record = readJson(path, 'Pier trial terminal') + const allowedKeys = new Set([ + 'schemaVersion', + 'kind', + 'status', + 'processExited', + 'containersRemoved', + 'removedContainers', + 'exitCode', + 'signal', + 'error', + 'recoveredByPid', + ]) + const hasMalformedOptionalField = + (record.removedContainers !== undefined && + (!Number.isSafeInteger(record.removedContainers) || (record.removedContainers as number) < 0)) || + (record.exitCode !== undefined && + record.exitCode !== null && + !Number.isSafeInteger(record.exitCode)) || + (record.signal !== undefined && + record.signal !== null && + (typeof record.signal !== 'string' || !/^SIG[A-Z0-9]+$/.test(record.signal))) || + (record.error !== undefined && typeof record.error !== 'string') || + (record.recoveredByPid !== undefined && + (!Number.isSafeInteger(record.recoveredByPid) || (record.recoveredByPid as number) < 1)) if ( + Object.keys(record).some((key) => !allowedKeys.has(key)) || record.schemaVersion !== 1 || record.kind !== 'pier-trial-terminal' || !['completed', 'stopped', 'failed'].includes(record.status as string) || typeof record.processExited !== 'boolean' || - typeof record.containersRemoved !== 'boolean' + typeof record.containersRemoved !== 'boolean' || + hasMalformedOptionalField ) { throw new Error('Pier trial terminal is malformed') } - return record as unknown as PersistedTrialTerminal + return { + schemaVersion: 1, + kind: 'pier-trial-terminal', + status: record.status as PersistedTrialTerminal['status'], + processExited: record.processExited, + containersRemoved: record.containersRemoved, + ...(record.removedContainers !== undefined + ? { removedContainers: record.removedContainers as number } + : {}), + ...(record.exitCode !== undefined ? { exitCode: record.exitCode as number | null } : {}), + ...(record.signal !== undefined ? { signal: record.signal as NodeJS.Signals | null } : {}), + ...(record.error !== undefined ? { error: record.error as string } : {}), + ...(record.recoveredByPid !== undefined + ? { recoveredByPid: record.recoveredByPid as number } + : {}), + } } function processIdentity(pid: number): ProcessIdentity { @@ -334,13 +407,21 @@ function trialProjects(identity: PersistedTrialIdentity): string[] { .map((entry) => sanitizeProject(entry.name)) } -function matchingContainers(identity: PersistedTrialIdentity): string[] { +function matchingContainers( + identity: PersistedTrialIdentity, + dockerEnvironment: Readonly>, +): string[] { const projects = trialProjects(identity) if (projects.length === 0) return [] const output = execFileSync( identity.dockerCommand, ['ps', '-a', '--format', '{{.ID}}\t{{.Label "com.docker.compose.project"}}'], - { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }, + { + encoding: 'utf8', + env: dockerEnvironment, + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }, ) return output .split('\n') @@ -359,20 +440,24 @@ function matchingContainers(identity: PersistedTrialIdentity): string[] { }) } -function removeTrialContainers(identity: PersistedTrialIdentity): number { - const containers = matchingContainers(identity) +function removeTrialContainers( + identity: PersistedTrialIdentity, + dockerEnvironment: Readonly>, +): number { + const containers = matchingContainers(identity, dockerEnvironment) if (containers.length > 0) { try { execFileSync(identity.dockerCommand, ['rm', '-f', ...containers], { encoding: 'utf8', + env: dockerEnvironment, timeout: 30_000, maxBuffer: 4 * 1024 * 1024, }) } catch (error) { - if (matchingContainers(identity).length > 0) throw error + if (matchingContainers(identity, dockerEnvironment).length > 0) throw error } } - const remaining = matchingContainers(identity) + const remaining = matchingContainers(identity, dockerEnvironment) if (remaining.length > 0) { throw new Error(`Pier task containers survived removal: ${remaining.join(', ')}`) } @@ -411,12 +496,27 @@ function assertRealDirectory(path: string, label: string): void { export class FilePierCandidateTrialController implements PierCandidateTrialController { private readonly directory: string private readonly launch?: NonNullable + private readonly dockerConnection: PierDockerConnection private readonly supervisorPath: string private readonly pollIntervalMs: number constructor(options: FilePierCandidateTrialControllerOptions) { this.directory = absolutePath(options.directory, 'Pier controller directory') this.launch = options.launch + const configuredDocker = options.dockerConnection + if (configuredDocker?.id === defaultDockerConnectionId) { + throw new Error(`${defaultDockerConnectionId} is reserved for the implicit local connection`) + } + const dockerConnection = configuredDocker ?? { + id: defaultDockerConnectionId, + env: {}, + } + const connectionId = stableDockerConnectionId(dockerConnection.id) + validateEnvironment(dockerConnection.env, 'Docker connection environment') + this.dockerConnection = Object.freeze({ + id: connectionId, + env: Object.freeze({ ...dockerConnection.env }), + }) this.supervisorPath = absolutePath( options.supervisorPath ?? fileURLToPath(new URL('./pier-trial-supervisor.mjs', import.meta.url)), @@ -452,6 +552,18 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr // before reserving durable identities or starting a process. const spec = this.launch(staged, context) this.validateSpec(spec) + for (const [name, value] of Object.entries(this.dockerConnection.env)) { + if ( + Object.prototype.hasOwnProperty.call(spec.env, name) && + spec.env[name] !== value + ) { + throw new Error(`Pier child environment conflicts with Docker connection variable ${name}`) + } + } + const pierEnvironment = Object.freeze({ + ...spec.env, + ...this.dockerConnection.env, + }) mkdirSync(spec.jobsDirectory, { recursive: true, mode: 0o700 }) assertRealDirectory(spec.jobsDirectory, 'Pier jobs directory') const dockerCommand = resolveExecutable( @@ -459,7 +571,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr process.env.PATH, 'Docker command', ) - const pierCommand = resolveExecutable(spec.command, spec.env.PATH, 'Pier command') + const pierCommand = resolveExecutable(spec.command, pierEnvironment.PATH, 'Pier command') const controlDirectory = this.controlDirectory(identity) try { @@ -498,6 +610,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr jobsDirectory: absolutePath(spec.jobsDirectory, 'Pier jobs directory'), jobName: nonEmpty(spec.jobName, 'Pier job name'), dockerCommand: nonEmpty(dockerCommand, 'Docker command'), + dockerConnectionId: this.dockerConnection.id, } try { writeJsonAtomic(controlDirectory, identityFile, allocating) @@ -510,6 +623,9 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr const supervisor = spawn(process.execPath, [this.supervisorPath, controlDirectory], { detached: true, + // Launch data arrives over fd 3. The trusted supervisor receives only + // explicitly declared process variables, never the evaluator environment. + env: { ...this.dockerConnection.env }, stdio: ['ignore', 'ignore', 'ignore', 'pipe'], }) if (supervisor.pid === undefined) throw new Error('Pier supervisor started without a pid') @@ -530,7 +646,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr command: pierCommand, args: spec.args, cwd: spec.cwd, - env: spec.env, + env: pierEnvironment, jobsDirectory: spec.jobsDirectory, jobName: spec.jobName, dockerCommand, @@ -571,6 +687,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr ) { throw new Error('persisted Pier trial identity differs from the stop request') } + this.assertDockerConnection(persisted) touchDurable(controlDirectory, stopFile) if (!existsSync(join(controlDirectory, terminalFile))) { @@ -586,7 +703,8 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr let terminal = parseTerminal(join(controlDirectory, terminalFile)) const latest = parsePersistedIdentity(join(controlDirectory, identityFile)) const liveProcesses = latest.pier ? sessionMembers(latest.pier.sessionId).length : 0 - const liveContainers = matchingContainers(latest).length + this.assertDockerConnection(latest) + const liveContainers = matchingContainers(latest, this.dockerConnection.env).length if (liveProcesses > 0 || liveContainers > 0) { await this.forceRecovery(controlDirectory) terminal = parseTerminal(join(controlDirectory, terminalFile)) @@ -603,6 +721,14 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr return join(this.directory, trialKey(identity)) } + private assertDockerConnection(identity: PersistedTrialIdentity): void { + if (identity.dockerConnectionId !== this.dockerConnection.id) { + throw new Error( + `Pier trial requires Docker connection ${identity.dockerConnectionId}; configured ${this.dockerConnection.id}`, + ) + } + } + private validateSpec(spec: PierCandidateProcessSpec): void { nonEmpty(spec.command, 'Pier command') if (!Array.isArray(spec.args) || spec.args.some((arg) => typeof arg !== 'string' || arg.includes('\0'))) { @@ -613,11 +739,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr if (!/^[A-Za-z0-9._-]{1,200}$/.test(spec.jobName)) { throw new Error('Pier job name must be filesystem-neutral') } - for (const [name, value] of Object.entries(spec.env)) { - if (!name || name.includes('\0') || value?.includes('\0')) { - throw new Error('Pier environment contains an invalid name or value') - } - } + validateEnvironment(spec.env, 'Pier child environment') } private async waitForResult( @@ -625,7 +747,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr spec: PierCandidateProcessSpec, deadlineAtMs: number, ): Promise { - const waitMs = Math.max(1, deadlineAtMs - Date.now() + 15_000) + const waitMs = Math.max(0, deadlineAtMs - Date.now()) const appeared = await waitUntil( () => existsSync(join(controlDirectory, terminalFile)), waitMs, @@ -646,6 +768,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr controlDirectory: string, ): Promise { const latest = parsePersistedIdentity(join(controlDirectory, identityFile)) + this.assertDockerConnection(latest) const errors: string[] = [] if (latest.pier) { try { @@ -682,7 +805,7 @@ export class FilePierCandidateTrialController implements PierCandidateTrialContr } let removedContainers = 0 try { - removedContainers = removeTrialContainers(latest) + removedContainers = removeTrialContainers(latest, this.dockerConnection.env) } catch (error) { errors.push(`could not remove Pier containers: ${error instanceof Error ? error.message : String(error)}`) }