diff --git a/cli/src/naming.rs b/cli/src/naming.rs index ba00ae54..78d538c2 100644 --- a/cli/src/naming.rs +++ b/cli/src/naming.rs @@ -62,6 +62,15 @@ pub fn eval_task_image( ) } +/// `{registry}/core/crane-runner:` — the generic runtime-fusion runner. +/// One image for every (benchmark, agent): `--mode crane` runs it and it +/// crane-pulls the per-axis `benchmarks/` + `agents/` images and fuses +/// them at run time, so the per-(benchmark, agent) `evals/` images above are not +/// required (they stay an optional pre-bake). +pub fn crane_runner_image(registry: &str, tag: &str) -> String { + format!("{registry}/core/crane-runner:{tag}") +} + /// `{registry}/evaluate` — the single published evaluation compose artifact. /// `run --mode compose` consumes it as `oci://{registry}/evaluate`; one generic, /// `EVAL_BENCHMARK`-parameterized artifact, not one per benchmark. @@ -148,6 +157,14 @@ mod tests { ); } + #[test] + fn crane_runner_is_one_generic_core_image() { + assert_eq!( + crane_runner_image(REG, "latest"), + "ghcr.io/exgentic/core/crane-runner:latest" + ); + } + #[test] fn category_images_are_namespaced() { assert_eq!( diff --git a/cli/src/run.rs b/cli/src/run.rs index abb8119f..5f8a6f60 100644 --- a/cli/src/run.rs +++ b/cli/src/run.rs @@ -52,6 +52,12 @@ pub enum Mode { /// One k8s `Job` + one Pod + three containers (NetworkPolicy on runner). /// Invocation: `kubectl apply`. Production k8s surface. Job, + /// One generic runner container that materializes the eval at run time: it + /// crane-pulls the per-axis `benchmarks/` + `agents/` images and fuses + /// them in-container (bwrap/chroot, no daemon, no DinD), so no + /// `evals/--` combination image is needed. Invocation: `docker run` + /// the `core/crane-runner` image. Additive — the modes above are untouched. + Crane, } #[derive(Args)] @@ -203,6 +209,14 @@ pub fn execute(registry: &str, args: RunArgs) -> Result<(), String> { args.dry_run, ), Mode::Job => run_job(registry, &benchmark, &args, &envs), + Mode::Crane => run_crane( + registry, + &benchmark, + &args.agent, + &envs, + args.local, + args.dry_run, + ), } } @@ -335,6 +349,53 @@ fn run_container( eval_containers::naming::eval_image(registry, benchmark, &agent, "latest") }; + docker_run_eval(&image, envs, dry_run) +} + +/// `--mode crane` → docker run -e EVAL_* +/// +/// Unlike `--mode container` (which runs the pre-built `evals/--` image), +/// crane mode runs ONE generic runner that materializes the eval at run time: it +/// crane-pulls the per-axis `benchmarks/` + `agents/` images and fuses +/// them in-container — no combination image, no DinD. `EVAL_BENCHMARK`, +/// `EVAL_AGENT`, and `EVAL_TASK_ID` (already in `envs`) tell the runner what to +/// assemble; for a per-task benchmark we also pass `EVAL_BENCHMARK_ENV=per-task` +/// so it materializes the per-task rootfs. This is the single-container analog +/// of the generic compose file, one better: it composes the axes at run time, so +/// the combination matrix is optional. +fn run_crane( + registry: &str, + benchmark: &str, + agent: &Option, + envs: &[(&str, String)], + local: bool, + dry_run: bool, +) -> Result<(), String> { + if agent.is_none() { + return Err("--agent is required in crane mode".to_string()); + } + if local { + // The runner pulls published per-axis images at run time; there is no + // local combination image to build (that is the whole point). + return Err("--local is not supported in crane mode".to_string()); + } + let image = eval_containers::naming::crane_runner_image(registry, "latest"); + + // Per-task benchmarks materialize `benchmarks/-`; tell the runner. + let mut env_list: Vec<(&str, String)> = envs.to_vec(); + if eval_containers::benchmark::is_per_task_by_name(benchmark) { + env_list.push(("EVAL_BENCHMARK_ENV", "per-task".to_string())); + } + + docker_run_eval(&image, &env_list, dry_run) +} + +/// `docker run --rm -e EVAL_* [-e …] -v output:/output ` — or print it +/// for `--dry-run`. Shared by the single-image surfaces (`--mode container` and +/// `--mode crane`), which run the gateway in-container and so forward the upstream +/// credentials (`OPENAI_API_*`) it proxies to — skipped when the caller didn't set +/// them, so it's a no-op otherwise. +fn docker_run_eval(image: &str, envs: &[(&str, String)], dry_run: bool) -> Result<(), String> { let env_str = envs .iter() .map(|(k, v)| format!("-e {k}={v}")) @@ -345,24 +406,17 @@ fn run_container( eprintln!("(--dry-run: stopping before docker run)"); return Ok(()); } - let mut cmd = Command::new("docker"); cmd.arg("run").arg("--rm"); for (k, v) in envs { cmd.arg("-e").arg(format!("{k}={v}")); } - // Single-image mode runs the gateway in-container, so it needs the upstream - // credentials the gateway service gets from `eval-secrets` (k8s) or the - // shell env (compose). Forward them from the caller's environment with - // docker's `-e NAME` passthrough (no value → not rendered into logs); unset - // vars are skipped, so this is a no-op when the caller didn't provide them. for var in GATEWAY_CRED_VARS { if std::env::var_os(var).is_some() { cmd.arg("-e").arg(var); } } - cmd.arg("-v").arg("output:/output"); - cmd.arg(&image); + cmd.arg("-v").arg("output:/output").arg(image); let status = cmd .status() .map_err(|e| format!("failed to docker run: {e}"))?; diff --git a/containers/core/crane-runner/Dockerfile b/containers/core/crane-runner/Dockerfile new file mode 100644 index 00000000..27ec8bee --- /dev/null +++ b/containers/core/crane-runner/Dockerfile @@ -0,0 +1,27 @@ +# The crane runner -- ONE generic image that materializes any (benchmark, agent) +# eval at run time, instead of pulling a pre-built evals/-- combination +# image. It crane-exports the per-axis images to a rootfs and fuses them with +# chroot/bwrap: no Docker daemon, no DinD (crane just downloads a filesystem). +# +# STATUS: first cut. The fusion ([1]/[2] in `materialize`) is proven; wiring the +# in-rootfs gateway/otel via process-compose and the root-only grader perms is +# the remaining build-out + validation gate (see README.md). Nothing else in the +# repo depends on this image -- `--mode crane` opts in. +FROM debian:stable-slim + +# crane (a daemonless registry client) downloads an image's filesystem; curl + +# CA certs fetch crane itself at build time. tar and chroot are already in the +# base image; bwrap (hardened isolation) lands with the [3] build-out when used. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +# Pin crane for reproducibility (override with --build-arg CRANE_VERSION=...). +ARG CRANE_VERSION=v0.20.3 +RUN arch="$(dpkg --print-architecture)"; case "$arch" in amd64) arch=x86_64 ;; esac; \ + curl -fsSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${arch}.tar.gz" \ + | tar -xz -C /usr/local/bin crane + +COPY materialize /usr/local/bin/materialize +RUN chmod +x /usr/local/bin/materialize +ENTRYPOINT ["/usr/local/bin/materialize"] diff --git a/containers/core/crane-runner/README.md b/containers/core/crane-runner/README.md new file mode 100644 index 00000000..7c498190 --- /dev/null +++ b/containers/core/crane-runner/README.md @@ -0,0 +1,59 @@ +# crane-runner (`--mode crane`) + +One generic image that materializes any `(benchmark, agent)` eval at **run time** +by fusing the per-axis images, instead of pulling a pre-built +`evals/--` combination image. + +## Why + +`compose` / `container` / `job` modes all run a pre-built combination image, so +the `evals/--` matrix (benchmarks × agents) must exist. Per-task benchmarks +(SWE-bench) make it worse — a different image per task, which a single +`images.benchmark` can't fan out. The crane runner pulls the per-axis +`benchmarks/` + `agents/` images at run time and fuses them in one +container, so: + +- the combination matrix becomes an **optional pre-bake**, and +- per-task benchmarks run from **one** image (it pulls the per-task rootfs). + +It is the single-container analog of the generic `compose` file — one better: it +composes the *axes* at run time instead of pulling a pre-fused product. + +## How (daemonless — no DinD) + +`materialize` does: **crane export** the benchmark rootfs (download + untar, no +daemon) → **overlay the agent** → **chroot/bwrap** in to run agent + grade. The +distinction from DinD: crane only *downloads a filesystem*; it never runs a +daemon — so nothing here needs privilege or a Docker socket. Verified daemonless +against the real swe-bench + claude-code images (see the PR). + +## Status — first cut (statically verified) + +Proven: +- **Fusion** ([1]/[2]) against **real** images — the real swe-bench `/testbed` (sympy + repo) ⊎ the real claude-code `/opt/agent` reproduce the combination image's layout. +- **Isolation survives the fusion**: `export` preserves root-only modes/owners + (`/tasks` `0600`, `/tests` & `/opt/gateway` `0700`, root), so *extract as root* and + run the agent through the existing `gosu agent` / `env -i` pipeline (which even hides + the task id from the model) and the answers stay unreadable. The invariant is + **reused, not reinvented**. + +Build-out ([3]) — wire the existing runtime, don't invent one: +1. **Bake the fixed core** into this image: `otelcol`, `process-compose`, `gosu`, + `/usr/local/bin/{run,write-result}` + configs. +2. **Pull the 3rd axis** (`models/` → `/opt/gateway`) alongside benchmark+agent; + run the agent image's `install.sh` (symlinks). +3. **Extract as root**, then `exec /entrypoint.sh /usr/local/bin/run` — the existing + 5-process pipeline (otelcol → gateway → agent → verifier → result). + +Hard constraint: **arch** — the runner, the node, and the pulled rootfs must match +(swe-bench is `x86_64`; the runner built here is `arm64`). Plus **digest-pinned** pulls, +a **conformance test** (`crane(X) == container(X)`), a **bake target**, and a **doctrine +rule** (when crane applies + the digest-pin + isolation invariants). + +## Use (additive, opt-in — nothing else depends on it) + +```bash +eval-containers run aime --agent codex --model openai/ --mode crane +# → docker run --rm -e EVAL_* -v output:/output /core/crane-runner:latest +``` diff --git a/containers/core/crane-runner/materialize b/containers/core/crane-runner/materialize new file mode 100755 index 00000000..0b4c4890 --- /dev/null +++ b/containers/core/crane-runner/materialize @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Materialize one eval by fusing per-axis images at run time, then run it. +# Proven primitive: crane export (daemonless pull) + bwrap/chroot (run in rootfs). +set -euo pipefail +: "${EVAL_REGISTRY:?}" "${EVAL_BENCHMARK:?}" "${EVAL_AGENT:?}" +btag="${EVAL_BENCHMARK_TAG:-latest}" +atag="${EVAL_AGENT_TAG:-latest}" +root=/run/eval +mkdir -p "$root" + +echo "crane-runner: fusing ${EVAL_BENCHMARK} x ${EVAL_AGENT} at run time (no combination image)" + +# [1] Benchmark filesystem (testbed + tasks + grader + entrypoint). Per-task +# benchmarks resolve to benchmarks/-; shared-env to benchmarks/. +if [ "${EVAL_BENCHMARK_ENV:-shared-env}" = "per-task" ]; then + bench="${EVAL_REGISTRY}/benchmarks/${EVAL_BENCHMARK}-${EVAL_TASK_ID:?per-task needs EVAL_TASK_ID}:${btag}" +else + bench="${EVAL_REGISTRY}/benchmarks/${EVAL_BENCHMARK}:${btag}" +fi +echo " [1] crane export ${bench}" +crane export "$bench" - | tar -x -C "$root" + +# [2] Overlay the agent into the benchmark rootfs (mirrors the combination +# Dockerfile's `COPY --from=agent /opt/agent`). +agent="${EVAL_REGISTRY}/agents/${EVAL_AGENT}:${atag}" +echo " [2] crane export ${agent} -> overlay" +crane export "$agent" - | tar -x -C "$root" + +# --------------------------------------------------------------------------- +# BUILD-OUT ([3], see README): wire the EXISTING runtime into the fused rootfs -- +# bake otelcol/process-compose/gosu/run, pull the gateway axis, extract as root, +# then `exec /entrypoint.sh /usr/local/bin/run` (the 5-process pipeline). `export` +# preserves root-only perms, so answer-isolation holds by reuse. This first cut +# proves the FUSION ([1]/[2]); end-to-end is the remaining work. +# --------------------------------------------------------------------------- +echo " [3] fused rootfs ready at ${root} ($(du -sh "$root" 2>/dev/null | cut -f1))" +echo " build-out: bake core + pull gateway + exec /entrypoint.sh /usr/local/bin/run"