feat(run): add --mode crane — materialize evals at run time

cli/src/naming.rs

@@ -62,6 +62,15 @@ pub fn eval_task_image(
     )
 }
 
+/// `{registry}/core/crane-runner:<tag>` — the generic runtime-fusion runner.
+/// One image for every (benchmark, agent): `--mode crane` runs it and it
+/// crane-pulls the per-axis `benchmarks/<b>` + `agents/<a>` images and fuses
+/// them at run time, so the per-(benchmark, agent) `evals/` images above are not
+/// required (they stay an optional pre-bake).
+pub fn crane_runner_image(registry: &str, tag: &str) -> String {
+    format!("{registry}/core/crane-runner:{tag}")
+}
+
 /// `{registry}/evaluate` — the single published evaluation compose artifact.
 /// `run --mode compose` consumes it as `oci://{registry}/evaluate`; one generic,
 /// `EVAL_BENCHMARK`-parameterized artifact, not one per benchmark.
@@ -148,6 +157,14 @@ mod tests {
         );
     }
 
+    #[test]
+    fn crane_runner_is_one_generic_core_image() {
+        assert_eq!(
+            crane_runner_image(REG, "latest"),
+            "ghcr.io/exgentic/core/crane-runner:latest"
+        );
+    }
+
     #[test]
     fn category_images_are_namespaced() {
         assert_eq!(

cli/src/run.rs

@@ -52,6 +52,12 @@ pub enum Mode {
     /// One k8s `Job` + one Pod + three containers (NetworkPolicy on runner).
     /// Invocation: `kubectl apply`. Production k8s surface.
     Job,
+    /// One generic runner container that materializes the eval at run time: it
+    /// crane-pulls the per-axis `benchmarks/<b>` + `agents/<a>` images and fuses
+    /// them in-container (bwrap/chroot, no daemon, no DinD), so no
+    /// `evals/<b>--<a>` combination image is needed. Invocation: `docker run`
+    /// the `core/crane-runner` image. Additive — the modes above are untouched.
+    Crane,
 }
 
 #[derive(Args)]
@@ -203,6 +209,14 @@ pub fn execute(registry: &str, args: RunArgs) -> Result<(), String> {
             args.dry_run,
         ),
         Mode::Job => run_job(registry, &benchmark, &args, &envs),
+        Mode::Crane => run_crane(
+            registry,
+            &benchmark,
+            &args.agent,
+            &envs,
+            args.local,
+            args.dry_run,
+        ),
     }
 }
 
@@ -335,6 +349,53 @@ fn run_container(
         eval_containers::naming::eval_image(registry, benchmark, &agent, "latest")
     };
 
+    docker_run_eval(&image, envs, dry_run)
+}
+
+/// `--mode crane` → docker run -e EVAL_* <core/crane-runner image>
+///
+/// Unlike `--mode container` (which runs the pre-built `evals/<b>--<a>` image),
+/// crane mode runs ONE generic runner that materializes the eval at run time: it
+/// crane-pulls the per-axis `benchmarks/<b>` + `agents/<a>` images and fuses
+/// them in-container — no combination image, no DinD. `EVAL_BENCHMARK`,
+/// `EVAL_AGENT`, and `EVAL_TASK_ID` (already in `envs`) tell the runner what to
+/// assemble; for a per-task benchmark we also pass `EVAL_BENCHMARK_ENV=per-task`
+/// so it materializes the per-task rootfs. This is the single-container analog
+/// of the generic compose file, one better: it composes the axes at run time, so
+/// the combination matrix is optional.
+fn run_crane(
+    registry: &str,
+    benchmark: &str,
+    agent: &Option<String>,
+    envs: &[(&str, String)],
+    local: bool,
+    dry_run: bool,
+) -> Result<(), String> {
+    if agent.is_none() {
+        return Err("--agent is required in crane mode".to_string());
+    }
+    if local {
+        // The runner pulls published per-axis images at run time; there is no
+        // local combination image to build (that is the whole point).
+        return Err("--local is not supported in crane mode".to_string());
+    }
+    let image = eval_containers::naming::crane_runner_image(registry, "latest");
+
+    // Per-task benchmarks materialize `benchmarks/<b>-<task>`; tell the runner.
+    let mut env_list: Vec<(&str, String)> = envs.to_vec();
+    if eval_containers::benchmark::is_per_task_by_name(benchmark) {
+        env_list.push(("EVAL_BENCHMARK_ENV", "per-task".to_string()));
+    }
+
+    docker_run_eval(&image, &env_list, dry_run)
+}
+
+/// `docker run --rm -e EVAL_* [-e <cred>…] -v output:/output <image>` — or print it
+/// for `--dry-run`. Shared by the single-image surfaces (`--mode container` and
+/// `--mode crane`), which run the gateway in-container and so forward the upstream
+/// credentials (`OPENAI_API_*`) it proxies to — skipped when the caller didn't set
+/// them, so it's a no-op otherwise.
+fn docker_run_eval(image: &str, envs: &[(&str, String)], dry_run: bool) -> Result<(), String> {
     let env_str = envs
         .iter()
         .map(|(k, v)| format!("-e {k}={v}"))
@@ -345,24 +406,17 @@ fn run_container(
         eprintln!("(--dry-run: stopping before docker run)");
         return Ok(());
     }
-
     let mut cmd = Command::new("docker");
     cmd.arg("run").arg("--rm");
     for (k, v) in envs {
         cmd.arg("-e").arg(format!("{k}={v}"));
     }
-    // Single-image mode runs the gateway in-container, so it needs the upstream
-    // credentials the gateway service gets from `eval-secrets` (k8s) or the
-    // shell env (compose). Forward them from the caller's environment with
-    // docker's `-e NAME` passthrough (no value → not rendered into logs); unset
-    // vars are skipped, so this is a no-op when the caller didn't provide them.
     for var in GATEWAY_CRED_VARS {
         if std::env::var_os(var).is_some() {
             cmd.arg("-e").arg(var);
         }
     }
-    cmd.arg("-v").arg("output:/output");
-    cmd.arg(&image);
+    cmd.arg("-v").arg("output:/output").arg(image);
     let status = cmd
         .status()
         .map_err(|e| format!("failed to docker run: {e}"))?;

containers/core/crane-runner/Dockerfile

@@ -0,0 +1,27 @@
+# The crane runner -- ONE generic image that materializes any (benchmark, agent)
+# eval at run time, instead of pulling a pre-built evals/<b>--<a> combination
+# image. It crane-exports the per-axis images to a rootfs and fuses them with
+# chroot/bwrap: no Docker daemon, no DinD (crane just downloads a filesystem).
+#
+# STATUS: first cut. The fusion ([1]/[2] in `materialize`) is proven; wiring the
+# in-rootfs gateway/otel via process-compose and the root-only grader perms is
+# the remaining build-out + validation gate (see README.md). Nothing else in the
+# repo depends on this image -- `--mode crane` opts in.
+FROM debian:stable-slim
+
+# crane (a daemonless registry client) downloads an image's filesystem; curl +
+# CA certs fetch crane itself at build time. tar and chroot are already in the
+# base image; bwrap (hardened isolation) lands with the [3] build-out when used.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates curl \
+ && rm -rf /var/lib/apt/lists/*
+
+# Pin crane for reproducibility (override with --build-arg CRANE_VERSION=...).
+ARG CRANE_VERSION=v0.20.3
+RUN arch="$(dpkg --print-architecture)"; case "$arch" in amd64) arch=x86_64 ;; esac; \
+    curl -fsSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${arch}.tar.gz" \
+      | tar -xz -C /usr/local/bin crane
+
+COPY materialize /usr/local/bin/materialize
+RUN chmod +x /usr/local/bin/materialize
+ENTRYPOINT ["/usr/local/bin/materialize"]

containers/core/crane-runner/README.md

@@ -0,0 +1,59 @@
+# crane-runner (`--mode crane`)
+
+One generic image that materializes any `(benchmark, agent)` eval at **run time**
+by fusing the per-axis images, instead of pulling a pre-built
+`evals/<benchmark>--<agent>` combination image.
+
+## Why
+
+`compose` / `container` / `job` modes all run a pre-built combination image, so
+the `evals/<b>--<a>` matrix (benchmarks × agents) must exist. Per-task benchmarks
+(SWE-bench) make it worse — a different image per task, which a single
+`images.benchmark` can't fan out. The crane runner pulls the per-axis
+`benchmarks/<b>` + `agents/<a>` images at run time and fuses them in one
+container, so:
+
+- the combination matrix becomes an **optional pre-bake**, and
+- per-task benchmarks run from **one** image (it pulls the per-task rootfs).
+
+It is the single-container analog of the generic `compose` file — one better: it
+composes the *axes* at run time instead of pulling a pre-fused product.
+
+## How (daemonless — no DinD)
+
+`materialize` does: **crane export** the benchmark rootfs (download + untar, no
+daemon) → **overlay the agent** → **chroot/bwrap** in to run agent + grade. The
+distinction from DinD: crane only *downloads a filesystem*; it never runs a
+daemon — so nothing here needs privilege or a Docker socket. Verified daemonless
+against the real swe-bench + claude-code images (see the PR).
+
+## Status — first cut (statically verified)
+
+Proven:
+- **Fusion** ([1]/[2]) against **real** images — the real swe-bench `/testbed` (sympy
+  repo) ⊎ the real claude-code `/opt/agent` reproduce the combination image's layout.
+- **Isolation survives the fusion**: `export` preserves root-only modes/owners
+  (`/tasks` `0600`, `/tests` & `/opt/gateway` `0700`, root), so *extract as root* and
+  run the agent through the existing `gosu agent` / `env -i` pipeline (which even hides
+  the task id from the model) and the answers stay unreadable. The invariant is
+  **reused, not reinvented**.
+
+Build-out ([3]) — wire the existing runtime, don't invent one:
+1. **Bake the fixed core** into this image: `otelcol`, `process-compose`, `gosu`,
+   `/usr/local/bin/{run,write-result}` + configs.
+2. **Pull the 3rd axis** (`models/<gateway>` → `/opt/gateway`) alongside benchmark+agent;
+   run the agent image's `install.sh` (symlinks).
+3. **Extract as root**, then `exec /entrypoint.sh /usr/local/bin/run` — the existing
+   5-process pipeline (otelcol → gateway → agent → verifier → result).
+
+Hard constraint: **arch** — the runner, the node, and the pulled rootfs must match
+(swe-bench is `x86_64`; the runner built here is `arm64`). Plus **digest-pinned** pulls,
+a **conformance test** (`crane(X) == container(X)`), a **bake target**, and a **doctrine
+rule** (when crane applies + the digest-pin + isolation invariants).
+
+## Use (additive, opt-in — nothing else depends on it)
+
+```bash
+eval-containers run aime --agent codex --model openai/<model> --mode crane
+# → docker run --rm -e EVAL_* -v output:/output <registry>/core/crane-runner:latest
+```

containers/core/crane-runner/materialize

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Materialize one eval by fusing per-axis images at run time, then run it.
+# Proven primitive: crane export (daemonless pull) + bwrap/chroot (run in rootfs).
+set -euo pipefail
+: "${EVAL_REGISTRY:?}" "${EVAL_BENCHMARK:?}" "${EVAL_AGENT:?}"
+btag="${EVAL_BENCHMARK_TAG:-latest}"
+atag="${EVAL_AGENT_TAG:-latest}"
+root=/run/eval
+mkdir -p "$root"
+
+echo "crane-runner: fusing ${EVAL_BENCHMARK} x ${EVAL_AGENT} at run time (no combination image)"
+
+# [1] Benchmark filesystem (testbed + tasks + grader + entrypoint). Per-task
+#     benchmarks resolve to benchmarks/<b>-<task>; shared-env to benchmarks/<b>.
+if [ "${EVAL_BENCHMARK_ENV:-shared-env}" = "per-task" ]; then
+  bench="${EVAL_REGISTRY}/benchmarks/${EVAL_BENCHMARK}-${EVAL_TASK_ID:?per-task needs EVAL_TASK_ID}:${btag}"
+else
+  bench="${EVAL_REGISTRY}/benchmarks/${EVAL_BENCHMARK}:${btag}"
+fi
+echo "  [1] crane export ${bench}"
+crane export "$bench" - | tar -x -C "$root"
+
+# [2] Overlay the agent into the benchmark rootfs (mirrors the combination
+#     Dockerfile's `COPY --from=agent /opt/agent`).
+agent="${EVAL_REGISTRY}/agents/${EVAL_AGENT}:${atag}"
+echo "  [2] crane export ${agent} -> overlay"
+crane export "$agent" - | tar -x -C "$root"
+
+# ---------------------------------------------------------------------------
+# BUILD-OUT ([3], see README): wire the EXISTING runtime into the fused rootfs --
+# bake otelcol/process-compose/gosu/run, pull the gateway axis, extract as root,
+# then `exec /entrypoint.sh /usr/local/bin/run` (the 5-process pipeline). `export`
+# preserves root-only perms, so answer-isolation holds by reuse. This first cut
+# proves the FUSION ([1]/[2]); end-to-end is the remaining work.
+# ---------------------------------------------------------------------------
+echo "  [3] fused rootfs ready at ${root} ($(du -sh "$root" 2>/dev/null | cut -f1))"
+echo "      build-out: bake core + pull gateway + exec /entrypoint.sh /usr/local/bin/run"