diff --git a/.gitignore b/.gitignore index 88f9cb1..f6ee0cf 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,18 @@ htmlcov/ .tox/ .hypothesis/ +# Eval harness output +evals/output/ +# Keep earlier local runs ignored after the default directory rename. +evals/results/ +evals/.env-pids +evals/.api_runserver.log +evals/.mock_flags.log + +# Booting a local Plane to run evals against is each developer's own setup, +# not part of this repo. The harness itself only needs the EVAL_PLANE_* vars. +localdev/ + # Mypy .mypy_cache/ .dmypy.json diff --git a/evals/DESIGN.md b/evals/DESIGN.md new file mode 100644 index 0000000..1fbce88 --- /dev/null +++ b/evals/DESIGN.md @@ -0,0 +1,461 @@ +# Plane MCP Tool-Surface Eval Harness + +This harness measures how well an LLM agent completes real Plane tasks through an MCP +tool surface. It exists to replace predictions about a surface with observations from +actual agent runs: whether the task succeeded, how many Plane calls it took, which tools +were selected, and how much tool-result content was returned to the model. + +This document explains why the harness is shaped this way. Operational commands live in +`evals/README.md`. + +## The questions it answers + +The original tool-consolidation question breaks down into four measurable questions: + +1. **Task success** — did the agent produce the verified Plane state or exact answer? +2. **Calls to done** — how much lookup, name-to-ID resolution, and sub-object fan-out did + successful repetitions actually require? +3. **Tool-use stability** — which tools were core across successful repetitions, and where + did repetitions choose different routes? +4. **Response bloat** — how much tool-result content was injected into the conversation? + +Success is the guardrail around the other three. A surface that uses fewer calls or returns +less text but fails the task is not an improvement. Conversely, success rate alone hides +extra calls, variable routes, and large responses. The harness therefore records all four +dimensions for the same task execution. + +The point is empirical comparison. Given the same task battery and declared treatment +dimensions, different surfaces can be compared from observed behavior rather than from tool +counts, schema inspection, or projected costs. Report identity validation refuses incompatible +batteries and undeclared canonical-identity differences before printing measurements. The +battery fingerprint records the selected task universe; per-task fingerprints preserve the +task-local payload needed for possible future intersection comparisons. + +## What is measured + +### Success + +Each task has an asynchronous verifier. Mutation tasks read Plane back through the API and +check the resulting state. Read tasks compare the final assistant text with facts obtained +from the seeded context or resolved through the API, using explicit answer contracts and +exact-value matchers where the task defines them. + +This avoids using the agent's explanation, confidence, or self-reported completion as the +source of truth. The model is also not asked to grade another model. Verification is tied to +the fixture and the Plane state the task was meant to affect. The canary reports which +verifiers were exercised, skipped, or errored and probes eligible verifiers with an empty +result plus plausible zero-call contract answers. CI can name an explicit strict set of task +ids that must be eligible in its environment. + +Skipped tasks and infrastructure failures are recorded separately. The report excludes +both from success denominators; a plan gate, unavailable fixture, provider failure, or MCP +process failure is not rewritten as an agent task failure. + +Caught exceptions follow one validity convention: continuing is allowed only when a local +fallback makes the result equivalent, and that catch documents why. Failures that can alter +the evaluated state, recorded evidence, cleanup, or report denominator are represented as +infrastructure, harness, or cleanup errors so run completeness cannot silently remain green. + +### Calls to done + +`num_calls` counts Plane MCP calls made during the task. The report shows the observed +distribution rather than assuming one run is representative. Every reported call-count +minimum, median, maximum, and Q1–Q3 span is conditioned on successful repetitions; a run +that failed early is not treated as a cost-to-success observation. There is no +author-declared call floor. + +Two-label reports pair tasks before making inferential comparisons. Their success-rate +difference uses a paired percentile-bootstrap interval that resamples tasks as the +independent units. Their mean call-count delta uses a paired sign-flip permutation test on +the actual magnitudes, retaining zero-delta ties. These procedures assume comparable task +instances under the two labels, independent tasks, and exchangeable A/B labels under the +permutation null; they do not account for shared environment drift or dependence between +tasks. The report prints the paired task count so small samples remain visible. + +Errored calls use that same successful, trace-intact row population. Within each task, the +absolute measure is the median errored-call count per repetition (parallel to the call-count +median), while the rate is the task's errored calls divided by its total calls. Cross-task +headlines average task values and paired intervals resample whole task deltas; they do not pool +calls across tasks. Reports retain task IDs for non-zero errors and print measured zeros. A +zero-attempt task has an undefined rate rather than an invented zero rate. `is_error` is the +MCP-level error flag: it counts all tool-reported failures, including an error that is the +correct outcome, and cannot detect an agent that successfully calls the wrong tool. + +That single count answers three unrelated questions at once, so errored calls are also split by +the kind of refusal they received, classified in the proxy where the payload still exists and +stored as a category rather than text: + +| reported as | from | means | +|---|---|---| +| navigation | `refused` | turned away without acting — a missing field, a stray argument, a value outside an enum. A property of the tool schema; the API was never asked. | +| surface friction | `rejected` | well formed, and the API refused its meaning. Undocumented preconditions live here. The number to act on. | +| answered existence question | first `not_found` per tool and action | an absent read is the answer, not an obstacle; asking has no cheaper form. A *repeat* is charged to surface friction, because the first answer did not land. | +| other | `denied`, `failed` | credentials, plan, or a broken server. Real, but not attributable to tool design. | +| unclassified | everything else | reported apart from `other` on purpose: a split reading zero surface friction because nothing was classified must not be mistaken for a surface with no friction. Rows written before this field existed land here in full. | + +Classification reads HTTP status and the FastMCP/Pydantic validation shape, never this server's +`ACTIONS` table, so a foreign surface still classifies by status — the same battery has scored +both a 28-tool and a 177-tool build. Two patterns matching this server's own refusal wording are +additive. Status outranks wording: a 404 whose body mentions a missing argument is still an +absent resource, since only a payload with no status at all can be a call that never reached the +API. The split cannot charge a surface for misleading an agent into a single wrong lookup. + +A refusal the server reports as a **successful** result is classified too, and counted in the +split while staying out of the errored-call total, which is keyed on the protocol's error flag. +Reports state that count separately (`N refusal(s) arrived flagged as successful results`) so the +two never silently merge. Without it the metric was blind to roughly a third of what agents +actually get told no about: a run measuring 12.8% refusals was really near 28%. Detection is +deliberately narrow — only wording this server owns, and the stray-argument form must carry both +halves of its sentence — so a result that merely quotes a refusal is not counted as one. + +Call **arguments** are recorded on every driver (`args_json`). A refusal that cannot be +attributed to the target it names answers half a question, and this is what separated an agent +linking the wrong work item from a create that does not persist, when both fit the same symptom. + +Arguments used to ride along with `--record-result-payloads` only. That coupled them to +`result_text`, which just the recording proxy sets, so the api driver — which calls tools +directly and never goes through the proxy — recorded arguments on none of its calls while a CLI +arm recorded them on nearly all of theirs. The arguments were in hand on both paths regardless: +`args_chars` is computed from them and `action` was already kept unconditionally. What a result +file gains is ids and short strings; what it gains in return is the redundant-lookup metric +below, which cannot be computed without them. + +Single-run success headlines use the same sampling unit: each evaluated task contributes +its repetition success rate, and a deterministic cluster bootstrap resamples whole tasks. +The pooled repetition rate and its Wilson interval remain visible as a descriptive figure, +but are labeled pooled rather than presented as the headline confidence interval. + +Client-local tools such as shell or tool-search helpers are retained separately as +`client_tool_calls`; they do not count as Plane calls. For an external server launched with +`--server-cmd`, the runner marks the row server as `external`; call counts and observed tool +distributions use the same rules as local-server rows. + +### Observed tool distribution + +The former author-declared optimal/alternate sets and mispick score were removed. Reports +now describe the tools agents used in successful repetitions: + +- `tool_rep_frequency` is the share of successful repetitions that used each tool at least + once. Repeated calls in one repetition count once for frequency. +- `tool_call_counts` is the total number of calls to each tool across those repetitions. +- Reports label the successful-repetition denominator as `success-only n=...` and show the + number of non-success, non-skip repetitions omitted from it as `failed excluded=...`. + The exclusion count includes recorded harness/infrastructure errors; skips did not run + the agent and remain in execution coverage instead. + +Failed repetitions are excluded because an early failure would otherwise make the tools in +successful runs appear variable. With fewer than two successful repetitions, variance is +not observable and the report shows `frequency=—` beside those counts. Frequency `1.0` is rendered as core use; lower +positive frequency is variable use. The fleet headline counts tasks with at least one +variable tool. Because the measurement is descriptive, external servers get the same metric +as local servers even when their tool names differ. + +### Response-token cost + +Every driver reports `result_chars` and `result_tokens` per Plane call. The character count +comes from the serialized result text actually observed by the harness. Token counts carry +an explicit provenance: + +- The API driver may use a backend token counter. If none is available or it fails, it uses + the shared deterministic character estimate. +- CLI drivers estimate from the proxy-recorded character count by default. +- With `--record-result-payloads`, CLI sidecars also retain the result text. The parent + harness uses `tiktoken` with `cl100k_base` when importable and otherwise falls back to the + same estimate. + +The estimate is `ceil(result_chars / 4)` for non-empty results. Rows and calls record +whether their values are measured, estimated, or mixed, and the report marks estimated and +mixed columns. An estimate is never presented as a measured tokenizer count. + +Payload recording is off by default because tool results contain live workspace data and +make sidecars larger. The character-derived estimate remains useful for surface comparison +because it is deterministic and monotonic in the recorded response size. + +### Cost, and what an unknown cost is allowed to look like + +`usage_total.input_tokens` does not mean the same thing across drivers. It is **inclusive** of +cached reads under OpenAI Responses and **exclusive** under Anthropic Messages and every CLI +vendor, and the same api driver produces both — so driver family cannot decide it. Backends now +declare `input_tokens_include_cache` and the driver records `cache_semantics`; +`evals.core.token_accounting` resolves declared → explicit total → no cache activity → model +family, and **refuses** rather than guessing when none apply. Where a vendor states both parts +and a total, the two must agree or the row is not priced: that disagreement means the shape +changed underneath us, and an unpriced row is visible where a wrong price is not. + +Cost has three outcomes, and the distinction is the point. `priced`; `unpriced` when usage exists +but the model is not in the table; `unmeasured` when the driver recorded no usage at all, which is +true of every antigravity row. A silent `$0.00` reads as *free* rather than as *unknown*, so it is +never printed — nor is a real sub-cent cost rounded into one. + +Prices go stale and a date alone detects nothing. Claude Code reports `total_cost_usd` per run, so +the table's own figure is compared against the vendor's on every run that has one. That check must +compare the table against the vendor rather than the reported cost against itself — the reported +figure already prefers the vendor, so summing it on both sides makes the check incapable of firing. + +### Failure kinds + +A failed verifier answers several unrelated questions at once. Kinds are read from the verifier's +own note text: `unproven`, `wrong_value`, `missing_write`, `partial_write`, `abandoned`, +`environment`, `unclassified`. `unproven` — the answer was right and the run could not evidence it +— is the largest family in the recorded corpus at over half of all failed rows, and is not an agent +defect; nor are `environment` and `abandoned`. Reports name the non-defect total separately so a +raw failure count is not mistaken for a defect count. + +Kinds come from note text only, so a write that landed on the *wrong entity* reports as missing or +partial: the right entity is empty either way, and telling those apart needs call arguments. +`unclassified` is counted and printed, so a zero in some kind never stands in for "the classifier +did not recognise it". + +### Redundant lookups + +A `search` or `list` on a resource whose id already appeared in an earlier call of the same row. +This is a **surface** property as much as an agent one — identifiers that stayed sticky across +turns would close the gap without either agent changing. Scoped to one row, since each repetition +is a fresh conversation, and the call that first resolves an id is never charged for the lookup +that produced it. A run without recorded arguments reports *not measured* rather than zero: "no +redundant lookups" and "we could not tell" are opposite conclusions. + +### Power + +A run's aggregate and its per-task rows have very different power and appear in the same table. At +2 repetitions a task that passed once is `1/2 UNSTABLE` with a 95% interval of roughly [0.09, 0.91] +— compatible with almost any true rate — while a paired aggregate across 35 tasks can resolve a +difference at p=0.0018. Reports print a POWER line when no task reaches 5 repetitions, stating that +per-task verdicts are not supported at that depth. + +### Provenance: what counts as proof the answer came from the surface + +A read verifier asks two independent questions — is the answer right, and did the agent get +it from the tool surface. This is the definition of the second one. It is a property, not a +list of approved call sequences: an enumeration of routes through 183 actions can never be +complete, and each gap in it fails an agent that answered correctly by an unlisted route. + +**A sentinel proves itself.** A sentinel is a per-run random string a seeder wrote into +Plane — a state name, a work item title, a comment phrase. It exists nowhere else, and the +agent's only route to Plane is the surface under measurement. So if a sentinel appears in a +response the agent received, the agent used the surface. Nothing further is required: not +which entity the request named, not which tool was called, not how many calls it took. + +**A count does not.** Where the seeded truth is a number, presence proves nothing — a small +integer appears by coincidence. Aggregate evidence therefore keeps a target binding: an exact +`total_count` counts only from a request whose arguments name a seeded entity. R2 binds one +project's count; L2 binds a work item's activity count; R6 accepts either one count per +project or one count grouped by project, since both are honest routes to its answer. + +Matching happens while the successful response is in memory, and only a non-sensitive +`observed_sentinels` label is persisted. Read-task provenance does not turn payload recording +back on. CLI proxies receive sentinel lengths and SHA-256 fingerprints, plus the target IDs the +aggregate rule needs, through a mode-0600 run-scoped file outside the agent cwd; the raw value +is absent even from that file. Every MCP proxy session reads the same file, and the driver +removes it with its temporary directory after the run. The response body never enters the +result row, and the evidence machinery persists only the matched label. The sentinel itself +is not secret after the fact: a correct answer to R1 *is* the seeded state name, so it +appears in the recorded `final_text`, and a failing verifier note names the value it wanted. +Result rows are therefore run data, not a redacted artifact. Unavailable or incomplete +matching is diagnosed and cannot pass a read verifier. + +### Threat model: a cooperative agent + +The harness measures an agent that is trying to do the task, not one trying to defeat the +measurement. That is a deliberate scope decision, and the numbers should be read with it in +mind. + +The agent under measurement runs as the same OS user, in the same filesystem, holding the same +Plane credentials as the harness. An agent that wanted to could read the evidence +configuration, edit an earlier row in the result file, fabricate a proxy sidecar session, or +skip MCP and call Plane directly — and off-surface indicators report such signals without +gating success. None of this is closed by any verifier rule; closing it requires running the +agent in a separate trust domain, which is a different project. + +For the question this harness exists to answer — how well does our own tool surface serve a +capable agent — a cooperative agent is the right subject. Treat every integrity property above +as holding against accident and drift, not against an adversary. + +Provider usage is a different measurement: where the driver supplies it, the harness keeps +input, output, cache-read, and cache-creation usage. Tool-result sizing describes one source +of context growth; it is not substituted for the provider's conversation-level usage. + +## Why calls are recorded at the transport boundary + +An agent's final answer is not a reliable call log. It may omit a failed lookup, summarize +several calls as one action, or claim an action it did not perform. Call-count and tool-use +metrics therefore come from execution evidence. + +The API driver owns the MCP session and records each call it executes. The four CLI drivers +put `evals.proxy` between the CLI and the stdio MCP server. The proxy relays JSON-RPC bytes +without reserializing them, pairs `tools/call` requests and responses by JSON-RPC ID, and +records a request sequence on each sidecar row. The sidecar loader restores request order. +A complete proxy sidecar is authoritative; CLI event or transcript parsing is retained as +a fallback when the sidecar is incomplete. Neither source depends on the agent describing +its own behavior. + +The proxy remains standard-library-only because it runs inside the server process tree with +a scrubbed `PYTHONPATH`. It records response payloads only when explicitly requested. +Tokenization and row mapping happen later in the parent harness, where optional dependencies +are safe to import. + +## Driver and backend boundaries + +All five driver implementations satisfy `AgentDriver.run_task(...) -> AgentRun`: + +- `ApiDriver` +- `ClaudeCliDriver` +- `CodexCliDriver` +- `AntigravityCliDriver` +- `OpencodeCliDriver` + +The runner has one path for all drivers: it supplies a prompt and MCP environment, receives a normalized +`AgentRun`, maps it to the common row shape, and invokes the task verifier. + +The API implementation has one further seam. `ApiDriver` owns provider-independent policy: +the stdio MCP session, tool execution loop, iteration budget, timing, result recording, +call-ID pairing, and usage accumulation. A `ModelBackend` owns provider conversation state +and wire format through three operations: + +```text +start(system, prompt, tools) +next_turn() -> Turn +add_tool_results(results) +``` + +This is the narrowest boundary that keeps provider-specific message roles, content blocks, +tool schemas, usage objects, and stop reasons out of the loop. `AnthropicBackend` translates +to the stable Messages API. `OpenAIBackend` translates to Chat Completions function tools +and imports the optional OpenAI SDK only when no client was injected. Both return neutral +turns containing text, tool calls, normalized usage, and a stop reason. + +CLI agents already own their model conversation and tool loop, so they implement +`AgentDriver` directly rather than pretending to be `ModelBackend` implementations. Their +subprocess, configuration, transcript, and usage differences stay within their driver +modules. + +CLI MCP configuration is isolated from ambient user state, but the strength of the +effective-config evidence differs by vendor: + +| Driver | Effective-config exclusivity evidence | +|---|---| +| Claude | **Readback-supported, not behaviorally proven for the evaluated invocation.** Real `claude mcp list` reads the same isolated `.claude.json` and observes only `plane`. The evaluated `claude -p` receives that file plus `--strict-mcp-config`; exclusion of project/ambient MCP servers rests on the CLI's documented strict-config contract, not a forbidden-server probe of that invocation. HOME, `CLAUDE_CONFIG_DIR`, and all XDG roots are isolated. | +| Codex | Proven by real `codex mcp list --json` readback under the isolated Codex home. | +| OpenCode | Proven by real `opencode debug config` readback under isolated HOME/XDG roots and the generated project config. | +| Antigravity | **Unverifiable.** Antigravity CLI has no MCP/effective-config introspection command. The harness relocates agy's whole state tree with the undocumented `--gemini_dir` and inspects generated files, but neither the harness nor this design treats that as observed effective-config exclusivity. HOME is deliberately *not* isolated: agy keeps its OAuth token in the macOS login keychain, which Security resolves under `$HOME/Library/Keychains`, so an isolated HOME made the credential unfindable and every run failed unauthenticated. | + +The Antigravity "unverifiable" regression test is documentation coverage: it guards this +claim, not runtime behavior. Separate behavioral tests cover the isolated gemini dir and +generated file placement, but those still cannot observe Antigravity's effective server set. + +Several loop rules are deliberately centralized in `ApiDriver`: + +- Tool results are paired to model calls by call ID, never by list position. Missing, + duplicate, or unknown IDs set `result_pair_mismatch`. +- A refusal-terminated turn records any included calls for audit but executes none of them. +- `hit_max_iterations` is set only when the iteration budget is exhausted while more tool + work remains, not merely because a valid final response used the last iteration. +- `wall_time_s` covers the model/tool loop after `list_tools`; server startup, teardown, and + post-loop token counting are outside it. +- The row records the requested model token, requested tier (when present), resolved model ID, + and the provider-reported model that actually ran when the provider returns one. + +## Task and run lifecycle + +The task catalog uses plain dictionaries. Each task stays beside its verifier in the module +for its task class. The catalog package assembles those lists in a pinned historical order, +builds `TASKS_BY_ID`, and computes the battery fingerprint. + +For each task repetition, the runner creates a fresh project and only the fixture groups +declared by that task. The live sequence is: + +```text +seed -> drive -> verify -> capture non-secret seed shape -> teardown -> append row +``` + +The row is assembled as the task progresses; teardown runs in `finally` before that row is +appended. Workspace-scoped fixture objects are tracked separately from the project. A fresh +stdio server is launched for each driven task. The server environment is built from `PATH`, +`HOME`, the three Plane connection values, and explicit `--server-env` additions; unrelated parent environment variables +are not inherited. + +Result rows retain only seeded entity kinds and randomization namespaces. They never contain +target entity IDs or randomized truth values. Each repetition has an independent persisted +`fixture_seed_id`, and the randomized truth it derives is recoverable from that seed plus +namespace without exposing any later repetition's independent sentinel. + +That seed is not a replay recipe for the whole fixture. Seeding also reads `date.today()` for +cycle and work-item dates, and identifier collisions retry with fresh `secrets` randomness +drawn outside the seeded namespace, so re-running a seed on another day — or after a +collision — does not reconstruct the same fixture. What the seed guarantees is narrower and is +what it was built for: each repetition's sentinels are independent, so no repetition can leak +another's answer. + +The first line of a new result file is a meta row containing the run identity, label, server, +battery, requested model/tier, resolved model, driver, provider, Git SHA, exact task-id list, +and repetition count. Reports compare raw `(task_id, rep)` histories with that exact set +before latest-wins deduplication. Resume checks run identity and only appends replacements. +A repeated key is valid only when every occurrence except the authoritative last row is +retryable; prior terminal rows remain genuine duplicates and make the run incomplete. +Result rows preserve the common fields consumed by `evals.report` and existing JSONL readers. + +## Module layout + +```text +evals/ + cli.py argparse, command dispatch, and model-tier resolution + __main__.py command entry point for python -m evals + runner/ + __init__.py public execution API + live.py live lifecycle and row assembly + resume.py resume skip and mismatch checks + meta.py run metadata and repository provenance + canary.py empty-agent verifier canary + tasks/ + __init__.py public task API re-exports + catalog.py ordered catalog assembly and fingerprinting + prompts.py task prompt binding + answers.py answer-contract matching + lookups.py Plane reads used to establish verifier truth + skip.py task skip signal + read.py R1-R7 tasks and verifiers + write.py W1-W11 tasks and verifiers + schema.py S1-S5 tasks and verifiers + cross.py C1-C2 tasks and verifiers + debias.py I1-I5 and L1-L5 tasks and verifiers + drivers/ + __init__.py the driver registry, loading only the surface it is asked for + api/ + base.py neutral backend protocol and turn/tool dataclasses + driver.py the owned model/tool loop + anthropic.py Anthropic Messages translation + openai.py OpenAI Chat Completions translation + cli/ + base.py the subprocess template vendors fill in + process.py subprocess lifecycle + sidecar.py recording-proxy command and sidecar handling + claude.py Claude Code CLI driver + codex.py Codex CLI driver + antigravity.py Antigravity CLI driver + opencode.py OpenCode CLI driver + core/ shared floor: may import only core (+ stdlib/third-party) + changelog.py changelog text normalization helpers + errors.py neutral exceptions (TaskSkipped, …) + evidence.py target-binding evidence labels and sentinels + fixtures.py seeded fixture name/title constants + results.py run/task result types and common row mapping + server_env.py stdio MCP server env construction + state_oracle.py Plane state lookups used as verifier truth + task_metadata.py task tags/needs/prompt persisted in the run's meta header + token_counting.py tool-result token sizing + tool_manifest.py tools/list capture and fingerprinting + tool_names.py whose MCP tool a call is, and what to call it + proxy.py stdlib-only JSON-RPC recording relay + seed/ Plane fixture creation and teardown + report/ summaries, A/B comparison, and multi-surface tables + +``` + +Booting a Plane instance to measure against is deliberately outside this tree. The +harness reaches its target through three `EVAL_PLANE_*` variables and knows nothing +else about how that instance runs, so a local plane-ee, a shared staging box, and a +hosted workspace are the same thing to it. + +The stable import and command surfaces are intentional: `from evals.tasks import ...`, +`from evals.drivers import ...`, and `python -m evals` remain the public boundaries even +though their implementations are split across packages and focused modules. diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..0a9eb61 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,356 @@ +# Eval harness — runbook + +Measures how well an LLM agent completes real Plane tasks through an MCP tool surface. +Every live task repetition uses a **live** Plane API with its own seeded fixtures and +teardown. Mutation tasks are verified by reading Plane back; read tasks match the final +answer against facts from the seed context or the API rather than trusting the agent's +claim that it succeeded. + +The harness is agent-agnostic and surface-agnostic: any stdio MCP server can be measured +(`--server-cmd`), driven by any of five driver implementations. `DESIGN.md` explains why it is +built this way; this file is how to run it. + +What you get per task: pass/fail, observed calls to done, core and variable tool use across +successful repetitions, response size and token-count provenance, errors, and the agent's +final text. + +## Prerequisites + +1. **A reachable Plane instance and API key.** Any reachable instance works, local or + hosted. The key must be able to create and delete the catalog's project and + workspace-scoped fixtures. Genuine plan or feature gates are recorded as skips where + the fixture code handles them. Configure the harness with exactly these three values: + + ```bash + export EVAL_PLANE_BASE_URL=https://your-plane.example.com + export EVAL_PLANE_WORKSPACE_SLUG=your-workspace-slug + export EVAL_PLANE_API_KEY=plane_api_your_key + ``` + +2. **Model access for the driver you pick.** The API driver uses + `ANTHROPIC_API_KEY` by default; OpenAI requires its SDK and `OPENAI_API_KEY`. + CLI drivers require their corresponding local CLI to already be authenticated. + +## Running + +```bash +# Provider-neutral API loop (default provider: Anthropic) +.venv/bin/python -m evals --driver api --provider anthropic --model standard \ + --label local --out evals/output/api.jsonl + +# Everything, one surface (free-form model IDs pass through to the CLI) +.venv/bin/python -m evals --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label local --out evals/output/local.jsonl + +# A few tasks while iterating +.venv/bin/python -m evals --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label local --tasks W5,W8 --out evals/output/spot.jsonl + +# Someone else's server (a PR branch, another repo) — "external mode" +.venv/bin/python -m evals --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ + --server-env KEY=VALUE --out evals/output/their-pr.jsonl +``` + +Useful flags: `--reps N` (repetitions per task), `--resume out.jsonl` (skip completed +`(task, rep, label)` keys and plan-gated skips; retry recorded errors, cleanup failures, +fixture-collision skips, and unknown skips), `--list` / `--dry-run` (no network). + +**External mode** (`--server-cmd`) records `server: "external"`. Success, call counts, +errors, and observed tool distributions use the same rules as local-server rows. + +### Drivers + +| Driver | Backend | Notes | +|---|---|---| +| `api` | Owned API + MCP loop | Provider-neutral; tiers resolve for `--provider anthropic` (default) or `openai` | +| `codex-cli` | OpenAI Codex CLI | `standard` and `fast` resolve to verified GPT-5.6 IDs | +| `claude-cli` | Claude Code CLI | `standard` resolves to `sonnet`; `fast` resolves to `haiku`; isolated HOME/config/XDG roots and strict MCP config | +| `antigravity-cli` | Antigravity CLI (`agy`) | Verified against `agy models`; isolated via `--gemini_dir` with HOME left real (agy's token lives in the macOS login keychain), but agy has no effective-config readback, so exclusivity is unverifiable | +| `opencode-cli` | OpenCode | Tiers are intentionally unmapped; pass an explicit ID listed by `opencode models` | + +### Model tiers + +The harness has exactly two provider-neutral tiers: `standard`, the workhorse used for the +battery, and `fast`, the lower-cost option. Resolution is scoped to both driver and provider: + +| Driver | Provider | `standard` | `fast` | +|---|---|---|---| +| `api` | Anthropic | `claude-sonnet-5` | `claude-haiku-4-5` | +| `api` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | +| `claude-cli` | Anthropic | `sonnet` | `haiku` | +| `codex-cli` | OpenAI | `gpt-5.6-sol` | `gpt-5.6-luna` | +| `antigravity-cli` | Google | `gemini-3.6-flash-high` | `gemini-3.6-flash-low` | +| `opencode-cli` | Project-configured | **unmapped** | **unmapped** | + +Only `standard` and `fast` are harness vocabulary. Every other `--model` value passes through +unchanged, including vendor aliases such as `sonnet` / `haiku` and full IDs such as +`claude-opus-5`, `gpt-5.6-sol`, or `openai/gpt-5.6-sol`. An unmapped tier fails before the run +and tells you to pass an explicit model ID; the harness never guesses one. + +Result and meta rows keep `requested_model`, `requested_tier`, and `resolved_model` separately. +The compatibility field `model` remains the provider-reported model when available, otherwise +the resolved ID. This makes old readers continue to work while preserving which tier produced +the row even after its mapping changes. + +Every CLI driver records the actual JSON-RPC traffic through a recording proxy, so tool +calls are normally counted from the wire rather than from whatever the agent claims it did. +If a sidecar is incomplete, the driver can fall back to its CLI event stream or transcript. +Claude transcripts used for fallback are copied out of disposable per-task config into +`.artifacts/claude-cli/` before the row is written, so `driver_raw_ref` remains +resolvable. A file-credential copy failure aborts the task before Claude starts. Refreshed +file credentials are intentionally not copied back into user auth because that would mutate +user state and create cross-task/concurrent refresh races; Claude rows record this limitation. + +The API driver executes MCP calls itself, records exact result character counts, and sizes +result tokens without making a provider request per result. A backend may supply a token +counter; otherwise rows set `result_tokens_estimated: true` and use a deterministic +character-based estimate. CLI drivers use that same shared estimate from the result character +counts in the recording sidecar, so every driver reports response-token cost and estimated +counts are explicitly marked. + +Exact CLI-side counting is opt-in with `--record-result-payloads`. It makes the proxy retain +the serialized tool-result text long enough for the harness to count it with a locally +importable tokenizer (`tiktoken`/`cl100k_base`); if that tokenizer is unavailable, the harness +falls back to the same marked estimate. The default stays **off** because payloads contain live +workspace data and bloat the sidecar. For comparing tool surfaces, the default chars-derived +estimate is monotonic in the thing being compared anyway. Do not enable payload recording by +habit; use it only when the more sensitive, larger sidecar is justified. + +Read-task provenance is stricter than “a call happened.” Seeders place a hidden per-run +sentinel — a random string that exists only inside Plane — and the API driver or CLI proxy +records whether a successful response exposed it. Because the agent's only route to Plane is +the surface under measurement, a sentinel in a response it received is proof of surface use by +itself; the harness deliberately does not also require the request to have named a particular +entity, because that rejected honest routes to the same answer. Where the seeded truth is a +count rather than a string, presence proves nothing and the target binding still applies: an +exact `total_count` counts only from a request naming a seeded entity. `evals/DESIGN.md` states +both rules and the threat model they hold under. + +CLI proxies receive one-way value fingerprints, plus the target IDs the count rule needs, +through a private run-scoped file; the raw sentinel is absent even if that file is inspected. +Every proxy session can read it, and the driver removes it with its temporary directory after +the run. The evidence machinery records the matched label and never the response body. The +sentinel value can still reach a row by the front door: a correct answer often *is* the seeded +value, so it appears in `final_text`, and a failing verifier note names what it expected. +Unavailable or incomplete matching is diagnosed and fails closed. + +### Reading results + +```bash +.venv/bin/python -m evals.report evals/output/local.jsonl # one surface +.venv/bin/python -m evals.report --table evals/output/*.jsonl # side by side +.venv/bin/python -m evals.report --table --markdown evals/output/*.jsonl # for a PR +``` + +Rows are deduped latest-wins per `(task_id, rep, label)`, so a re-run of a single task +supersedes its earlier row in the same file. Skipped tasks are excluded from success +denominators, as are rows with recorded errors. Result-token columns use `~` for estimates, +`*` for mixed measured/estimated values, and `?` for legacy values whose provenance was not +recorded. + +Reports keep three verdicts separate: model success among evaluated rows, execution coverage +(evaluated rows / expected rows, including skipped task IDs and capability reasons), and run +completeness. A plan-gated-only run can therefore be **RUN COMPLETE** below 100% execution +coverage. The live runner and report commands exit 0 when the evaluation completed cleanly; +exit 0 does **not** mean the agent passed. Callers that need a pass-threshold exit must apply +that as a separate opt-in policy rather than overloading the completeness status. + +With `--reps N`, each `(task, rep)` is independently seeded, run, verified, and torn down. +Multi-rep reports show each task's pass count, Wilson interval, and whether its pass/fail +answer changed across completed repetitions. Instability remains descriptive; it is not +converted into an ad-hoc threshold for declaring surface differences meaningful. Two-file +A/B reports instead pair shared tasks, report a paired-bootstrap 95% interval for the mean +per-task success-rate difference, and use a paired sign-flip permutation test for mean call +deltas. Zero call-delta ties remain in that paired sample. The inference treats tasks as +independent sampling units and assumes comparable task instances under both labels, so the +printed paired task count—and the resulting wide interval for small samples—matters. + +Every path that reports call cost also reports errored-call friction on the same successful, +trace-intact rows: an absolute per-task median and an errored/total call rate, with paired task +deltas in A/B output and task IDs for investigation. This is a proxy, not a pure schema-error +counter: MCP `is_error` also marks correct expected failures, while a successful call to the +wrong tool is invisible to it. + +Because one count conflates three different things, the same reports break errored calls into +**surface friction** (a well-formed call the API refused on meaning — the number to act on), +**navigation** (the schema correcting a malformed call), **answered existence questions** (a +first absent read, which is an answer rather than an obstacle), plus `other` and +`unclassified`. A non-zero `unclassified` prints a "split incomplete" line, so zero surface +friction never stands in for nothing having been classified — including when reading a result +file recorded before the split existed. See DESIGN.md for the classification rules. + +A refusal the server hands back as a *successful* result is counted too, reported on its own +line (`N refusal(s) arrived flagged as successful results`) because it cannot join a total keyed +on the protocol's error flag. It is worth watching: one measured surface refused roughly twice +as often as its errored-call count implied. + +Request args (`args_json`) are recorded on every driver, not just under +`--record-result-payloads`. A recorded result whose target is unknown cannot say *which* item a +call acted on — exactly the question a failing write raises — and the redundant-lookup metric +cannot be computed without them. Result *payloads* remain opt-in; args are ids and short strings. + +Every result row carries a `battery` fingerprint derived from the selected catalog's task IDs, +prompts, and catalog revision, plus a `task_fingerprint` over that row's task ID, prompt, and +fixture names. The battery contains exactly what the agent is asked and no expectation about +how the answer should be produced. All report paths refuse with exit 2 before printing +measurements when persisted battery identities differ, including mixed rows within one file. + +Canonical model, provider, driver, and server differences are comparison treatments. Declare +each intentional difference with `--vary`, for example `--vary resolved_model` or +`--vary provider,resolved_model`; any undeclared difference also refuses. Battery cannot be +declared as varying. Requested tier/model names remain provenance rather than canonical +identity, and the provider-reported realized model is printed as evidence instead of being +mechanically equated with the configured model. + +The hash excludes fixtures and verifier bodies. `CATALOG_REVISION` in `tasks/catalog.py` +closes that gap: bump it whenever an excluded change redefines what a task asks, and explain +the comparison consequence in its docstring. + +## Running surfaces in parallel + +Tasks that touch **workspace-scoped** fixtures (release tags, customer properties) collide +if two runs share a workspace. Give each concurrent run its own workspace: + +```bash +EVAL_PLANE_WORKSPACE_SLUG=ws1 .venv/bin/python -m evals \ + --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label local --out evals/output/local.jsonl & +EVAL_PLANE_WORKSPACE_SLUG=ws2 .venv/bin/python -m evals \ + --driver codex-cli --model YOUR_CODEX_MODEL_ID \ + --label their-pr --server-cmd "/path/to/their/.venv/bin/plane-mcp-server stdio" \ + --out evals/output/their-pr.jsonl & +wait +``` + +Keep the workspaces **empty apart from eval fixtures**. Unrelated projects and work items +in one workspace but not another skew every workspace-wide task in that column. + +## Adding a task + +Tasks live in the `evals/tasks/` package, grouped by task class and kept beside their +verifiers: + +- `read.py`: R1-R7 +- `write.py`: W1-W11 +- `schema.py`: S1-S5 +- `cross.py`: C1-C2 +- `debias.py`: I1-I5 and L1-L5 + +Prompt binding, answer matching, Plane lookups, and the skip signal live in `prompts.py`, +`answers.py`, `lookups.py`, and `skip.py`. `catalog.py` assembles the class lists in the +pinned catalog order, while `tasks/__init__.py` re-exports the public task API. +A task is a dict: + +```python +{ + "id": "W11", + "tags": {"write", "tier1"}, + "prompt": f"In project {{project}}, ...", # {project} is bound at run time + "needs": {"items", "cycles"}, # fixtures to seed + "verify": verify_w11, +} +``` + +`needs` tokens: `items`, `labels`, `bug_type`, `cycles`, `cycles_open_past`, `module`, +`intake`, `customer`, `release`, `activity_feed`, `second_project`, +`leave_cycles_worklogs_off` (S5: cycles + worklogs + workspace customers off), +`leave_worklogs_off` (W11: worklogs only). Each task gets its own freshly seeded project, so fixture +variants (e.g. `cycles_open_past`) don't leak between tasks. + +### Writing a verifier + +Verifiers are `async def verify_x(plane, ctx, run) -> (ok: bool, note: str)`. Keep a new task +and its verifier in the same class module, add it to that module's exported task list, and +preserve the assembly order in `tasks/catalog.py`. + +Mutation verifiers must read the resulting state through the Plane API. Read verifiers must +derive the expected facts from the API or seed context and match an explicit answer contract +instead of scanning free-form prose. Use exact `field: value` lines and the shared contract +matchers; for numeric answers, prefer a prompt such as `Answer with a line 'count: N'`. A +loose substring can make `4` match `24`, and prose matching can accidentally grade an agent's +writing habits instead of its answer. + +**Check the shape the API actually returns.** Dates come back as timestamps +(`2026-08-12T00:00:00Z`), so comparing one to a bare `2026-08-12` silently never matches — +a verifier that can only fail is worse than no verifier. Have the test stub return the real +shape. + +Then prove the verifier can fail: + +```bash +.venv/bin/python -m evals --canary --label local +# CI capability contract: these ids must be eligible and verified. +.venv/bin/python -m evals --canary --canary-strict R1,R2,W8 --label local +``` + +The canary seeds each task, calls its verifier with both an **empty** agent result and +plausible zero-call canned contract answers, then reports verified, skipped, and errored +task ids separately. It exits non-zero for a false pass, verifier/teardown error, zero +verified tasks, or a skipped id named by `--canary-strict`. Plan-gated skips outside that +explicit strict set remain allowed. Run it after touching tasks, fixtures, or verifiers. + +**Make the task achievable before blaming a surface.** For example, W6 declares the +`cycles_open_past` fixture variant because it asks the agent to close Sprint 12; the seeder +must not pre-close the cycle that the task is meant to change. + +## Running a local Plane + +Any reachable Plane works, so how you get one is your own setup and is not kept in this +repo. If you run plane-ee locally, two things make it usable for evals: + +- Raise `API_KEY_RATE_LIMIT`; a full battery makes far more API calls than the default + allows. +- Optionally point `FEATURE_FLAG_SERVER_BASE_URL` at a flag server that answers every + flag as on. This is **not** required. A capability the plan excludes makes its task + record `env:plan-gated:` and drop out of the denominator, the same way L2 + handles a missing activity worker; the rest of the battery runs unaffected. Pointing at + a permissive flag server simply means those tasks are measured rather than skipped. + + Which tasks that covers: C2 (releases), L4 (customers), R6 and S1 (work item types). + +Keep such scripts outside version control — `localdev/` is ignored for exactly this. + +## Local gotchas + +- If seeded comments do not materialize as activities, the activity-feed task self-skips + with `env:no-activity-worker` rather than failing the agent. +- A reviewed capability the workspace's plan excludes self-skips with + `env:plan-gated:`. The closed allowlist is `customers`, `releases`, + `work-item-types`, `initiatives`, and `teamspaces`; a typo or new name is unexpected + until its real gate site is reviewed and the allowlist is deliberately extended. + Only a refusal that names a plan limit counts: 402, or 403/400 whose body says so. A + bare 403 is an ordinary permission denial and stays a real error, because classifying it + as a gate would let a permission bug leave the denominator and read as "nothing to see". +- Run completeness uses an explicit skip taxonomy: known capabilities the environment does + not provide (an allowlisted `env:plan-gated:` or the exact reason + `env:no-activity-worker`) are expected skips. + The task/capability pair must also match the task's declared fixture needs: for example, + `env:plan-gated:customers` is expected for L4 but unexpected for W1 or C1. + They reduce **EXECUTION COVERAGE** but do not break **RUN COMPLETE**. A dirty environment + (`env:fixture-collision:*`) and every unrecognised reason are unexpected and make the run + incomplete; there is intentionally no catch-all for new `env:*` reasons. +- New result headers declare the exact task-id subset and repetition count. Completeness + compares raw `(task_id, rep)` occurrences with that declaration before latest-wins + deduplication, naming missing and unexpected keys (including duplicate excess). +- Report headlines use a task-cluster bootstrap interval; the pooled repetition rate and + Wilson interval remain visible but are explicitly labeled as pooled. A/B and multi-file + surface-table reports refuse a comparison when any input lacks a tool-manifest + fingerprint; missing is an explicit unidentified value, not a wildcard. +- A feature switched **off for a project** is not a plan gate — it is configuration the + harness sets itself, and W11 exists to measure what an agent does when it meets one. +- **Gated endpoints returning 402 on a workspace that should work.** Feature flags are + cached per workspace, and the cache does not record which flag server answered. Any + process that touches the DB while pointed at a *different* flag server than the running + API — sourcing `plane-ee/apps/api/.env` gets you the remote one, where these flags are + off — caches that answer for the workspace, and + the API then serves the cached miss instead of asking its own mock. The tell is a canary + that reports every verifier broken at once. Clear `ff::*` and rotate + `ff_ver:` (`plane.payment.flags.cache`) and the next request refetches. Observed + while creating a workspace from a Django shell; a plan/licence problem looks identical + from the outside, so check this first. +- A workspace licence is **not** required for local runs. With a permissive flag server an + unlicensed workspace seeds every fixture (verified by canary against a workspace with no + licence row); without one, the gated tasks skip and the rest still run. +- **Offline tests** cover the harness itself and need no Plane instance: + `env -u REDIS_HOST -u REDIS_PORT .venv/bin/python -m pytest -q --ignore=tests/test_integration.py` diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 0000000..4347f13 --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1,9 @@ +"""Plane MCP tool-surface eval harness.""" + +from pathlib import Path + +# Repository root: the harness launches this repo's MCP server and resolves +# task working directories against it. +REPO_ROOT = Path(__file__).resolve().parent.parent + +__all__ = ["REPO_ROOT"] diff --git a/evals/__main__.py b/evals/__main__.py new file mode 100644 index 0000000..97248a5 --- /dev/null +++ b/evals/__main__.py @@ -0,0 +1,6 @@ +"""Command entry point: python -m evals""" + +from evals.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/cleanup.py b/evals/cleanup.py new file mode 100644 index 0000000..712e8b1 --- /dev/null +++ b/evals/cleanup.py @@ -0,0 +1,285 @@ +"""Delete leftover eval projects or fixed-name workspace sentinels. + +``python -m evals.cleanup [--prefix "EVAL " | --sentinels] [--yes]`` — dry-run lists only; +``--yes`` is required before anything is deleted. Credentials come from EVAL_PLANE_*. +""" + +from __future__ import annotations + +import argparse +import sys +from typing import Any + +from plane.models.query_params import PaginatedQueryParams + +from evals.seed.customers import ( + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, +) +from evals.seed.item_types import ( + BUG_TYPE_NAME, + FIXTURE_WORK_ITEM_TYPE_NAMES, + is_severity_property, + is_work_item_type_named, + list_workspace_properties_for_type, + list_workspace_work_item_types, +) +from evals.seed.releases import EVALUATION_RELEASE_TAG_VERSION +from evals.seed.workspace import list_workspace_rows + + +def list_projects_with_prefix(plane: Any, workspace_slug: str, prefix: str) -> list[Any]: + """Return projects whose name starts with ``prefix`` (paginated list). + + Matches the SDK contract used elsewhere in the repo: pass + ``params=PaginatedQueryParams(...)`` and stop when ``not page.next_page_results``. + Do not fall back on ``next_cursor`` alone — the SDK always populates it. + """ + matches: list[Any] = [] + cursor = None + while True: + params = PaginatedQueryParams(per_page=100, cursor=cursor) + page = plane.projects.list(workspace_slug=workspace_slug, params=params) + results = page.results if hasattr(page, "results") else page + for proj in results or []: + # Prefix may include a trailing space (default "EVAL ") so "EVALUATION" is excluded. + name = getattr(proj, "name", None) or "" + if name.startswith(prefix): + matches.append(proj) + if not getattr(page, "next_page_results", False): + break + cursor = page.next_cursor + return matches + + +def delete_projects( + plane: Any, + workspace_slug: str, + projects: list[Any], + *, + yes: bool, +) -> tuple[int, int]: + """Delete projects when yes=True. Returns (deleted, failed). Dry-run: (0, 0).""" + if not yes: + return 0, 0 + deleted = failed = 0 + for proj in projects: + pid = getattr(proj, "id", None) + name = getattr(proj, "name", pid) + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=pid) + deleted += 1 + print(f" deleted {name!r} ({pid})") + except Exception as exc: + failed += 1 + print(f" FAILED {name!r} ({pid}): {exc}", file=sys.stderr) + return deleted, failed + + +def list_sentinel_workspace_artifacts(plane: Any, workspace_slug: str) -> list[dict[str, Any]]: + """Return fixed-name workspace fixtures that can false-pass eval tasks.""" + customers = plane.customers + specs = ( + ( + "customer", + customers, + lambda row: is_evaluation_customer_name(getattr(row, "name", None)), + lambda row: (getattr(row, "name", None) or "").strip(), + ), + ( + "release_tag", + plane.releases.tags, + lambda row: (getattr(row, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION, + lambda row: (getattr(row, "version", None) or "").strip(), + ), + ( + "customer_property", + customers.properties, + lambda row: ( + (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip().casefold() + == EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + ), + lambda row: (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip(), + ), + ) + artifacts: list[dict[str, Any]] = [] + for kind, api, matches, display_name in specs: + for row in list_workspace_rows(api, workspace_slug): + object_id = getattr(row, "id", None) + if object_id is not None and matches(row): + artifacts.append({"kind": kind, "id": object_id, "name": display_name(row)}) + + type_api = getattr(plane, "workspace_work_item_types", None) + if callable(getattr(type_api, "list", None)): + for row in list_workspace_work_item_types(plane, workspace_slug): + object_id = getattr(row, "id", None) + if object_id is None: + continue + # Every fixture name, not just Incident. Bug used to be reachable only as the type + # whose Severity property gets removed, so leftover Bug types were both undeletable + # by this tool and counted as "nothing to delete" -- a workspace reported clean while + # holding types that skew any task reading the workspace-level list. Duplicates of + # one name each match, so a double-seeded type is fully removed. + matched = next( + (name for name in FIXTURE_WORK_ITEM_TYPE_NAMES if is_work_item_type_named(row, name)), + None, + ) + if matched is not None: + artifacts.append({"kind": "work_item_type", "id": object_id, "name": matched}) + + property_api = getattr(plane, "workspace_work_item_properties", None) + links_api = getattr(type_api, "properties", None) + if callable(getattr(property_api, "list", None)) and callable(getattr(links_api, "list", None)): + for row in list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME): + object_id = getattr(row, "id", None) + if object_id is not None and is_severity_property(row): + display = getattr(row, "display_name", None) or getattr(row, "name", None) or "" + artifacts.append({"kind": "work_item_property", "id": object_id, "name": display.strip()}) + return artifacts + + +def list_unowned_workspace_work_item_types(plane: Any, workspace_slug: str) -> list[dict[str, Any]]: + """Return workspace-level work item types this harness never creates. + + Reported rather than deleted by default: a type the harness did not create may be a real + workspace's configuration, and this runs against instances it does not own. They still + have to be *visible*, because a workspace holding types another workspace lacks skews + every task that reads the workspace-level list, and silence there reads as clean. + """ + type_api = getattr(plane, "workspace_work_item_types", None) + if not callable(getattr(type_api, "list", None)): + return [] + unowned: list[dict[str, Any]] = [] + for row in list_workspace_work_item_types(plane, workspace_slug): + object_id = getattr(row, "id", None) + if object_id is None: + continue + if any(is_work_item_type_named(row, name) for name in FIXTURE_WORK_ITEM_TYPE_NAMES): + continue + unowned.append({"kind": "work_item_type", "id": object_id, "name": (getattr(row, "name", "") or "").strip()}) + return unowned + + +def _sentinel_description(artifact: dict[str, Any]) -> str: + kind = str(artifact["kind"]).replace("_", " ") + return f"{kind} {artifact['name']!r} ({artifact['id']})" + + +def delete_sentinel_workspace_artifacts( + plane: Any, + workspace_slug: str, + artifacts: list[dict[str, Any]], + *, + yes: bool, +) -> tuple[int, int]: + """Delete explicitly selected sentinel artifacts. Returns (deleted, failed).""" + if not yes: + return 0, 0 + deleted = failed = 0 + for artifact in artifacts: + try: + if artifact["kind"] == "customer": + plane.customers.delete(workspace_slug=workspace_slug, customer_id=artifact["id"]) + elif artifact["kind"] == "release_tag": + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=artifact["id"]) + elif artifact["kind"] == "customer_property": + plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=artifact["id"]) + elif artifact["kind"] == "work_item_type": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=artifact["id"]) + elif artifact["kind"] == "work_item_property": + plane.workspace_work_item_properties.delete( + workspace_slug=workspace_slug, + property_id=artifact["id"], + ) + else: + raise ValueError(f"unknown sentinel kind: {artifact['kind']}") + deleted += 1 + print(f" deleted sentinel {_sentinel_description(artifact)}") + except Exception as exc: + failed += 1 + print(f" FAILED sentinel {_sentinel_description(artifact)}: {exc}", file=sys.stderr) + return deleted, failed + + +def _cleanup_sentinels(plane: Any, workspace_slug: str, *, yes: bool, unowned: bool = False) -> int: + artifacts = list_sentinel_workspace_artifacts(plane, workspace_slug) + others = list_unowned_workspace_work_item_types(plane, workspace_slug) + if unowned: + artifacts = artifacts + others + print(f"workspace={workspace_slug} sentinel_matches={len(artifacts)}") + if others and not unowned: + # Never let a zero match count imply a clean workspace while these sit here. + print(f"note: {len(others)} workspace work item type(s) present that this tool did not create:") + for artifact in others: + print(f" {_sentinel_description(artifact)}") + print(" add --unowned to delete them too") + if not artifacts: + print("nothing to delete") + return 0 + if not yes: + for artifact in artifacts: + print(f" would delete sentinel {_sentinel_description(artifact)}") + print("dry-run: re-run with --sentinels --yes to delete these sentinel fixture(s)") + return 0 + deleted, failed = delete_sentinel_workspace_artifacts(plane, workspace_slug, artifacts, yes=True) + print(f"summary: deleted={deleted} failed={failed} matched={len(artifacts)}") + return 1 if failed else 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description="Clean up leftover eval fixtures (dry-run by default)") + p.add_argument("--prefix", type=str, default="EVAL ", help='Project name prefix (default: "EVAL ")') + p.add_argument( + "--sentinels", + action="store_true", + help="Clean fixed-name workspace sentinels instead of projects", + ) + p.add_argument( + "--unowned", + action="store_true", + help="With --sentinels, also delete workspace work item types this harness never creates", + ) + p.add_argument( + "--yes", + action="store_true", + help="Actually delete matched objects (default is dry-run list only)", + ) + args = p.parse_args(argv) + + from evals.seed import make_plane_client + + try: + plane, workspace_slug = make_plane_client() + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + if args.sentinels: + return _cleanup_sentinels(plane, workspace_slug, yes=args.yes, unowned=args.unowned) + if args.unowned: + print("error: --unowned only applies with --sentinels", file=sys.stderr) + return 2 + + projects = list_projects_with_prefix(plane, workspace_slug, args.prefix) + print(f"workspace={workspace_slug} prefix={args.prefix!r} matches={len(projects)}") + for proj in projects: + pid = getattr(proj, "id", "?") + name = getattr(proj, "name", "?") + ident = getattr(proj, "identifier", "") + print(f" {name!r} id={pid} identifier={ident}") + + if not projects: + print("nothing to delete") + return 0 + + if not args.yes: + print(f"dry-run: would delete {len(projects)} project(s); re-run with --yes to delete") + return 0 + + deleted, failed = delete_projects(plane, workspace_slug, projects, yes=True) + print(f"summary: deleted={deleted} failed={failed} matched={len(projects)}") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/cli.py b/evals/cli.py new file mode 100644 index 0000000..900df96 --- /dev/null +++ b/evals/cli.py @@ -0,0 +1,341 @@ +"""Command-line wiring and model resolution for the eval harness.""" + +from __future__ import annotations + +import argparse +import asyncio +import shlex +import sys +import uuid +from pathlib import Path +from typing import Any + +from evals.drivers import KNOWN_DRIVERS +from evals.drivers.api import ( + KNOWN_API_PROVIDERS, + MODEL_TIERS, + UnmappedModelTierError, + backend_model_aliases, + resolve_backend_model, +) +from evals.runner import run_canary, run_live +from evals.seed import seed_plan +from evals.tasks.catalog import TASKS, get_tasks +from evals.tasks.prompts import format_task_prompt + +API_MODEL_TIERS: dict[str, dict[str, str]] = { + provider: aliases for provider in KNOWN_API_PROVIDERS if (aliases := backend_model_aliases(provider)) +} +# CLI drivers have an implicit provider selected by their own authentication +# and configuration. Keep the provider dimension explicit so a tier never +# crosses vendor boundaries by accident. +CLI_DRIVER_PROVIDERS: dict[str, str | None] = { + "claude-cli": "anthropic", + "codex-cli": "openai", + "antigravity-cli": "google", + # OpenCode is multi-provider and location-configured. Its installed catalog + # is the only reliable source, so the harness does not guess a default. + "opencode-cli": None, +} +CLI_MODEL_TIERS: dict[str, dict[str, dict[str, str]]] = { + "claude-cli": { + "anthropic": {"standard": "sonnet", "fast": "haiku"}, + }, + "codex-cli": { + "openai": backend_model_aliases("openai"), + }, + "antigravity-cli": { + "google": { + "standard": "gemini-3.6-flash-high", + "fast": "gemini-3.6-flash-low", + }, + }, + "opencode-cli": {}, +} + +DEFAULT_OUT_DIR = Path(__file__).resolve().parent / "output" + + +def resolve_model_for_driver(driver_name: str, model: str, *, provider: str | None = None) -> str: + """Resolve a harness tier for a driver/provider, or pass a model ID through. + + Only ``standard`` and ``fast`` are tier names. Any other string, including + vendor aliases and qualified provider/model IDs, is passed through exactly. + """ + key = (driver_name or "api").strip().lower() + if key == "api": + return resolve_backend_model(provider or "anthropic", model) + if model not in MODEL_TIERS: + return model + if key not in CLI_DRIVER_PROVIDERS: + raise ValueError(f"unknown driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}") + provider_id = provider.strip().lower() if provider else CLI_DRIVER_PROVIDERS[key] + if provider_id is None: + raise UnmappedModelTierError( + f"model tier {model!r} is not mapped for driver {key!r}; OpenCode models depend on " + "the providers configured for this project. Pass an explicit provider/model ID with " + "--model, using one listed by 'opencode models'" + ) + table = CLI_MODEL_TIERS.get(key, {}).get(provider_id, {}) + try: + return table[model] + except KeyError as exc: + raise UnmappedModelTierError( + f"model tier {model!r} is not mapped for driver {key!r} and provider {provider_id!r}; " + "pass an explicit model ID with --model" + ) from exc + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Plane MCP tool-surface eval harness") + p.add_argument("--list", action="store_true", help="Print task table (no network)") + p.add_argument("--dry-run", action="store_true", help="Print resolved prompts + seed plan (no network)") + p.add_argument("--tasks", type=str, default=None, help="Comma-separated task ids (default: all)") + p.add_argument( + "--model", + type=str, + default="standard", + help=( + "Harness tier (standard/fast) or a free-form model ID. " + "Tiers resolve per driver and provider; all other strings pass through unchanged." + ), + ) + p.add_argument("--reps", type=int, default=1, help="Repetitions per task") + p.add_argument( + "--label", + type=str, + default="local", + help="Column label for this run in reports (default: local).", + ) + p.add_argument( + "--server-cmd", + type=str, + default=None, + help=( + "External MCP stdio server launch command (shlex-split). Enables external " + "mode while retaining the same observed tool-use metrics." + ), + ) + p.add_argument( + "--server-env", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra env var for the (external) MCP server child; repeatable.", + ) + p.add_argument( + "--driver", + type=str, + default="api", + choices=sorted(KNOWN_DRIVERS), + help=( + "Agent backend: api | claude-cli | codex-cli | antigravity-cli | opencode-cli. Not required for --canary." + ), + ) + p.add_argument( + "--provider", + type=str, + default="anthropic", + choices=sorted(KNOWN_API_PROVIDERS), + help="Model API provider for --driver api (default: anthropic).", + ) + p.add_argument( + "--record-result-payloads", + action="store_true", + help=( + "CLI drivers only: record serialized tool-result text for tokenizer counting, and " + "the request args beside it so a recorded result can be attributed to its target " + "(off by default; sidecars and rows may contain live workspace data)" + ), + ) + p.add_argument( + "--max-iterations", + type=int, + default=None, + help=( + "API driver only: cap on agent loop iterations (default 15). CLI drivers discard " + "this and let their own loop decide, so a cap that binds penalises only the API " + "side; raise it when comparing an API arm against a CLI arm of the same model." + ), + ) + p.add_argument("--out", type=str, default=None, help="JSONL output path") + p.add_argument( + "--resume", + type=str, + default=None, + metavar="OUT.jsonl", + help=( + "Resume into an existing JSONL (also the --out target). Skip " + "(task_id, rep, label) keys that completed or were plan-gated; re-run rows " + "with errors, cleanup failures, fixture collisions, or unknown skips." + ), + ) + p.add_argument( + "--canary", + action="store_true", + help=( + "Verifier canary: seed each task, call verify with an empty agent result " + "(no driver/model), teardown. Exit 1 if any verifier returns ok=True on do-nothing." + ), + ) + p.add_argument( + "--canary-strict", + type=str, + default=None, + metavar="TASK_IDS", + help=( + "Strict canary coverage: comma-separated task ids that must be verified. " + "Plan-gated skips outside this explicit eligible set remain allowed." + ), + ) + return p.parse_args(argv) + + +def _task_ids(raw: str | None) -> list[str] | None: + if raw is None: + return None + return [t.strip() for t in raw.split(",") if t.strip()] + + +def cmd_list() -> int: + print(f"{'id':<6} {'tags':<18} prompt") + print("-" * 100) + for task in TASKS: + tags = ",".join(sorted(task["tags"])) + prompt = task["prompt"].replace("\n", " ") + if len(prompt) > 70: + prompt = prompt[:67] + "..." + print(f"{task['id']:<6} {tags:<18} {prompt}") + return 0 + + +def cmd_dry_run(tasks: list[dict[str, Any]]) -> int: + needs: set[str] = set() + for task in tasks: + needs |= set(task.get("needs") or set()) + print("Seed plan:") + for line in seed_plan(needs): + print(f" {line}") + print() + sample_ctx = {"project_name": "EVAL deadbeef"} + for task in tasks: + resolved = format_task_prompt(task, sample_ctx, strict=False) + print(f"=== {task['id']} ===") + print(f"needs: {sorted(task.get('needs') or [])}") + print(f"author: {task.get('author') or 'claude'}") + print(f"prompt:\n {resolved}") + print() + return 0 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + + if args.list: + return cmd_list() + + ids = _task_ids(args.tasks) + try: + tasks = get_tasks(ids) + except SystemExit as exc: + print(exc, file=sys.stderr) + return 2 + + if args.dry_run: + return cmd_dry_run(tasks) + + if args.reps < 1: + print("error: --reps must be at least 1", file=sys.stderr) + return 2 + + label = (args.label or "local").strip() or "local" + server_cmd: list[str] | None = None + if args.server_cmd: + server_cmd = shlex.split(args.server_cmd) + if not server_cmd: + print("error: --server-cmd is empty", file=sys.stderr) + return 2 + + server_env: dict[str, str] = {} + for pair in args.server_env: + key, sep, val = pair.partition("=") + if not sep or not key: + print(f"error: --server-env expects KEY=VAL, got {pair!r}", file=sys.stderr) + return 2 + server_env[key] = val + + # Canary: live env only — no driver/model required. + if args.canary: + required_ids = set(_task_ids(args.canary_strict) or []) + if args.canary_strict is not None and not required_ids: + print("error: --canary-strict requires at least one task id", file=sys.stderr) + return 2 + known_ids = {str(task["id"]) for task in TASKS} + unknown_required = sorted(required_ids - known_ids) + if unknown_required: + print(f"error: unknown --canary-strict task id(s): {', '.join(unknown_required)}", file=sys.stderr) + return 2 + return asyncio.run(run_canary(tasks, label=label, required_task_ids=required_ids)) + if args.canary_strict is not None: + print("error: --canary-strict requires --canary", file=sys.stderr) + return 2 + + driver_name = (getattr(args, "driver", None) or "api").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + + if args.resume: + out = Path(args.resume) + elif args.out: + out = Path(args.out) + else: + out = DEFAULT_OUT_DIR / f"{uuid.uuid4().hex}.jsonl" + + try: + model_id = resolve_model_for_driver( + driver_name, + args.model, + provider=args.provider if driver_name == "api" else None, + ) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + return asyncio.run( + run_live( + tasks, + model_alias=args.model, + reps=args.reps, + label=label, + out_path=out, + driver_name=driver_name, + provider=args.provider, + server_cmd=server_cmd, + server_env=server_env or None, + resume=bool(args.resume), + record_result_payloads=bool(args.record_result_payloads), + **({"max_iterations": args.max_iterations} if args.max_iterations else {}), + resolved_model_id=model_id, + ) + ) + + +__all__ = [ + "API_MODEL_TIERS", + "CLI_DRIVER_PROVIDERS", + "CLI_MODEL_TIERS", + "DEFAULT_OUT_DIR", + "MODEL_TIERS", + "cmd_dry_run", + "cmd_list", + "main", + "parse_args", + "resolve_model_for_driver", +] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/core/__init__.py b/evals/core/__init__.py new file mode 100644 index 0000000..9d69611 --- /dev/null +++ b/evals/core/__init__.py @@ -0,0 +1,8 @@ +"""Shared floor of the evals package. + +Modules here may import only other ``evals.core`` modules (plus the +standard library and third-party packages). They must not import +runners, drivers, seeders, tasks, report code, or the recording proxy. +Callers import ``evals.core.`` directly — this package does not +re-export its submodules. +""" diff --git a/evals/core/changelog.py b/evals/core/changelog.py new file mode 100644 index 0000000..220cc3f --- /dev/null +++ b/evals/core/changelog.py @@ -0,0 +1,41 @@ +"""Shared release changelog response normalization.""" + +from __future__ import annotations + +import re +from html import unescape +from typing import Any + + +def _field(value: Any, name: str) -> Any: + return value.get(name) if isinstance(value, dict) else getattr(value, name, None) + + +def normalize_changelog_text(value: Any) -> str: + """Extract normalized text from a changelog API response or stored text.""" + nested = _field(value, "changelog") + candidates = ( + value if isinstance(value, str) else _field(value, "description_html"), + nested if isinstance(nested, str) else _field(nested, "description_html"), + ) + for candidate in candidates: + if isinstance(candidate, str) and candidate.strip(): + without_tags = re.sub(r"<[^>]*>", " ", candidate) + return " ".join(unescape(without_tags).split()) + return "" + + +def changelog_items(value: Any) -> list[str]: + """Extract exact item text following each ``Changelog entry …:`` label.""" + text = normalize_changelog_text(value) + markers = list(re.finditer(r"Changelog entry\s+[^:]+:\s*", text, flags=re.IGNORECASE)) + items: list[str] = [] + for index, marker in enumerate(markers): + end = markers[index + 1].start() if index + 1 < len(markers) else len(text) + item = text[marker.end() : end].strip().rstrip(".").strip() + if item: + items.append(item) + return items + + +__all__ = ["changelog_items", "normalize_changelog_text"] diff --git a/evals/core/error_class.py b/evals/core/error_class.py new file mode 100644 index 0000000..8509858 --- /dev/null +++ b/evals/core/error_class.py @@ -0,0 +1,138 @@ +"""What kind of "no" a tool call received. + +One errored-call count answers three unrelated questions at once, and the answers +pull in different directions: + + refused the server turned the call away without acting on it -- a required + field missing, an argument the action does not take, a value outside + an enum. Entirely a property of the tool schema. The API was never + asked anything. + rejected the call was well formed and the API refused its meaning: an + undocumented precondition, a conflict. This is where tool-design + defects live, and it is the number worth reading. + not_found a read that came back absent. Usually an answer rather than an + obstacle -- "is there an estimate on this project?" has no cheaper + form than asking -- so it is reported apart from friction. + denied credentials or plan. Says nothing about the tool surface. + failed the server or transport broke. + +Classification runs where the payload still exists (the proxy), and stores only +the category, never the text. + +Deliberately not coupled to this server: the categories are read off HTTP status +and the FastMCP/Pydantic validation shape, both of which any MCP server over a +REST API produces. Two patterns for this server's own refusal wording are +additive -- a foreign surface that does not match them still classifies by +status. That is what let one battery score both a 28-tool and a 177-tool server. +""" + +from __future__ import annotations + +import re + +REFUSED = "refused" +REJECTED = "rejected" +NOT_FOUND = "not_found" +DENIED = "denied" +FAILED = "failed" +UNCLASSIFIED = "unclassified" + +ERROR_CLASSES = (REFUSED, REJECTED, NOT_FOUND, DENIED, FAILED, UNCLASSIFIED) + +#: Classes counted as friction attributable to the tool surface's design. +SURFACE_FRICTION_CLASSES = (REJECTED,) +#: Classes counted as the cost of navigating the schema rather than the API. +NAVIGATION_CLASSES = (REFUSED,) + +_STATUS = re.compile(r"\b(?:HTTP|status(?:[ _]code)?[:= ]*)\s*(\d{3})\b", re.IGNORECASE) + +# Shapes any FastMCP server emits when a call fails its own signature, before +# the tool body runs. +_VALIDATION = ( + "validation error for", + "missing required argument", + "input should be", + "unexpected keyword argument", +) + +# This server's own refusals, which are deliberate answers rather than failures +# of validation, so they carry no status and no pydantic shape. +_OWN_REFUSALS = ( + "requires an action. it takes:", + "does not take:", +) + +_BY_STATUS = { + 400: REJECTED, + 409: REJECTED, + 422: REJECTED, + 401: DENIED, + 402: DENIED, + 403: DENIED, + 404: NOT_FOUND, +} + + +def detect_refusal(payload: str | None) -> str | None: + """Return ``REFUSED`` for a refusal that arrived flagged as a *successful* result. + + This server answers a malformed call with a plain result whose text begins + "Error: ", so the protocol reports success and a caller counting failures sees + none -- about 47 per 35-task battery. Classifying those anyway keeps the metric + honest without asking the server to change what every agent receives. + + Deliberately narrow. Only wording this server owns counts, and the stray-argument + form must carry both of its halves, so an ordinary tool result that happens to + quote one phrase is not miscounted as a refusal. + """ + text = (payload or "").lower() + if "requires an action. it takes:" in text: + return REFUSED + if "does not take:" in text and "it takes:" in text: + return REFUSED + return None + + +def classify_error(payload: str | None) -> str: + """Return the category of a failed call from its error payload. + + Status wins over wording: a 404 whose body happens to mention a missing + argument is still an absent resource. Only when no status is present does the + validation shape decide, because that is the case where the call never + reached the API at all. + """ + text = (payload or "").strip() + if not text: + return UNCLASSIFIED + lowered = text.lower() + + match = _STATUS.search(text) + if match: + status = int(match.group(1)) + if status in _BY_STATUS: + return _BY_STATUS[status] + if 500 <= status <= 599: + return FAILED + if 400 <= status <= 499: + return REJECTED + + if any(marker in lowered for marker in _OWN_REFUSALS): + return REFUSED + if any(marker in lowered for marker in _VALIDATION): + return REFUSED + return UNCLASSIFIED + + +__all__ = [ + "ERROR_CLASSES", + "detect_refusal", + "NAVIGATION_CLASSES", + "SURFACE_FRICTION_CLASSES", + "classify_error", + "DENIED", + "FAILED", + "NOT_FOUND", + "REFUSED", + "REJECTED", + "UNCLASSIFIED", +] diff --git a/evals/core/errors.py b/evals/core/errors.py new file mode 100644 index 0000000..25ac589 --- /dev/null +++ b/evals/core/errors.py @@ -0,0 +1,51 @@ +"""Neutral evaluation control-flow exceptions.""" + +from __future__ import annotations + + +def describe_exception(exc: BaseException, *, limit: int = 4) -> str: + """Render an exception for a result row, flattening any ExceptionGroup. + + An ExceptionGroup's own message names only how many sub-exceptions it holds, so recording + ``f"{type(exc).__name__}: {exc}"`` on one throws the diagnosis away: an OpenAI 400 naming + the exact unsupported parameter was persisted as "unhandled errors in a TaskGroup + (1 sub-exception)", and finding it again meant reproducing the call by hand. anyio wraps + everything the driver does in a task group, so this is the normal shape here, not an edge + case. Nested groups are flattened; ``limit`` bounds a pathological fan-out. + """ + leaves: list[str] = [] + + def walk(node: BaseException) -> None: + subs = getattr(node, "exceptions", None) + if subs: + for sub in subs: + if len(leaves) >= limit: + return + walk(sub) + return + leaves.append(f"{type(node).__name__}: {node}") + + walk(exc) + if not leaves: + return f"{type(exc).__name__}: {exc}" + head = f"{type(exc).__name__}: {exc}" if getattr(exc, "exceptions", None) else "" + body = " | ".join(leaves) + return f"{head} -> {body}" if head else body + + +class TaskSkipped(Exception): + """A task that cannot run in this environment without blaming the agent. + + ``reason`` is matched exactly by the skip taxonomy and must stay stable, so the + refusal that caused the skip travels in ``detail`` instead. Without it, an + intermittent gate is unexplainable after the fact: the status code that would say + whether it was a plan limit, a feature toggle, or a transient failure is gone. + """ + + def __init__(self, reason: str, *, detail: str | None = None) -> None: + self.reason = str(reason) + self.detail = str(detail) if detail else None + super().__init__(self.reason if not self.detail else f"{self.reason} ({self.detail})") + + +__all__ = ["TaskSkipped"] diff --git a/evals/core/evidence.py b/evals/core/evidence.py new file mode 100644 index 0000000..a7e0a66 --- /dev/null +++ b/evals/core/evidence.py @@ -0,0 +1,500 @@ +"""Response-evidence matching without retaining response bodies. + +Provenance asks one question: did the answer come from the tool surface? Two kinds of +evidence answer it, under different rules, because they differ in how guessable they are. + +A **sentinel** is a per-run random string a seeder wrote into Plane. The agent's only +route to Plane is the surface, so the string appearing in a response the agent received +proves surface use by itself. Nothing else needs to hold. + +An **aggregate** is a count. A small integer is guessable, so a count proves nothing on +its own: it counts only from a request that named a seeded entity. + +Drivers compare each Plane response against the configured evidence and retain only the +labels that matched. CLI proxies consume the matching configuration from a file before the +agent starts; sentinel values never enter agent-visible argv/config, result rows, or +payload-free sidecars. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Mapping, Sequence +from hashlib import sha256 +from pathlib import Path +from typing import Any + +EVIDENCE_SENTINELS_ENV = "EVAL_EVIDENCE_SENTINELS_JSON" +TARGET_ENTITY_EVIDENCE = "target-entity-hidden-fact" + + +def normalize_evidence_sentinels(value: Any) -> dict[str, tuple[str, ...]]: + """Return a validated label-to-sentinel mapping, dropping empty values.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[str, ...]] = {} + for raw_label, raw_values in value.items(): + label = str(raw_label or "").strip() + if not label: + continue + values: Sequence[Any] + if isinstance(raw_values, str): + values = (raw_values,) + elif isinstance(raw_values, Sequence): + values = raw_values + else: + continue + clean_values: list[str] = [] + for item in values: + if item is None: + continue + text = str(item).strip() + if text: + clean_values.append(text) + clean = tuple(dict.fromkeys(clean_values)) + if clean: + normalized[label] = clean + return normalized + + +def normalize_evidence_targets(value: Any) -> dict[str, tuple[str, ...]]: + """Return a validated label-to-target-ID mapping, dropping empty values.""" + return normalize_evidence_sentinels(value) + + +def normalize_evidence_aggregates(value: Any) -> dict[str, tuple[dict[str, Any], ...]]: + """Validate the two narrow aggregate response shapes used by R2 and R6.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[dict[str, Any], ...]] = {} + for raw_label, raw_specs in value.items(): + label = str(raw_label or "").strip() + if not label or not isinstance(raw_specs, Sequence) or isinstance(raw_specs, (str, bytes, bytearray)): + continue + specs: list[dict[str, Any]] = [] + for raw_spec in raw_specs: + if not isinstance(raw_spec, Mapping): + continue + kind = raw_spec.get("kind") + if kind == "total_count": + try: + specs.append({"kind": kind, "value": int(raw_spec["value"])}) + except (KeyError, TypeError, ValueError): + continue + elif kind == "grouped_counts" and isinstance(raw_spec.get("values"), Mapping): + try: + values = {str(key): int(count) for key, count in raw_spec["values"].items()} + except (TypeError, ValueError): + continue + if values: + specs.append({"kind": kind, "values": values}) + if specs: + normalized[label] = tuple(specs) + return normalized + + +def evidence_aggregate_shapes(value: Any) -> dict[str, tuple[dict[str, str], ...]]: + """Reduce aggregate truth to the response shapes safe for an agent-visible proxy.""" + return { + label: tuple({"kind": str(spec["kind"])} for spec in specs) + for label, specs in normalize_evidence_aggregates(value).items() + } + + +def normalize_evidence_aggregate_shapes(value: Any) -> dict[str, tuple[dict[str, str], ...]]: + """Validate aggregate extraction instructions that contain no expected values.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[dict[str, str], ...]] = {} + for raw_label, raw_specs in value.items(): + label = str(raw_label or "").strip() + if not label or not isinstance(raw_specs, Sequence) or isinstance(raw_specs, (str, bytes, bytearray)): + continue + specs: list[dict[str, str]] = [] + for raw_spec in raw_specs: + if not isinstance(raw_spec, Mapping) or raw_spec.get("kind") not in {"total_count", "grouped_counts"}: + continue + spec = {"kind": str(raw_spec["kind"])} + if spec not in specs: + specs.append(spec) + if specs: + normalized[label] = tuple(specs) + return normalized + + +def configured_evidence_labels(sentinels: Any, targets: Any, aggregates: Any = None) -> tuple[str, ...]: + """Return labels this run can actually prove, by whichever rule governs their kind. + + A sentinel proves itself. A count does not — a small integer is guessable, so it + counts only from a request that named a seeded entity, and an aggregate label with + no registered target can never match. + """ + sentinel_labels = normalize_evidence_sentinels(sentinels).keys() + targets_by_label = normalize_evidence_targets(targets) + aggregate_labels = normalize_evidence_aggregates(aggregates).keys() & targets_by_label.keys() + return tuple(sorted(sentinel_labels | aggregate_labels)) + + +def fingerprint_evidence_sentinels(value: Any) -> dict[str, tuple[tuple[int, str], ...]]: + """Replace raw values with character lengths and one-way SHA-256 fingerprints.""" + return { + label: tuple((len(item), sha256(item.encode("utf-8")).hexdigest()) for item in values) + for label, values in normalize_evidence_sentinels(value).items() + } + + +def normalize_evidence_fingerprints(value: Any) -> dict[str, tuple[tuple[int, str], ...]]: + """Validate serialized response-value fingerprints, dropping malformed entries.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, tuple[tuple[int, str], ...]] = {} + for raw_label, raw_specs in value.items(): + label = str(raw_label or "").strip() + if not label or not isinstance(raw_specs, Sequence) or isinstance(raw_specs, (str, bytes, bytearray)): + continue + specs: list[tuple[int, str]] = [] + for raw_spec in raw_specs: + if isinstance(raw_spec, Mapping): + raw_length = raw_spec.get("length") + raw_digest = raw_spec.get("sha256") + elif ( + isinstance(raw_spec, Sequence) + and not isinstance(raw_spec, (str, bytes, bytearray)) + and len(raw_spec) == 2 + ): + raw_length, raw_digest = raw_spec + else: + continue + try: + length = int(raw_length) + except (TypeError, ValueError): + continue + digest = str(raw_digest or "").strip().lower() + if length > 0 and len(digest) == 64 and all(char in "0123456789abcdef" for char in digest): + specs.append((length, digest)) + clean = tuple(dict.fromkeys(specs)) + if clean: + normalized[label] = clean + return normalized + + +def encode_evidence_sentinels(value: Any) -> str: + """Serialize a temporary driver/proxy configuration, never a result-row field.""" + normalized = normalize_evidence_sentinels(value) + return json.dumps(normalized, ensure_ascii=True, separators=(",", ":")) + + +def decode_evidence_sentinels(value: str | None) -> dict[str, tuple[str, ...]]: + """Decode a temporary driver/proxy configuration, failing closed on bad input.""" + if not value: + return {} + try: + raw = json.loads(value) + except (TypeError, ValueError): + return {} + return normalize_evidence_sentinels(raw) + + +def encode_evidence_config(sentinels: Any, targets: Any, aggregates: Any = None) -> str: + """Serialize targets and extraction shapes, never raw sentinels or aggregate truth.""" + fingerprints = fingerprint_evidence_sentinels(sentinels) + return json.dumps( + { + "fingerprints": { + label: [{"length": length, "sha256": digest} for length, digest in specs] + for label, specs in fingerprints.items() + }, + "targets": normalize_evidence_targets(targets), + "aggregates": evidence_aggregate_shapes(aggregates), + }, + ensure_ascii=True, + separators=(",", ":"), + ) + + +def decode_evidence_config( + value: str | None, +) -> tuple[ + dict[str, tuple[tuple[int, str], ...]], + dict[str, tuple[str, ...]], + dict[str, tuple[dict[str, Any], ...]], +]: + """Decode proxy-only matching configuration, failing closed on malformed input.""" + if not value: + return {}, {}, {} + try: + raw = json.loads(value) + except (TypeError, ValueError): + return {}, {}, {} + if not isinstance(raw, Mapping): + return {}, {}, {} + return ( + normalize_evidence_fingerprints(raw.get("fingerprints")), + normalize_evidence_targets(raw.get("targets")), + normalize_evidence_aggregate_shapes(raw.get("aggregates")), + ) + + +def write_evidence_config(path: Path, sentinels: Any, targets: Any, aggregates: Any = None) -> None: + """Create a private, run-scoped proxy configuration outside the agent cwd.""" + payload = encode_evidence_config(sentinels, targets, aggregates) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(payload) + + +def consume_evidence_config( + path: Path | None, +) -> tuple[ + dict[str, tuple[tuple[int, str], ...]], + dict[str, tuple[str, ...]], + dict[str, tuple[dict[str, Any], ...]], +]: + """Read a reusable run-scoped proxy configuration, failing closed. + + The historical name is retained for callers. A CLI may start multiple MCP + proxy sessions during one task, so the driver's TemporaryDirectory owns + deletion after every session has exited. + """ + if path is None: + return {}, {}, {} + try: + raw = path.read_text(encoding="utf-8") + except OSError: + return {}, {}, {} + return decode_evidence_config(raw) + + +def _request_targets(request_args: Any, target_ids: Sequence[str]) -> bool: + targets = set(target_ids) + + def contains(value: Any) -> bool: + if isinstance(value, Mapping): + return any(contains(item) for item in value.values()) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return any(contains(item) for item in value) + if value is None: + return False + text = str(value) + return any(target == text or target in text for target in targets) + + return contains(request_args) + + +def observed_sentinel_labels(response_text: str, sentinels: Any) -> list[str]: + """Return labels whose hidden value appeared in this response. + + A sentinel is a per-run random string written into Plane at seed time, and the + agent's only route to Plane is the tool surface. Its presence in a response the + agent received is therefore proof of surface use on its own, with no need to also + inspect which entity the request named. + """ + text = str(response_text or "") + if not text: + return [] + normalized = normalize_evidence_sentinels(sentinels) + return sorted(label for label, values in normalized.items() if any(value in text for value in values)) + + +def observed_fingerprint_labels(response_text: str, fingerprints: Any) -> list[str]: + """Match sentinel fingerprints without ever receiving the raw values.""" + text = str(response_text or "") + if not text: + return [] + eligible = normalize_evidence_fingerprints(fingerprints) + if not eligible: + return [] + + expected_by_length: dict[int, set[str]] = {} + labels_by_spec: dict[tuple[int, str], set[str]] = {} + for label, specs in eligible.items(): + for length, digest in specs: + expected_by_length.setdefault(length, set()).add(digest) + labels_by_spec.setdefault((length, digest), set()).add(label) + + matched: set[str] = set() + for length, expected in expected_by_length.items(): + if length > len(text): + continue + remaining = set(expected) + for start in range(len(text) - length + 1): + digest = sha256(text[start : start + length].encode("utf-8")).hexdigest() + if digest not in remaining: + continue + matched.update(labels_by_spec[(length, digest)]) + remaining.remove(digest) + if not remaining: + break + return sorted(matched) + + +def _decoded_documents(response_text: str) -> list[Any]: + """Decode JSON-RPC/MCP wrappers and JSON strings embedded inside them.""" + documents: list[Any] = [] + pending: list[Any] = [response_text] + seen_strings: set[str] = set() + while pending: + value = pending.pop() + documents.append(value) + if isinstance(value, Mapping): + pending.extend(value.values()) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + pending.extend(value) + elif isinstance(value, str): + text = value.strip() + if text in seen_strings or not text or text[0] not in "[{": + continue + seen_strings.add(text) + try: + pending.append(json.loads(text)) + except (TypeError, ValueError): + continue + return documents + + +def observed_aggregates( + response_text: str, + aggregates: Any, + *, + request_args: Any, + evidence_targets: Any, +) -> list[dict[str, Any]]: + """Extract target-bound aggregate values without receiving expected truth.""" + specs_by_label = normalize_evidence_aggregate_shapes(aggregates) + targets_by_label = normalize_evidence_targets(evidence_targets) + documents = _decoded_documents(response_text) + observations: list[dict[str, Any]] = [] + for label, specs in specs_by_label.items(): + targets = targets_by_label.get(label, ()) + if not targets or not _request_targets(request_args, targets): + continue + for spec in specs: + if spec["kind"] == "total_count": + for value in documents: + if not isinstance(value, Mapping): + continue + count = value.get("total_count") + if isinstance(count, int) and not isinstance(count, bool): + observations.append({"label": label, "kind": "total_count", "value": count}) + break + elif spec["kind"] == "grouped_counts": + for value in documents: + if not isinstance(value, Mapping) or not isinstance(value.get("grouped_counts"), Mapping): + continue + grouped = value["grouped_counts"] + observed: dict[str, int] = {} + for target in targets: + entry = grouped.get(target) + count = entry.get("count") if isinstance(entry, Mapping) else None + if not isinstance(count, int) or isinstance(count, bool): + break + observed[target] = count + if len(observed) == len(targets): + observations.append({"label": label, "kind": "grouped_counts", "values": observed}) + break + return observations + + +def observed_aggregate_labels(observations: Any, aggregates: Any) -> list[str]: + """Compare proxy observations with seed truth inside the post-agent harness.""" + if not isinstance(observations, Sequence) or isinstance(observations, (str, bytes, bytearray)): + return [] + expected_by_label = normalize_evidence_aggregates(aggregates) + matched: set[str] = set() + for observation in observations: + if not isinstance(observation, Mapping): + continue + label = str(observation.get("label") or "") + for expected in expected_by_label.get(label, ()): + if expected["kind"] != observation.get("kind"): + continue + if expected["kind"] == "total_count" and observation.get("value") == expected["value"]: + matched.add(label) + elif expected["kind"] == "grouped_counts" and observation.get("values") == expected["values"]: + matched.add(label) + return sorted(matched) + + +def _register_targets(context: dict[str, Any], target_ids: Sequence[Any], *, what: str) -> None: + """Add seeded entity IDs to the label's target set, keeping any already registered.""" + clean = tuple(dict.fromkeys(str(value).strip() for value in target_ids if value is not None and str(value).strip())) + if not clean: + raise RuntimeError(f"{what} has no seeded target entity ids") + targets = context.setdefault("evidence_targets", {}) + current = targets.get(TARGET_ENTITY_EVIDENCE, ()) + targets[TARGET_ENTITY_EVIDENCE] = tuple(dict.fromkeys((*current, *clean))) + + +def _add_aggregate_specs(context: dict[str, Any], specs: Sequence[dict[str, Any]]) -> None: + """Append acceptable aggregate shapes rather than replacing the registered ones. + + A task may reach its answer by more than one honest call shape — R6's winner is + provable by two per-project counts or by one count grouped by project. Replacing + here privileged whichever seeder ran last, and scored every other path unproven. + """ + aggregates = context.setdefault("evidence_aggregates", {}) + registered = list(aggregates.get(TARGET_ENTITY_EVIDENCE, ())) + for spec in specs: + if spec not in registered: + registered.append(spec) + aggregates[TARGET_ENTITY_EVIDENCE] = tuple(registered) + + +def set_target_evidence(context: dict[str, Any], values: Sequence[Any]) -> None: + """Register the API-confirmed hidden values whose presence proves surface use.""" + clean_values: list[str] = [] + for value in values: + if value is None: + continue + text = str(value).strip() + if text: + clean_values.append(text) + clean = tuple(dict.fromkeys(clean_values)) + if not clean: + raise RuntimeError("target evidence has no API-confirmed sentinel values") + context["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: clean} + + +def set_target_count_evidence(context: dict[str, Any], *counts: int, target_ids: Sequence[Any]) -> None: + """Allow an exact ``total_count`` response whose request names a seeded target.""" + if not counts: + raise RuntimeError("target count evidence has no API-confirmed counts") + _register_targets(context, target_ids, what="target count evidence") + _add_aggregate_specs(context, [{"kind": "total_count", "value": int(count)} for count in counts]) + + +def set_target_grouped_count_evidence(context: dict[str, Any], values: Mapping[Any, int]) -> None: + """Allow grouped counts only when every seeded target id has its exact count.""" + clean = {str(target): int(count) for target, count in values.items() if str(target).strip()} + if not clean: + raise RuntimeError("target grouped-count evidence has no seeded targets") + _register_targets(context, clean, what="target grouped-count evidence") + _add_aggregate_specs(context, [{"kind": "grouped_counts", "values": clean}]) + + +__all__ = [ + "EVIDENCE_SENTINELS_ENV", + "TARGET_ENTITY_EVIDENCE", + "configured_evidence_labels", + "consume_evidence_config", + "decode_evidence_config", + "decode_evidence_sentinels", + "encode_evidence_config", + "encode_evidence_sentinels", + "evidence_aggregate_shapes", + "fingerprint_evidence_sentinels", + "normalize_evidence_fingerprints", + "normalize_evidence_aggregate_shapes", + "normalize_evidence_aggregates", + "normalize_evidence_sentinels", + "normalize_evidence_targets", + "observed_sentinel_labels", + "observed_fingerprint_labels", + "observed_aggregate_labels", + "observed_aggregates", + "set_target_count_evidence", + "set_target_evidence", + "set_target_grouped_count_evidence", + "write_evidence_config", +] diff --git a/evals/core/failure_kind.py b/evals/core/failure_kind.py new file mode 100644 index 0000000..7cf4eb3 --- /dev/null +++ b/evals/core/failure_kind.py @@ -0,0 +1,147 @@ +"""What kind of wrong a failed task was. + +One failed verifier answers several unrelated questions at once. In one measured +battery, ``W7`` put a link on the wrong work item, ``I1`` set ``urgent`` where +``high`` was asked, and ``S2`` spent 43 calls and produced nothing -- three +different defects, reported identically, separable only by reading notes by hand. + + unproven the answer was right and the run could not evidence it. Not an + agent defect at all, and the largest single family in the recorded + corpus, so folding it into the others would misattribute most + failures to the model. + wrong_value a value was written or reported, and it differs from the one asked + for. + missing_write the thing was never created; the verifier found nothing. + partial_write a multi-part change half landed -- the shape that hides a wrong + target, since writing correctly to the wrong entity leaves the + right entity empty. + abandoned the run hit its iteration or token ceiling, so the note describes + an unfinished state rather than a defect. + environment a capability the environment does not have. Not a defect either. + +Same contract as ``error_class``: a narrow pattern table over text the verifiers +own, and ``unclassified`` is a first-class member that is counted and printed. A +zero in some kind must mean "none of these", never "the classifier did not +recognise it". + +Deliberately note-only. Whether a write went to the *wrong target* is not knowable +from a note that reports the right target as empty -- that needs call arguments, +which is a different measurement. +""" + +from __future__ import annotations + +import re + +UNPROVEN = "unproven" +WRONG_VALUE = "wrong_value" +MISSING_WRITE = "missing_write" +PARTIAL_WRITE = "partial_write" +ABANDONED = "abandoned" +ENVIRONMENT = "environment" +UNCLASSIFIED = "unclassified" + +FAILURE_KINDS = ( + UNPROVEN, + WRONG_VALUE, + MISSING_WRITE, + PARTIAL_WRITE, + ABANDONED, + ENVIRONMENT, + UNCLASSIFIED, +) + +#: Kinds that are properties of the run or the environment, not of the agent. +NON_DEFECT_KINDS = (UNPROVEN, ENVIRONMENT, ABANDONED) + +#: stop_reason values that mean the run was cut off rather than finished. +_CAPPED_STOP_REASONS = frozenset({"max_turns", "max_tokens", "max_iterations"}) + +#: The verifier states its verdict before its evidence, so these settle the note. +_ANSWER_CORRECT = "answer_correct=true" +_ANSWER_WRONG = "answer_correct=false" + +#: Wording for something the verifier looked for and did not find. +_ABSENT = ( + "not found", + "was not created", + "missing", + "have []", +) + +#: Wording for something it did find. Only meaningful next to an absence, where the +#: pair means a change landed in part. +_PRESENT = ( + " present", + "names ", + " linked", +) + +#: A stated expectation, which implies a value was compared rather than absent. +_EXPECTATION = ("(want ", "want ") + +#: Absence phrasings that carry no "missing"/"not found" wording. Narrow on purpose -- +#: a bare "not " would swallow "not closed: end_date=X (want Y)", which is a value +#: mismatch rather than an absence. +_ABSENT_PATTERN = re.compile(r"\bno \d|\bno comments\b|\bnot archived\b|\bnot created\b") + + +def classify_failure( + note: str | None, + *, + stop_reason: str | None = None, + hit_max_iterations: bool = False, +) -> str: + """Return the kind of failure a verifier note describes. + + Structural signals win over the note: a run that hit its ceiling has an + unfinished state to report regardless of what the note says about it. + """ + text = (note or "").strip() + lowered = text.lower() + capped = hit_max_iterations or (stop_reason or "").strip().lower() in _CAPPED_STOP_REASONS + + if lowered.startswith("env:"): + return ENVIRONMENT + + # A proven wrong answer outranks the cap. Running out of iterations explains why a + # run stopped, not why what it wrote was wrong, and calling that combination + # "abandoned" would file a demonstrated defect as a non-defect. + # + # The false marker is tested first because the true one is searched anywhere in the + # note, and a wrong value quoted back by the verifier can itself contain the string. + if _ANSWER_WRONG in lowered: + return WRONG_VALUE + if _ANSWER_CORRECT in lowered: + return UNPROVEN + + if capped: + return ABANDONED + if not text: + return UNCLASSIFIED + + absent = bool(_ABSENT_PATTERN.search(lowered)) or any(marker in lowered for marker in _ABSENT) + present = any(marker in lowered for marker in _PRESENT) + if absent and present: + return PARTIAL_WRITE + if absent: + # Checked before the expectation markers on purpose. "missing X ... (want 5)" + # is nothing written, not a wrong value. + return MISSING_WRITE + if any(marker in lowered for marker in _EXPECTATION): + return WRONG_VALUE + return UNCLASSIFIED + + +__all__ = [ + "ABANDONED", + "ENVIRONMENT", + "FAILURE_KINDS", + "MISSING_WRITE", + "NON_DEFECT_KINDS", + "PARTIAL_WRITE", + "UNCLASSIFIED", + "UNPROVEN", + "WRONG_VALUE", + "classify_failure", +] diff --git a/evals/core/fixtures.py b/evals/core/fixtures.py new file mode 100644 index 0000000..e43100e --- /dev/null +++ b/evals/core/fixtures.py @@ -0,0 +1,254 @@ +"""Neutral fixture names shared by seeders, task prompts, and cleanup. + +This module must not import either :mod:`evals.seed` or :mod:`evals.tasks`; both +packages re-export these names for backward compatibility. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +CUSTOMER_NAME = "Acme Corp" +CUSTOMER_REQUEST_NAME = "SSO support" +EVALUATION_CUSTOMER_PROPERTY_NAME = "Eval Industry" +_EVALUATION_CUSTOMER_NAMES = {CUSTOMER_NAME.casefold(), "acme"} + +RELEASE_NAME = "1.2.0" +RELEASE_CHANGELOG_TEXT = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." +EVALUATION_RELEASE_TAG_VERSION = "eval-rc1" + +INTAKE_BILLING_TITLE = "Billing: invoice PDF missing line items" +INTAKE_SPAM_TITLE = "SPAM: cheap crypto pumps guaranteed" + +CYCLE_PAST = "Sprint 12" +CYCLE_CURRENT = "Sprint 13" + +MODULE_NAME = "Checkout revamp" +MODULE_COMPLETED_TITLES = ( + "Module done: cart totals", + "Module done: tax lines", + "Module done: shipping quote", +) + +# Fixed fixture titles for the ``items`` group. Exactly four are urgent. +WORK_ITEM_FIXTURES: list[tuple[str, str]] = [ + ("Payment webhook drops retries", "urgent"), + ("Checkout times out on 3DS challenge", "urgent"), + ("Session cookie not rotated after login", "urgent"), + ("Inventory count goes negative under load", "urgent"), + ("Search results ignore archived projects", "high"), + ("CSV export truncates multi-byte chars", "high"), + ("Webhook secret rotation docs missing", "medium"), + ("Dark mode contrast fails WCAG AA", "medium"), + ("Onboarding email template stale", "medium"), + ("Sidebar collapse flickers on resize", "low"), + ("Tooltip clipped inside modal dialog", "low"), + ("Footer year still says 2024", "none"), +] + +PAYMENT_WEBHOOK_TITLE = WORK_ITEM_FIXTURES[0][0] +CHECKOUT_TIMEOUT_TITLE = "Checkout times out on 3DS challenge" +CHECKOUT_COMMENT_PHRASES = ( + "stripe callback race", + "retry budget exhausted", +) +SIDEBAR_TITLE = "Sidebar collapse flickers on resize" +DARK_MODE_TITLE = "Dark mode contrast fails WCAG AA" +BLOCKING_SOURCE_TITLE = "Search results ignore archived projects" +BLOCKING_TARGET_TITLE = "CSV export truncates multi-byte chars" +BLOCKING_REFERENCE_ADDRESS = "https://example.com/eval/runbook-w7" +DUE_THIS_WEEK_TITLES = ( + "Webhook secret rotation docs missing", + "Onboarding email template stale", +) +UNFINISHED_CYCLE_TITLES = ( + "Inventory count goes negative under load", + "Tooltip clipped inside modal dialog", +) + +# Historical public aliases used by task catalog modules and downstream scripts. +DEBIAS_CUSTOMER_PROP_DISPLAY = EVALUATION_CUSTOMER_PROPERTY_NAME +DEBIAS_RELEASE_TAG_VERSION = EVALUATION_RELEASE_TAG_VERSION +ITEM_FIXTURES = WORK_ITEM_FIXTURES +R1_TITLE = PAYMENT_WEBHOOK_TITLE +R3_DUE_TITLES = DUE_THIS_WEEK_TITLES +R5_COMMENT_PHRASES = CHECKOUT_COMMENT_PHRASES +R5_TITLE = CHECKOUT_TIMEOUT_TITLE +W2_TITLE = SIDEBAR_TITLE +W3_TITLE = DARK_MODE_TITLE +W6_UNFINISHED_TITLES = UNFINISHED_CYCLE_TITLES +W7_SOURCE_TITLE = BLOCKING_SOURCE_TITLE +W7_TARGET_TITLE = BLOCKING_TARGET_TITLE +W7_URL = BLOCKING_REFERENCE_ADDRESS +W8_TITLE = PAYMENT_WEBHOOK_TITLE + + +def is_evaluation_customer_name(name: str | None) -> bool: + """Return whether a customer name matches an eval fixture alias.""" + return (name or "").strip().casefold() in _EVALUATION_CUSTOMER_NAMES + + +# Project names. Nothing here may look like an id. +# +# Seeded projects used to be called "EVAL 3c128f21". An agent is told only the project name +# and has to resolve it to a UUID, since project_id is required by 121 of the 183 actions — +# and a weaker model skipped the resolution and submitted a hex-looking substring as the id. +# Moving the hex into parentheses made it worse, not better: it became a cleaner token to +# extract, and non-UUID project_id attempts went from 4 to 17 across six repetitions. +# +# So the name carries no hex at all. Teardown deletes by recorded project_id, so per-run +# uniqueness in the *name* is not required for correctness; the word suffix exists only so a +# leftover project from a crashed run cannot make an agent's name lookup ambiguous, and so R6 +# can tell its two projects apart. `python -m evals.cleanup --prefix "EVAL "` still matches. +EVAL_PROJECT_PREFIX = "EVAL " +PROJECT_TITLES = ("Delivery Planning", "Platform Migration") +# 64 words: one per run, derived from the seed so a name is reproducible from its run id. +PROJECT_SUFFIX_WORDS = ( + "Kestrel", + "Osprey", + "Falcon", + "Harrier", + "Merlin", + "Kite", + "Buzzard", + "Goshawk", + "Heron", + "Egret", + "Curlew", + "Plover", + "Godwit", + "Dunlin", + "Sanderling", + "Turnstone", + "Petrel", + "Fulmar", + "Gannet", + "Guillemot", + "Razorbill", + "Puffin", + "Skua", + "Tern", + "Swift", + "Martin", + "Swallow", + "Wagtail", + "Pipit", + "Dipper", + "Wren", + "Dunnock", + "Redstart", + "Whinchat", + "Wheatear", + "Fieldfare", + "Redwing", + "Blackcap", + "Chiffchaff", + "Firecrest", + "Treecreeper", + "Nuthatch", + "Jackdaw", + "Chough", + "Raven", + "Rook", + "Magpie", + "Jay", + "Linnet", + "Twite", + "Redpoll", + "Siskin", + "Crossbill", + "Hawfinch", + "Brambling", + "Yellowhammer", + "Corncrake", + "Lapwing", + "Woodcock", + "Snipe", + "Avocet", + "Oystercatcher", + "Shelduck", + "Wigeon", +) + + +def _suffix_word_index(run_prefix: str) -> int: + """Map a run prefix onto ``PROJECT_SUFFIX_WORDS``, tolerating non-hex prefixes.""" + try: + return int(str(run_prefix)[:8], 16) + except ValueError: + return sum(ord(ch) for ch in str(run_prefix)) + + +def eval_project_name(run_prefix: str, *, second: bool = False) -> str: + """Build a seeded project's display name: readable, and never id-shaped. + + Deterministic in ``run_prefix`` so the same run always produces the same name, which + keeps a resumed run and its teardown in agreement. + """ + word = PROJECT_SUFFIX_WORDS[_suffix_word_index(run_prefix) % len(PROJECT_SUFFIX_WORDS)] + title = PROJECT_TITLES[1 if second else 0] + return f"{EVAL_PROJECT_PREFIX}{title} {word}" + + +def eval_project_name_variants(run_prefix: str, *, second: bool = False) -> Iterator[str]: + """Yield the deterministic name, then every other word in order. + + The first name is exactly ``eval_project_name(run_prefix, second=second)``, so a + resumed run and its teardown still agree on it. The rest exist only so a leftover + project from a crashed run cannot fail a fresh one: Plane rejects a duplicate project + name with a 409, and the word pool is small enough that residue makes that collision + a matter of when. Walking forward from the deterministic index keeps the fallback + order reproducible too. + """ + words = PROJECT_SUFFIX_WORDS + start = _suffix_word_index(run_prefix) % len(words) + title = PROJECT_TITLES[1 if second else 0] + for offset in range(len(words)): + yield f"{EVAL_PROJECT_PREFIX}{title} {words[(start + offset) % len(words)]}" + + +__all__ = [ + "EVAL_PROJECT_PREFIX", + "PROJECT_SUFFIX_WORDS", + "PROJECT_TITLES", + "eval_project_name", + "eval_project_name_variants", + "BLOCKING_REFERENCE_ADDRESS", + "BLOCKING_SOURCE_TITLE", + "BLOCKING_TARGET_TITLE", + "CHECKOUT_COMMENT_PHRASES", + "CHECKOUT_TIMEOUT_TITLE", + "CUSTOMER_NAME", + "CUSTOMER_REQUEST_NAME", + "CYCLE_CURRENT", + "CYCLE_PAST", + "DARK_MODE_TITLE", + "DEBIAS_CUSTOMER_PROP_DISPLAY", + "DEBIAS_RELEASE_TAG_VERSION", + "DUE_THIS_WEEK_TITLES", + "EVALUATION_CUSTOMER_PROPERTY_NAME", + "EVALUATION_RELEASE_TAG_VERSION", + "INTAKE_BILLING_TITLE", + "INTAKE_SPAM_TITLE", + "ITEM_FIXTURES", + "MODULE_COMPLETED_TITLES", + "MODULE_NAME", + "PAYMENT_WEBHOOK_TITLE", + "R1_TITLE", + "R3_DUE_TITLES", + "R5_COMMENT_PHRASES", + "R5_TITLE", + "RELEASE_CHANGELOG_TEXT", + "RELEASE_NAME", + "SIDEBAR_TITLE", + "UNFINISHED_CYCLE_TITLES", + "W2_TITLE", + "W3_TITLE", + "W6_UNFINISHED_TITLES", + "W7_SOURCE_TITLE", + "W7_TARGET_TITLE", + "W7_URL", + "W8_TITLE", + "WORK_ITEM_FIXTURES", + "is_evaluation_customer_name", +] diff --git a/evals/core/pricing.py b/evals/core/pricing.py new file mode 100644 index 0000000..03a3623 --- /dev/null +++ b/evals/core/pricing.py @@ -0,0 +1,159 @@ +"""What a run cost, or an honest statement that we do not know. + +Cost is the decision metric for a tool-surface project, and for two days nothing +computed it -- so it was derived by hand and stated wrongly twice. The failure mode +this module is built against is not an absent number but a **zero**: an arm with no +usage recorded, priced at $0.00, reads as free rather than as unmeasured. So there +are three outcomes and they are kept distinct: + + priced usage present, model known + unpriced usage present, but the model is not in the table, or its cache + semantics could not be resolved + unmeasured the driver recorded no usage at all -- true of every antigravity row + +Prices go stale, and a wrong price is worse than no price. ``PRICES_AS_OF`` dates the +table, but a date detects nothing on its own. The mechanism that actually detects +staleness is ``vendor_usd``: Claude Code reports ``total_cost_usd`` per run, so the +computed figure can be checked against the vendor's own on every run. That check +earned its keep immediately -- published 5-minute cache-write rates priced a measured +arm at $2.46 against a reported $3.18, and the 1-hour TTL multiplier reproduced the +reported figure exactly. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from evals.core.token_accounting import has_token_counts, normalize_usage + +#: The day the rates below were last checked against published pricing. +PRICES_AS_OF = "2026-08-24" + +PRICED = "priced" +UNPRICED = "unpriced" +UNMEASURED = "unmeasured" + +COST_OUTCOMES = (PRICED, UNPRICED, UNMEASURED) + + +@dataclass(frozen=True) +class ModelPrice: + """US dollars per million tokens. + + ``cache_creation`` is the rate for tokens *written* to cache. Anthropic charges + for writes on a multiple of the input rate that depends on the cache TTL -- 1.25x + at five minutes, 2x at one hour -- and Claude Code uses the one-hour tier, which + is what reproduces its reported cost. OpenAI does not bill cache writes at all + and reports zero such tokens, so the value is unexercised there. + """ + + input: float + cached_input: float + output: float + cache_creation: float | None = None + + def cache_creation_rate(self) -> float: + return self.input if self.cache_creation is None else self.cache_creation + + +#: Keyed by model family prefix, longest match first. Dated ids such as +#: ``claude-haiku-4-5-20251001`` match their undated family. +PRICES: dict[str, ModelPrice] = { + "gpt-5.6-luna": ModelPrice(input=0.20, cached_input=0.02, output=1.20), + "claude-haiku-4-5": ModelPrice(input=1.00, cached_input=0.10, output=5.00, cache_creation=2.00), + "claude-sonnet-5": ModelPrice(input=3.00, cached_input=0.30, output=15.00, cache_creation=6.00), + "claude-opus-5": ModelPrice(input=5.00, cached_input=0.50, output=25.00, cache_creation=10.00), +} + + +@dataclass(frozen=True) +class RowCost: + """The cost of one row, and how confident that figure is.""" + + outcome: str + usd: float | None + model_id: str | None + vendor_usd: float | None = None + + @property + def billed_usd(self) -> float | None: + """The vendor's own figure where it exists, else ours. + + A provider that reports what it charged is more authoritative than any table. + """ + return self.vendor_usd if self.vendor_usd is not None else self.usd + + +def resolve_model_id(usage_total: Mapping[str, Any] | None, *, model: str | None) -> str | None: + """Return the most specific model identifier this row carries. + + ``row.model`` is what the driver was asked for, which under a CLI is a tier alias + -- claude-cli records ``"haiku"``. The real identifier is inside ``modelUsage``, + so that wins when a single model produced the run. + """ + if usage_total: + model_usage = usage_total.get("modelUsage") + if isinstance(model_usage, Mapping) and model_usage: + if len(model_usage) > 1: + # Several models produced this run and the counters are already summed, + # so no single rate is correct for them. Falling back to the row alias + # would price Opus tokens at the Haiku rate whenever that alias happened + # to be priceable. + return None + only = next(iter(model_usage)) + if isinstance(only, str) and only: + return only + return model or None + + +def lookup_price(model_id: str | None) -> ModelPrice | None: + if not model_id: + return None + lowered = model_id.strip().lower() + for prefix in sorted(PRICES, key=len, reverse=True): + if lowered.startswith(prefix): + return PRICES[prefix] + return None + + +def price_usage(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> RowCost: + """Price one row's ``usage_total``.""" + vendor = usage_total.get("total_cost_usd") if usage_total else None + vendor_usd = float(vendor) if isinstance(vendor, (int, float)) else None + + if not usage_total or not has_token_counts(usage_total): + # A usage dict with no counts in it is metadata, not a measurement -- but a vendor + # that stated what it charged still told us something, and discarding that would + # throw away the most authoritative figure available. + return RowCost(outcome=UNMEASURED, usd=None, model_id=model or None, vendor_usd=vendor_usd) + model_id = resolve_model_id(usage_total, model=model) + + accounting = normalize_usage(usage_total, model=model_id) + price = lookup_price(model_id) + if accounting is None or price is None: + return RowCost(outcome=UNPRICED, usd=None, model_id=model_id, vendor_usd=vendor_usd) + + usd = ( + accounting.uncached_input * price.input + + accounting.cached_input * price.cached_input + + accounting.cache_creation * price.cache_creation_rate() + + accounting.output * price.output + ) / 1e6 + return RowCost(outcome=PRICED, usd=round(usd, 10), model_id=model_id, vendor_usd=vendor_usd) + + +__all__ = [ + "COST_OUTCOMES", + "PRICED", + "PRICES", + "PRICES_AS_OF", + "UNMEASURED", + "UNPRICED", + "ModelPrice", + "RowCost", + "lookup_price", + "price_usage", + "resolve_model_id", +] diff --git a/evals/core/results.py b/evals/core/results.py new file mode 100644 index 0000000..6709052 --- /dev/null +++ b/evals/core/results.py @@ -0,0 +1,768 @@ +"""Declared persisted schema for eval task-result JSONL rows.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Literal + +from evals.core.token_counting import ( + TOKEN_ESTIMATE_METHOD, + count_result_text_tokens, + estimate_result_tokens, +) +from evals.core.tool_names import split_plane_and_client_calls + +RESULT_SCHEMA_VERSION = 6 +TRACE_INTEGRITY_SCHEMA_VERSION = 5 + +TraceIntegrityReason = Literal["recorder_loss", "protocol_violation", "result_pair_mismatch"] + +# ``apply_agent_result`` owns this explicit partition. A reflection test compares +# it with every TaskResult dataclass field so additions cannot disappear silently. +AGENT_RESULT_COPY_FIELDS = ( + "final_text", + "stop_reason", + "provider_stop_reason", + "hit_max_iterations", + "result_pair_mismatch", + "trace_integrity", + "trace_integrity_reason", + "tool_manifest_fingerprint", + "token_count_failures", + "result_tokens_estimated", + "calls", + "num_calls", + "errored_calls", + "total_result_tokens", + "usage_per_iteration", + "cum_input_tokens", + "cum_input_tokens_reason", + "wall_time_s", + "client_tool_calls", + "client_tool_call_count", + "result_tokens_mode", + "result_token_count_method", + "usage_scope", + "call_source", + "evidence_trace_available", + "driver_raw_ref", + "driver_notes", + "usage", + "usage_total", + "result_tokens_skipped_reason", +) +AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS = ("provider", "model", "requested_model") +TASK_RESULT_HARNESS_FIELDS = ( + "schema_version", + "row_type", + "run_id", + "fixture_seed_id", + "ts", + "git_sha", + "battery", + "task_fingerprint", + "label", + "driver", + "server", + "requested_tier", + "resolved_model", + "task_id", + "author", + "rep", + "expected_rows", + "success", + "verify_note", + "skipped", + "error", + "error_class", + "cleanup_error", + "seeded_entity_kinds", + "randomized_seed_namespaces", +) + + +@dataclass(frozen=True, slots=True) +class Usage: + """Provider-neutral token usage for one model turn.""" + + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_input_tokens: int = 0 + cache_creation_input_tokens: int = 0 + + +@dataclass(slots=True) +class CallRecord: + """Persisted metrics for one Plane or client tool call.""" + + tool: str + args_chars: int = 0 + result_tokens: int | None = None + result_chars: int = 0 + result_kind: str = "text" + is_error: bool = False + result_tokens_estimated: bool | None = None + result_token_count_method: str | None = None + duration_ms: float | int | None = None + action: str | None = None + raw_tool: str | None = None + # Which kind of "no" an errored call received; None when the call succeeded. + error_class: str | None = None + # The request body, recorded on every driver. Without it a recorded refusal can be read + # but not attributed to the target it names, which is the question it is kept to answer. + # Long string values are truncated (see ARGS_VALUE_LIMIT): the tool surface accepts rich + # HTML, page bodies and attachment URLs, so an untruncated copy is unbounded in size. + args_json: str | None = None + result_tokens_skipped: str | None = None + # None means the response was not checked; [] means checked with no match. + observed_sentinels: list[str] | None = None + + +@dataclass +class AgentRun: + """Normalized result of one agent task execution.""" + + # Plane MCP tools only: {tool, args, origin='plane', raw_tool?} + calls: list[dict[str, Any]] + final_text: str + usage: Usage | dict[str, Any] | None + stopped_reason: str + raw_ref: str | None = None + # Client/harness built-ins (ToolSearch, Bash, …) are retained separately. + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + # Cache-aware run totals (CLI); do not put uncached-only input_tokens into cum_input_tokens + usage_total: dict[str, Any] | None = None + # Harness extras (optional; defaults keep CLI paths simple) + usage_scope: str = "run" # 'run' | 'iteration' + call_source: str = "unknown" # 'json' | 'transcript' | 'stream' | 'api' + hit_max_turns: bool = False + wall_time_s: float = 0.0 + experimental: bool = False + notes: list[str] = field(default_factory=list) + usage_per_iteration: list[Usage] = field(default_factory=list) + cum_input_tokens: int | None = None + result_pair_mismatch: bool = False + trace_integrity: bool | None = True + trace_integrity_reason: TraceIntegrityReason | None = None + tool_manifest_fingerprint: str | None = None + token_count_failures: int = 0 + # False means a tokenizer/backend counter was used for every result; True + # means at least one result used the shared character estimate. None lets + # the common row mapper determine the status from the recorded calls. + result_tokens_estimated: bool | None = None + evidence_trace_available: bool = False + provider: str | None = None + model: str | None = None + requested_model: str | None = None + # Raw provider finish/stop value. API drivers keep this beside the + # harness-owned normalized ``stopped_reason`` for diagnostics. + provider_stop_reason: str | None = None + + +@dataclass(slots=True) +class TaskResult: + """One task repetition and the complete persisted row schema. + + schema_version 0 marks rows written before this type existed; from_row defaults every + field added since. Version 1 defines wall_time_s as CLI invocation time only — earlier + Claude/Antigravity/OpenCode rows also include a few ms of harness setup. Version 2 adds + run-completeness metadata and cleanup failure recording. Version 3 records only + response-evidence labels (never Plane response bodies) plus trace availability. Version + 4 adds the task-local question fingerprint used by future intersection comparisons. + Version 5 adds typed trace integrity and the observed tool-manifest fingerprint. + Version 6 adds the reproducible per-repetition fixture seed id, non-secret fixture kinds, + and randomization namespaces. Randomized truth values are deliberately excluded; request + arguments, which name the target a call acted on, are recorded with long values truncated. + """ + + schema_version: int = RESULT_SCHEMA_VERSION + row_type: str | None = None + run_id: str = "" + fixture_seed_id: str = "" + ts: str = "" + git_sha: str = "" + battery: str = "" + task_fingerprint: str = "" + label: str = "" + driver: str = "" + provider: str | None = None + server: Literal["local", "external"] = "local" + model: str | None = None + requested_model: str | None = None + requested_tier: str | None = None + resolved_model: str | None = None + task_id: str = "" + author: str = "" + rep: int = 0 + expected_rows: int = 0 + success: bool = False + verify_note: str = "" + skipped: str | None = None + error: str | None = None + error_class: str | None = None + cleanup_error: str | None = None + seeded_entity_kinds: list[str] = field(default_factory=list) + randomized_seed_namespaces: list[str] = field(default_factory=list) + final_text: str = "" + stop_reason: str | None = None + provider_stop_reason: str | None = None + hit_max_iterations: bool = False + result_pair_mismatch: bool = False + trace_integrity: bool | None = True + trace_integrity_reason: TraceIntegrityReason | None = None + tool_manifest_fingerprint: str | None = None + token_count_failures: int = 0 + result_tokens_estimated: bool | None = None + calls: list[CallRecord] = field(default_factory=list) + num_calls: int = 0 + errored_calls: int = 0 + total_result_tokens: int = 0 + usage_per_iteration: list[Usage] = field(default_factory=list) + cum_input_tokens: int | None = 0 + cum_input_tokens_reason: str | None = None + wall_time_s: float = 0.0 + client_tool_calls: list[CallRecord] = field(default_factory=list) + client_tool_call_count: int = 0 + result_tokens_mode: str | None = None + result_token_count_method: str | None = None + usage_scope: str | None = None + call_source: str | None = None + evidence_trace_available: bool = False + driver_raw_ref: str | None = None + driver_notes: list[str] = field(default_factory=list) + usage: Usage | dict[str, Any] | None = None + usage_total: dict[str, Any] | None = None + result_tokens_skipped_reason: str | None = None + + def apply_agent_result(self, agent: TaskResult) -> None: + """Copy the driver-owned portion of an agent result onto this task row.""" + for field_name in AGENT_RESULT_COPY_FIELDS: + setattr(self, field_name, getattr(agent, field_name)) + for field_name in AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS: + value = getattr(agent, field_name) + if value is not None: + setattr(self, field_name, value) + + def to_row(self) -> dict[str, Any]: + """Serialize the versioned persisted JSONL row schema. + + Per-iteration usage deliberately retains the established short keys + ``in``, ``out``, ``cache_read``, and ``cache_write``. Nothing in this + repository reads those keys; they are archival data for humans and + ad-hoc analysis, so the on-disk spelling remains stable here. + """ + + def usage_row(item: Usage) -> dict[str, int]: + return { + "in": item.input_tokens, + "out": item.output_tokens, + "cache_read": item.cache_read_input_tokens, + "cache_write": item.cache_creation_input_tokens, + } + + calls: list[dict[str, Any]] = [] + for call in self.calls: + item: dict[str, Any] = { + "tool": call.tool, + "args_chars": call.args_chars, + "result_tokens": call.result_tokens, + "result_chars": call.result_chars, + "result_kind": call.result_kind, + "is_error": call.is_error, + "result_tokens_estimated": call.result_tokens_estimated, + "result_token_count_method": call.result_token_count_method, + } + if call.duration_ms is not None: + item["duration_ms"] = call.duration_ms + if call.action is not None: + item["action"] = call.action + if call.result_tokens_skipped is not None: + item["result_tokens_skipped"] = call.result_tokens_skipped + if call.error_class is not None: + item["error_class"] = call.error_class + if call.args_json is not None: + item["args_json"] = call.args_json + if call.observed_sentinels is not None: + item["observed_sentinels"] = list(call.observed_sentinels) + calls.append(item) + + client_calls = [ + { + "tool": call.tool, + "args_chars": call.args_chars, + "raw_tool": call.raw_tool or call.tool, + } + for call in self.client_tool_calls + ] + row: dict[str, Any] = { + "schema_version": self.schema_version, + "run_id": self.run_id, + "fixture_seed_id": self.fixture_seed_id, + "ts": self.ts, + "git_sha": self.git_sha, + "battery": self.battery, + "task_fingerprint": self.task_fingerprint, + "label": self.label, + "driver": self.driver, + "provider": self.provider, + "server": self.server, + "model": self.model, + "requested_model": self.requested_model, + "requested_tier": self.requested_tier, + "resolved_model": self.resolved_model, + "task_id": self.task_id, + "author": self.author, + "rep": self.rep, + "expected_rows": self.expected_rows, + "success": self.success, + "verify_note": self.verify_note, + "skipped": self.skipped, + "error": self.error, + "error_class": self.error_class, + "cleanup_error": self.cleanup_error, + "seeded_entity_kinds": list(self.seeded_entity_kinds), + "randomized_seed_namespaces": list(self.randomized_seed_namespaces), + "final_text": self.final_text, + "stop_reason": self.stop_reason, + "provider_stop_reason": self.provider_stop_reason, + "hit_max_iterations": self.hit_max_iterations, + "result_pair_mismatch": self.result_pair_mismatch, + "trace_integrity": self.trace_integrity, + "trace_integrity_reason": self.trace_integrity_reason, + "tool_manifest_fingerprint": self.tool_manifest_fingerprint, + "token_count_failures": self.token_count_failures, + "result_tokens_estimated": self.result_tokens_estimated, + "calls": calls, + "num_calls": self.num_calls, + "errored_calls": self.errored_calls, + "total_result_tokens": self.total_result_tokens, + "usage_per_iteration": [usage_row(item) for item in self.usage_per_iteration], + "cum_input_tokens": self.cum_input_tokens, + "cum_input_tokens_reason": self.cum_input_tokens_reason, + "wall_time_s": self.wall_time_s, + "client_tool_calls": client_calls, + "client_tool_call_count": self.client_tool_call_count, + "result_tokens_mode": self.result_tokens_mode, + "result_token_count_method": self.result_token_count_method, + "usage_scope": self.usage_scope, + "call_source": self.call_source, + "evidence_trace_available": self.evidence_trace_available, + "driver_raw_ref": self.driver_raw_ref, + "driver_notes": list(self.driver_notes), + "usage": usage_row(self.usage) if isinstance(self.usage, Usage) else self.usage, + "usage_total": self.usage_total, + } + if self.row_type is not None: + row["row_type"] = self.row_type + if self.result_tokens_skipped_reason is not None: + row["result_tokens_skipped_reason"] = self.result_tokens_skipped_reason + return row + + @classmethod + def from_row(cls, row: dict[str, Any]) -> TaskResult: + """Read a persisted row, retaining defaults for unrelated older fields.""" + raw_calls = row.get("calls") if isinstance(row.get("calls"), list) else [] + calls: list[CallRecord] = [] + for raw in raw_calls: + if not isinstance(raw, dict): + continue + calls.append( + CallRecord( + tool=str(raw.get("tool") or ""), + args_chars=int(raw.get("args_chars") or 0), + result_tokens=(int(raw["result_tokens"]) if raw.get("result_tokens") is not None else None), + result_chars=int(raw.get("result_chars") or 0), + result_kind=str(raw.get("result_kind") or "text"), + is_error=bool(raw.get("is_error")), + error_class=(str(raw["error_class"]) if raw.get("error_class") is not None else None), + result_tokens_estimated=( + bool(raw["result_tokens_estimated"]) if raw.get("result_tokens_estimated") is not None else None + ), + result_token_count_method=( + str(raw["result_token_count_method"]) + if raw.get("result_token_count_method") is not None + else None + ), + duration_ms=raw.get("duration_ms"), + action=(str(raw["action"]) if raw.get("action") is not None else None), + args_json=(str(raw["args_json"]) if raw.get("args_json") is not None else None), + result_tokens_skipped=( + str(raw["result_tokens_skipped"]) if raw.get("result_tokens_skipped") is not None else None + ), + observed_sentinels=( + [str(value) for value in raw["observed_sentinels"]] + if isinstance(raw.get("observed_sentinels"), list) + else None + ), + ) + ) + + raw_client_calls = row.get("client_tool_calls") if isinstance(row.get("client_tool_calls"), list) else [] + client_calls: list[CallRecord] = [] + for raw in raw_client_calls: + if not isinstance(raw, dict): + continue + client_calls.append( + CallRecord( + tool=str(raw.get("tool") or raw.get("raw_tool") or ""), + args_chars=int(raw.get("args_chars") or 0), + raw_tool=str(raw.get("raw_tool") or raw.get("tool") or ""), + ) + ) + + raw_usage = row.get("usage_per_iteration") + usage_per_iteration: list[Usage] = [] + if isinstance(raw_usage, list): + for item in raw_usage: + if not isinstance(item, dict): + continue + usage_per_iteration.append( + Usage( + input_tokens=int(item.get("in") or 0), + output_tokens=int(item.get("out") or 0), + cache_read_input_tokens=int(item.get("cache_read") or 0), + cache_creation_input_tokens=int(item.get("cache_write") or 0), + ) + ) + + return cls( + schema_version=int(row.get("schema_version") or 0), + row_type=(str(row["row_type"]) if row.get("row_type") is not None else None), + run_id=str(row.get("run_id") or ""), + fixture_seed_id=str(row.get("fixture_seed_id") or ""), + ts=str(row.get("ts") or ""), + git_sha=str(row.get("git_sha") or ""), + battery=str(row.get("battery") or ""), + task_fingerprint=str(row.get("task_fingerprint") or ""), + label=str(row.get("label") or ""), + driver=str(row.get("driver") or ""), + provider=(str(row["provider"]) if row.get("provider") is not None else None), + server="external" if row.get("server") == "external" else "local", + model=(str(row["model"]) if row.get("model") is not None else None), + requested_model=(str(row["requested_model"]) if row.get("requested_model") is not None else None), + requested_tier=(str(row["requested_tier"]) if row.get("requested_tier") is not None else None), + resolved_model=(str(row["resolved_model"]) if row.get("resolved_model") is not None else None), + task_id=str(row.get("task_id") or ""), + author=str(row.get("author") or ""), + rep=int(row.get("rep") or 0), + expected_rows=int(row.get("expected_rows") or 0), + success=bool(row.get("success")), + verify_note=str(row.get("verify_note") or ""), + skipped=(str(row["skipped"]) if row.get("skipped") is not None else None), + error=(str(row["error"]) if row.get("error") is not None else None), + error_class=(str(row["error_class"]) if row.get("error_class") is not None else None), + cleanup_error=(str(row["cleanup_error"]) if row.get("cleanup_error") is not None else None), + seeded_entity_kinds=( + [str(kind) for kind in row["seeded_entity_kinds"]] + if isinstance(row.get("seeded_entity_kinds"), list) + else [] + ), + randomized_seed_namespaces=( + [str(namespace) for namespace in row["randomized_seed_namespaces"]] + if isinstance(row.get("randomized_seed_namespaces"), list) + else [] + ), + final_text=str(row.get("final_text") or ""), + stop_reason=(str(row["stop_reason"]) if row.get("stop_reason") is not None else None), + provider_stop_reason=( + str(row["provider_stop_reason"]) if row.get("provider_stop_reason") is not None else None + ), + hit_max_iterations=bool(row.get("hit_max_iterations")), + result_pair_mismatch=bool(row.get("result_pair_mismatch")), + trace_integrity=(bool(row["trace_integrity"]) if row.get("trace_integrity") is not None else None), + trace_integrity_reason=( + str(row["trace_integrity_reason"]) + if row.get("trace_integrity_reason") + in { + "recorder_loss", + "protocol_violation", + "result_pair_mismatch", + } + else None + ), + tool_manifest_fingerprint=( + str(row["tool_manifest_fingerprint"]) if row.get("tool_manifest_fingerprint") is not None else None + ), + token_count_failures=int(row.get("token_count_failures") or 0), + result_tokens_estimated=( + bool(row["result_tokens_estimated"]) if row.get("result_tokens_estimated") is not None else None + ), + calls=calls, + num_calls=int(row.get("num_calls") if row.get("num_calls") is not None else len(calls)), + errored_calls=int( + row.get("errored_calls") + if row.get("errored_calls") is not None + else sum(1 for call in calls if call.is_error) + ), + total_result_tokens=int( + row.get("total_result_tokens") + if row.get("total_result_tokens") is not None + else sum(call.result_tokens or 0 for call in calls) + ), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=(int(row["cum_input_tokens"]) if row.get("cum_input_tokens") is not None else None), + cum_input_tokens_reason=( + str(row["cum_input_tokens_reason"]) if row.get("cum_input_tokens_reason") is not None else None + ), + wall_time_s=float(row.get("wall_time_s") or 0.0), + client_tool_calls=client_calls, + client_tool_call_count=int( + row.get("client_tool_call_count") + if row.get("client_tool_call_count") is not None + else len(client_calls) + ), + result_tokens_mode=(str(row["result_tokens_mode"]) if row.get("result_tokens_mode") is not None else None), + result_token_count_method=( + str(row["result_token_count_method"]) if row.get("result_token_count_method") is not None else None + ), + usage_scope=(str(row["usage_scope"]) if row.get("usage_scope") is not None else None), + call_source=(str(row["call_source"]) if row.get("call_source") is not None else None), + evidence_trace_available=bool(row.get("evidence_trace_available")), + driver_raw_ref=(str(row["driver_raw_ref"]) if row.get("driver_raw_ref") is not None else None), + driver_notes=[str(item) for item in row.get("driver_notes") or []], + usage=row.get("usage") if isinstance(row.get("usage"), dict) else None, + usage_total=(row.get("usage_total") if isinstance(row.get("usage_total"), dict) else None), + result_tokens_skipped_reason=( + str(row["result_tokens_skipped_reason"]) + if row.get("result_tokens_skipped_reason") is not None + else None + ), + ) + + +#: Longest string value kept inside a recorded argument dict. Identifiers, actions and +#: enum values are far shorter than this, so what gets cut is prose: descriptions, HTML +#: bodies, comment text and query strings. A 1MB description_html would otherwise be copied +#: verbatim into the result file, which is neither useful for analysis nor safe to assume small. +ARGS_VALUE_LIMIT = 256 + +_TRUNCATION_MARK = "\u2026[truncated]" + + +def _bounded_args(args: dict[str, Any]) -> dict[str, Any]: + """Copy an argument dict with long string values cut to a fixed length. + + Structure is preserved so the target of a call stays legible; only bulk content + is dropped. Nested containers are bounded through their serialized form, since a + list of ids is worth keeping whole and a list of page bodies is not. + """ + bounded: dict[str, Any] = {} + for name, value in args.items(): + if isinstance(value, str) and len(value) > ARGS_VALUE_LIMIT: + bounded[name] = value[:ARGS_VALUE_LIMIT] + _TRUNCATION_MARK + elif isinstance(value, (list, tuple, dict)): + try: + encoded = json.dumps(value, default=str, ensure_ascii=False) + except Exception: + encoded = str(value) + bounded[name] = value if len(encoded) <= ARGS_VALUE_LIMIT else encoded[:ARGS_VALUE_LIMIT] + _TRUNCATION_MARK + else: + bounded[name] = value + return bounded + + +def agent_run_to_task_result( + run: AgentRun, +) -> TaskResult: + """Map an ``AgentRun`` onto the typed driver-owned portion of a task result. + + Only Plane MCP tools count toward num_calls; client built-ins go to client_tool_calls. + CLI drivers never fill cum_input_tokens from bare usage.input_tokens — under Claude + Code that is uncached-only and misreads cached runs as ~10 tokens. + """ + # Re-split in case callers passed a mixed list + plane_src, client_extra = split_plane_and_client_calls(list(run.calls)) + client_src = list(run.client_tool_calls) + client_extra + + is_cli = run.call_source in ("json", "transcript", "stream", "proxy") or run.usage_scope == "run" + calls: list[CallRecord] = [] + local_token_count_failures = 0 + for c in plane_src: + tool = c.get("tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + result_chars = int(c["result_chars"]) if c.get("result_chars") is not None else 0 + result_tokens = c.get("result_tokens") + estimated = c.get("result_tokens_estimated") + count_method = c.get("result_token_count_method") + if result_tokens is not None: + result_tokens = int(result_tokens) + if estimated is None: + estimated = bool(run.result_tokens_estimated) + if count_method is None: + count_method = TOKEN_ESTIMATE_METHOD if estimated else "backend" + elif isinstance(c.get("result_text"), str): + count = count_result_text_tokens(c["result_text"]) + result_tokens = count.value + estimated = count.estimated + count_method = count.method + local_token_count_failures += int(count.tokenizer_failed) + else: + result_tokens = estimate_result_tokens(result_chars) + estimated = True + count_method = TOKEN_ESTIMATE_METHOD + + rec = CallRecord( + tool=str(tool), + args_chars=args_chars, + result_tokens=result_tokens, + result_chars=result_chars, + result_kind=str(c.get("result_kind") or "text"), + is_error=bool(c.get("is_error")), + error_class=(str(c["error_class"]) if c.get("error_class") is not None else None), + result_tokens_estimated=bool(estimated), + result_token_count_method=str(count_method), + duration_ms=c.get("duration_ms"), + observed_sentinels=( + [str(value) for value in c["observed_sentinels"]] + if isinstance(c.get("observed_sentinels"), list) + else None + ), + ) + # Action-dispatch surfaces: the action arg IS the second half of the + # tool choice — keep it (args content is otherwise not persisted). + if isinstance(args, dict) and isinstance(args.get("action"), str): + rec.action = args["action"] + # Arguments are recorded for every driver. They were previously conditioned on + # result_text, which only the recording proxy sets -- so the api driver, which + # calls tools directly and never goes through the proxy, recorded arguments on + # none of its calls while a CLI arm recorded them on all of theirs. That is a + # coupling to an unrelated flag rather than a policy: the arguments are in hand + # on both paths, args_chars is already computed from them, and `action` above is + # already persisted unconditionally. What this adds to a result file is ids and + # short strings. + if isinstance(args, dict) and args: + try: + rec.args_json = json.dumps(_bounded_args(args), default=str, ensure_ascii=False) + except Exception: + rec.args_json = str(args)[:ARGS_VALUE_LIMIT] + calls.append(rec) + + client_tool_calls: list[CallRecord] = [] + for c in client_src: + tool = c.get("tool") or c.get("raw_tool") or "" + args = c.get("args") or {} + try: + args_chars = len(json.dumps(args, default=str)) + except Exception: + args_chars = len(str(args)) + client_tool_calls.append( + CallRecord( + tool=str(tool), + args_chars=args_chars, + raw_tool=str(c.get("raw_tool") or tool), + ) + ) + + stop_reason = run.stopped_reason + hit_max = run.hit_max_turns + if hit_max: + stop_reason = stop_reason if stop_reason not in ("end_turn", "completed", None, "") else "max_turns" + + errored = sum(1 for c in calls if c.is_error) + + # CLI path: never write misleading cum_input_tokens from uncached-only field. + # usage_total is driver-owned — do not re-derive it here (Claude vs Codex + # shapes differ; a generic Claude rebuild mislabels other vendors). + usage_total = run.usage_total + + if run.usage_per_iteration: + usage_per_iteration = list(run.usage_per_iteration) + cum_input = ( + run.cum_input_tokens + if run.cum_input_tokens is not None + else sum(item.input_tokens for item in usage_per_iteration) + ) + cum_reason = None + elif is_cli: + cum_input: int | None = None + cum_reason: str | None = ( + "CLI driver: Claude usage.input_tokens is uncached-only; " + "see usage_total (cache_read/cache_creation/output/cost) for run accounting" + ) + usage_per_iteration: list[Usage] = [] + else: + cum_input = 0 + cum_reason = None + usage_per_iteration = [] + + estimated_states = [bool(c.result_tokens_estimated) for c in calls] + if estimated_states: + result_tokens_estimated = any(estimated_states) + result_tokens_mode = ( + "estimated" if all(estimated_states) else "measured" if not any(estimated_states) else "mixed" + ) + else: + result_tokens_estimated = ( + bool(run.result_tokens_estimated) if run.result_tokens_estimated is not None else is_cli + ) + result_tokens_mode = "estimated" if result_tokens_estimated else "measured" + + count_methods = {str(c.result_token_count_method) for c in calls} + if not count_methods: + result_token_count_method = "none" + elif len(count_methods) == 1: + result_token_count_method = next(iter(count_methods)) + else: + result_token_count_method = "mixed" + return TaskResult( + final_text=run.final_text, + calls=calls, + num_calls=len(calls), + client_tool_calls=client_tool_calls, + client_tool_call_count=len(client_tool_calls), + errored_calls=errored, + total_result_tokens=sum(int(c.result_tokens or 0) for c in calls), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=cum_input, + cum_input_tokens_reason=cum_reason, + wall_time_s=run.wall_time_s, + stop_reason=stop_reason, + provider_stop_reason=run.provider_stop_reason, + hit_max_iterations=hit_max, + result_pair_mismatch=run.result_pair_mismatch, + trace_integrity=run.trace_integrity, + trace_integrity_reason=run.trace_integrity_reason, + tool_manifest_fingerprint=run.tool_manifest_fingerprint, + token_count_failures=run.token_count_failures + local_token_count_failures, + result_tokens_estimated=result_tokens_estimated, + result_tokens_mode=result_tokens_mode, + result_token_count_method=result_token_count_method, + usage_scope=run.usage_scope, + call_source=run.call_source, + evidence_trace_available=run.evidence_trace_available, + driver_raw_ref=run.raw_ref, + driver_notes=list(run.notes), + usage=run.usage, + usage_total=usage_total, + provider=run.provider, + model=run.model, + requested_model=run.requested_model, + ) + + +def agent_run_to_harness_dict( + run: AgentRun, +) -> dict[str, Any]: + """Map an agent run to the public persisted-row dictionary.""" + return agent_run_to_task_result(run).to_row() + + +__all__ = [ + "AGENT_RESULT_COPY_FIELDS", + "AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS", + "RESULT_SCHEMA_VERSION", + "TRACE_INTEGRITY_SCHEMA_VERSION", + "TASK_RESULT_HARNESS_FIELDS", + "AgentRun", + "CallRecord", + "TaskResult", + "TraceIntegrityReason", + "Usage", + "agent_run_to_harness_dict", + "agent_run_to_task_result", +] diff --git a/evals/core/server_env.py b/evals/core/server_env.py new file mode 100644 index 0000000..65bc05a --- /dev/null +++ b/evals/core/server_env.py @@ -0,0 +1,30 @@ +"""Environment construction for a stdio MCP server child process. + +A foundational leaf, shared by the live runner and the standalone tool-token listing. It +lived in ``runner.live``, so importing a pure token-counting helper pulled in the whole +live-run composition root — every driver, seeder, task and report module with it. +""" + +from __future__ import annotations + +import os + +DEFAULT_PLANE_BASE_URL = "https://api.plane.so" + + +def stdio_server_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from scratch — never inherit os.environ (F6).""" + environment: dict[str, str] = {} + if path := os.environ.get("PATH"): + environment["PATH"] = path + if home := os.environ.get("HOME"): + environment["HOME"] = home + environment["PLANE_API_KEY"] = os.environ["EVAL_PLANE_API_KEY"] + environment["PLANE_WORKSPACE_SLUG"] = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + environment["PLANE_BASE_URL"] = os.environ.get("EVAL_PLANE_BASE_URL", DEFAULT_PLANE_BASE_URL) + if extra: + environment.update(extra) + return environment + + +__all__ = ["DEFAULT_PLANE_BASE_URL", "stdio_server_env"] diff --git a/evals/core/state_oracle.py b/evals/core/state_oracle.py new file mode 100644 index 0000000..9203907 --- /dev/null +++ b/evals/core/state_oracle.py @@ -0,0 +1,41 @@ +"""Neutral Plane response normalization shared by seeders and verifiers.""" + +from __future__ import annotations + +from typing import Any + + +def state_name_group_pairs(rows: list[Any]) -> list[str]: + """Return exact ``NAME | group: GROUP`` pairs, rejecting incomplete rows.""" + pairs: list[str] = [] + for state in rows: + name = str(getattr(state, "name", None) or "").strip() + raw_group = getattr(state, "group", None) + group = str(getattr(raw_group, "value", raw_group) or "").strip() + if not name or not group: + raise RuntimeError(f"project state lacks name or group: {state!r}") + pairs.append(f"{name} | group: {group}") + return pairs + + +def worklog_summary_item_ids(summary: Any) -> list[str]: + """Return the distinct work item ids a project worklog summary reports, in order. + + Plane spells the field ``issue_id`` on some payload shapes and ``work_item_id`` on + others. The seeder builds L1's oracle from this and the verifier compares against it, + so they must read the payload identically. + """ + raw = summary if isinstance(summary, list) else (getattr(summary, "results", None) or summary or []) + item_ids: list[str] = [] + for row in list(raw or []): + dump = row.model_dump() if hasattr(row, "model_dump") else (row if isinstance(row, dict) else {}) + value = getattr(row, "issue_id", None) or getattr(row, "work_item_id", None) + if value is None and isinstance(dump, dict): + value = dump.get("issue_id") or dump.get("work_item_id") + item_id = str(value or "").strip() + if item_id and item_id not in item_ids: + item_ids.append(item_id) + return item_ids + + +__all__ = ["state_name_group_pairs", "worklog_summary_item_ids"] diff --git a/evals/core/task_metadata.py b/evals/core/task_metadata.py new file mode 100644 index 0000000..61ff25a --- /dev/null +++ b/evals/core/task_metadata.py @@ -0,0 +1,97 @@ +"""The task facts a report needs, persisted with the run instead of read from the checkout. + +Reports derived three things from the live catalog: whether a task mutates Plane, its prompt +text, and the fixtures it needs (which decides whether a plan-gated skip was expected). All +three are properties of *the run that was executed*, so reading them from the working tree +meant a result file could be reinterpreted after the catalog changed — the one thing the +battery fingerprint and identity validation exist to prevent. + +The run writes them into its meta header. A file that predates the header has no metadata, +and the reader says so rather than quietly substituting today's catalog. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +TaskMetadata = Mapping[str, Mapping[str, Any]] + +METADATA_FIELD = "task_metadata" +MUTATION_TAGS = frozenset({"write"}) + + +def build_task_metadata(tasks: Iterable[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: + """Capture the report-relevant facts of the tasks this run is about to execute.""" + metadata: dict[str, dict[str, Any]] = {} + for task in tasks: + task_id = str(task.get("id") or "") + if not task_id: + continue + metadata[task_id] = { + "tags": sorted(str(tag) for tag in (task.get("tags") or ())), + "needs": sorted(str(need) for need in (task.get("needs") or ())), + "prompt": str(task.get("prompt") or ""), + } + return metadata + + +def normalize_task_metadata(value: Any) -> dict[str, dict[str, Any]]: + """Validate a persisted metadata map, dropping entries that cannot be trusted.""" + if not isinstance(value, Mapping): + return {} + normalized: dict[str, dict[str, Any]] = {} + for raw_id, raw_entry in value.items(): + task_id = str(raw_id or "").strip() + if not task_id or not isinstance(raw_entry, Mapping): + continue + normalized[task_id] = { + "tags": sorted(str(tag) for tag in (raw_entry.get("tags") or ()) if str(tag)), + "needs": sorted(str(need) for need in (raw_entry.get("needs") or ()) if str(need)), + "prompt": str(raw_entry.get("prompt") or ""), + } + return normalized + + +def task_metadata_from_rows(rows: Iterable[Any]) -> dict[str, dict[str, Any]]: + """Merge the metadata declared by every meta header in the loaded rows.""" + merged: dict[str, dict[str, Any]] = {} + for row in rows: + if not isinstance(row, Mapping) or row.get("row_type") != "meta": + continue + merged.update(normalize_task_metadata(row.get(METADATA_FIELD))) + return merged + + +def entry_requires_mutation(entry: Mapping[str, Any] | None) -> bool: + """Whether a task was expected to change Plane state, from its persisted tags.""" + if not isinstance(entry, Mapping): + return False + return bool(MUTATION_TAGS.intersection(str(tag) for tag in (entry.get("tags") or ()))) + + +def entry_needs(entry: Mapping[str, Any] | None) -> tuple[str, ...]: + """The fixtures a task declared, from its persisted needs.""" + if not isinstance(entry, Mapping): + return () + return tuple(str(need) for need in (entry.get("needs") or ()) if str(need)) + + +def entry_prompt(entry: Mapping[str, Any] | None) -> str: + """The prompt text a task ran with, from its persisted metadata.""" + if not isinstance(entry, Mapping): + return "" + return str(entry.get("prompt") or "") + + +__all__ = [ + "METADATA_FIELD", + "MUTATION_TAGS", + "TaskMetadata", + "build_task_metadata", + "entry_needs", + "entry_prompt", + "entry_requires_mutation", + "normalize_task_metadata", + "task_metadata_from_rows", +] diff --git a/evals/core/token_accounting.py b/evals/core/token_accounting.py new file mode 100644 index 0000000..f3cad08 --- /dev/null +++ b/evals/core/token_accounting.py @@ -0,0 +1,189 @@ +"""One reading of ``usage_total``, whatever driver produced it. + +``usage_total.input_tokens`` does not mean the same thing everywhere, and reading it +naively misprices by roughly 4x on one side of any cross-driver comparison: + + inclusive the field already contains the cached reads (OpenAI Responses). The + uncached portion is the remainder. + exclusive the field is net of cache, and the cached reads sit beside it + (Anthropic Messages, and every CLI vendor measured). + +Driver family cannot decide this. The same api driver runs both providers, so an +Anthropic arm and an OpenAI arm arrive with identical ``source: "iterations"`` and +opposite meanings. What decides it, in order of authority: + + declared the driver recorded ``cache_semantics`` outright. Always trusted. + explicit_total ``total_input_tokens_including_cache`` is present, which only an + exclusive shape carries, and it states the total directly. + no_cache nothing was cached, so both readings coincide and no guess is + needed -- this covers unknown models safely. + model_family inferred from the model name, the last resort for rows recorded + before ``cache_semantics`` existed. + +When none of those apply the answer is ``None``. Refusing is deliberate: a guess here +is invisible in the output and wrong by a factor that reverses conclusions. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +INCLUSIVE = "inclusive" +EXCLUSIVE = "exclusive" + +#: Substrings that identify a provider's cache convention from a model name. +#: Only families whose semantics have been verified appear here; an unmatched name +#: with cache activity yields ``None`` rather than a default. +_MODEL_FAMILIES: tuple[tuple[tuple[str, ...], str], ...] = ( + (("claude", "anthropic"), EXCLUSIVE), + (("gpt", "o1-", "o3-", "o4-", "openai"), INCLUSIVE), +) + + +@dataclass(frozen=True) +class TokenAccounting: + """Input split into what was paid for fresh, read from cache, and written to it. + + ``uncached_input + cached_input + cache_creation == total_input`` always holds, so + a caller can price the three parts at three rates without knowing the source shape. + """ + + uncached_input: int + cached_input: int + cache_creation: int + output: int + total_input: int + semantics: str + semantics_source: str + + +def _int(usage: Mapping[str, Any], name: str) -> int: + try: + return max(0, int(usage.get(name) or 0)) + except (TypeError, ValueError): + return 0 + + +def _family_semantics(model: str | None) -> str | None: + lowered = (model or "").strip().lower() + if not lowered: + return None + for markers, semantics in _MODEL_FAMILIES: + if any(marker in lowered for marker in markers): + return semantics + return None + + +#: The fields that carry an actual measurement, as opposed to describing one. +_TOKEN_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + "total_input_tokens_including_cache", +) + + +def has_token_counts(usage_total: Mapping[str, Any] | None) -> bool: + """True when this usage carries a real measurement rather than only metadata. + + The api driver writes a ``usage_total`` on every run whether or not any turn + reported usage, so a dict holding a ``source`` and a ``cache_semantics`` and no + counts is routine. Pricing that at $0.00 is the "unknown reads as free" failure + this module exists to prevent, so an all-zero shape counts as no measurement. + """ + if not usage_total: + return False + return any(_int(usage_total, name) for name in _TOKEN_FIELDS) + + +def cache_semantics_of(usage_total: Mapping[str, Any] | None, *, model: str | None = None) -> str | None: + """Return how this row's ``input_tokens`` treats cache, or None if undecidable.""" + if not usage_total: + return None + declared = usage_total.get("cache_semantics") + if declared in (INCLUSIVE, EXCLUSIVE): + return str(declared) + if usage_total.get("total_input_tokens_including_cache") is not None: + return EXCLUSIVE + if _int(usage_total, "cache_read_input_tokens") + _int(usage_total, "cache_creation_input_tokens") == 0: + # Both readings agree when nothing was cached. + return INCLUSIVE + return _family_semantics(model) + + +def normalize_usage( + usage_total: Mapping[str, Any] | None, + *, + model: str | None = None, +) -> TokenAccounting | None: + """Normalise one row's usage, or None when it is absent or undecidable. + + A caller distinguishes the two None cases by the input: a falsy ``usage_total`` + means the driver recorded no usage at all (report it as unmeasured), while a + populated one means the shape could not be read (report it as unpriced). + """ + if not usage_total or not has_token_counts(usage_total): + return None + + cached = _int(usage_total, "cache_read_input_tokens") + creation = _int(usage_total, "cache_creation_input_tokens") + output = _int(usage_total, "output_tokens") + reported_input = _int(usage_total, "input_tokens") + + declared = usage_total.get("cache_semantics") + explicit_total = usage_total.get("total_input_tokens_including_cache") + + if declared in (INCLUSIVE, EXCLUSIVE): + semantics, source = str(declared), "declared" + elif explicit_total is not None: + semantics, source = EXCLUSIVE, "explicit_total" + elif cached + creation == 0: + semantics, source = INCLUSIVE, "no_cache" + else: + inferred = _family_semantics(model) + if inferred is None: + return None + semantics, source = inferred, "model_family" + + if semantics == EXCLUSIVE: + uncached = reported_input + total = uncached + cached + creation + else: + total = reported_input + uncached = total - cached - creation + + if explicit_total is not None and total != _int(usage_total, "total_input_tokens_including_cache"): + # The vendor states the total as well as the parts. Both agreeing is what makes + # this shape self-validating; disagreement means the shape changed underneath us, + # and an unpriced row is a visible failure where a wrong price is not. + # + # Checked whenever a total is present, not only when it chose the semantics: a + # declaration may interpret the parts, but it does not get to overrule + # arithmetic that contradicts it. + return None + + if uncached < 0: + # An inclusive reading whose cache exceeds its total is not a reading at all. + return None + + return TokenAccounting( + uncached_input=uncached, + cached_input=cached, + cache_creation=creation, + output=output, + total_input=total, + semantics=semantics, + semantics_source=source, + ) + + +__all__ = [ + "EXCLUSIVE", + "INCLUSIVE", + "TokenAccounting", + "cache_semantics_of", + "normalize_usage", +] diff --git a/evals/core/token_counting.py b/evals/core/token_counting.py new file mode 100644 index 0000000..dd28661 --- /dev/null +++ b/evals/core/token_counting.py @@ -0,0 +1,75 @@ +"""Tool-result token sizing for evaluation drivers.""" + +from __future__ import annotations + +from dataclasses import dataclass + +TOKEN_ESTIMATE_METHOD = "chars_div_4" +TOKENIZER_ENCODING = "cl100k_base" + + +@dataclass(frozen=True) +class ResultTokenCount: + """A tool-result token count and how it was obtained.""" + + value: int + estimated: bool + method: str + tokenizer_failed: bool = False + + +def estimate_result_tokens(result_chars: int) -> int: + """Deterministically estimate tokens from a recorded character count.""" + chars = max(0, int(result_chars)) + if chars == 0: + return 0 + return max(1, (chars + 3) // 4) + + +def count_result_text_tokens(text: str) -> ResultTokenCount: + """Count serialized result text with tiktoken, or identify an estimate. + + The optional import stays here, in the harness analysis process. The stdlib- + only recording proxy never imports this module. + """ + try: + import tiktoken + except ImportError: + return ResultTokenCount( + estimate_result_tokens(len(text)), + estimated=True, + method=TOKEN_ESTIMATE_METHOD, + ) + except Exception: + return ResultTokenCount( + estimate_result_tokens(len(text)), + estimated=True, + method=TOKEN_ESTIMATE_METHOD, + tokenizer_failed=True, + ) + + try: + encoding = tiktoken.get_encoding(TOKENIZER_ENCODING) + encode_ordinary = getattr(encoding, "encode_ordinary", None) + tokens = encode_ordinary(text) if callable(encode_ordinary) else encoding.encode(text) + return ResultTokenCount( + len(tokens), + estimated=False, + method=f"tiktoken:{TOKENIZER_ENCODING}", + ) + except Exception: + return ResultTokenCount( + estimate_result_tokens(len(text)), + estimated=True, + method=TOKEN_ESTIMATE_METHOD, + tokenizer_failed=True, + ) + + +__all__ = [ + "TOKEN_ESTIMATE_METHOD", + "TOKENIZER_ENCODING", + "ResultTokenCount", + "count_result_text_tokens", + "estimate_result_tokens", +] diff --git a/evals/core/tool_manifest.py b/evals/core/tool_manifest.py new file mode 100644 index 0000000..5beaec8 --- /dev/null +++ b/evals/core/tool_manifest.py @@ -0,0 +1,113 @@ +"""Canonical, route-agnostic fingerprints for advertised MCP tool manifests.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any + + +def _json_value(value: Any) -> Any: + """Return a stable JSON-compatible representation, omitting null object fields.""" + dump = getattr(value, "model_dump", None) + if callable(dump): + value = dump(by_alias=True, exclude_none=True) + elif not isinstance(value, (dict, list, tuple, str, int, float, bool)) and value is not None: + try: + value = vars(value) + except TypeError: + value = str(value) + + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in value.items() if item is not None} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def canonical_tool_descriptors(tools: list[Any]) -> list[dict[str, Any]]: + """Canonicalize complete advertised descriptors and sort them by tool name.""" + descriptors: list[dict[str, Any]] = [] + for tool in tools: + descriptor = _json_value(tool) + if not isinstance(descriptor, dict): + descriptor = {"name": str(getattr(tool, "name", "")), "descriptor": descriptor} + descriptors.append(descriptor) + return sorted( + descriptors, + key=lambda item: ( + str(item.get("name") or ""), + json.dumps(item, sort_keys=True, separators=(",", ":"), ensure_ascii=False), + ), + ) + + +def tool_manifest_fingerprint(tools: list[Any]) -> str: + """Hash the complete canonical advertised tool descriptors.""" + payload = json.dumps( + canonical_tool_descriptors(tools), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def tools_page(page: Any) -> tuple[list[Any], str | None]: + """Extract the advertised tools and next cursor from a tools/list result page.""" + if isinstance(page, dict): + raw_tools = page.get("tools") + cursor = page.get("nextCursor", page.get("next_cursor")) + else: + raw_tools = getattr(page, "tools", None) + cursor = getattr(page, "nextCursor", None) + if cursor is None: + cursor = getattr(page, "next_cursor", None) + tools = list(raw_tools) if isinstance(raw_tools, (list, tuple)) else [] + return tools, str(cursor) if cursor is not None else None + + +@dataclass(slots=True) +class ToolManifestCapture: + """Aggregate one passive paginated tools/list snapshot.""" + + descriptors: list[Any] = field(default_factory=list) + expected_cursor: str | None = None + active: bool = False + fingerprint: str | None = None + + def observe_page(self, page: Any, *, request_cursor: str | None) -> None: + """Observe one response page; publish only a complete root-to-final snapshot.""" + if request_cursor is None: + self.descriptors = [] + self.expected_cursor = None + self.active = True + self.fingerprint = None + elif not self.active or request_cursor != self.expected_cursor: + self.invalidate() + return + + tools, next_cursor = tools_page(page) + self.descriptors.extend(tools) + self.expected_cursor = next_cursor + if next_cursor is None: + self.fingerprint = tool_manifest_fingerprint(self.descriptors) + self.active = False + + def invalidate(self) -> None: + """Drop a partial or stale snapshot.""" + self.descriptors = [] + self.expected_cursor = None + self.active = False + self.fingerprint = None + + +__all__ = [ + "ToolManifestCapture", + "canonical_tool_descriptors", + "tool_manifest_fingerprint", + "tools_page", +] diff --git a/evals/core/tool_names.py b/evals/core/tool_names.py new file mode 100644 index 0000000..e82d682 --- /dev/null +++ b/evals/core/tool_names.py @@ -0,0 +1,101 @@ +"""Read an MCP tool name: whose tool it is, and what to call it. + +Agent CLIs expose MCP tools under a vendor prefix (``mcp__plane__list_work_items``) +and mix them with their own built-ins (``Bash``, ``ToolSearch``). Drivers and the +result mapper both have to tell those apart before calls are counted, so this +sits beside the result schema rather than inside one driver package. +""" + +from __future__ import annotations + +import re +from typing import Any + +# mcp__plane__list_work_items → list_work_items +# mcp__plane-mcp-server__foo → foo +_MCP_PREFIX_RE = re.compile(r"^mcp__[^_]+(?:_[^_]+)*__(.+)$") +# Alternate: mcp__server__tool with multi-segment server names +_MCP_PREFIX_RE2 = re.compile(r"^mcp__.+?__(.+)$") + + +def strip_mcp_prefix(name: str) -> str: + """Strip Claude/Codex MCP tool name prefixes. + + Examples: + mcp__plane__list_work_items → list_work_items + mcp__plane-mcp-server__find_work_items → find_work_items + """ + if not name: + return name + m = _MCP_PREFIX_RE2.match(name) + if m: + return m.group(1) + return name + + +def is_plane_mcp_tool(name: str) -> bool: + """True when the raw tool name is from our Plane MCP server (pre-strip). + + Claude surfaces MCP tools as ``mcp____``. Our config registers + the server as ``plane``, so names look like ``mcp__plane__find_work_items``. + Built-ins (``ToolSearch``, ``Bash``, …) have no ``mcp__`` prefix. + """ + if not name: + return False + # mcp__plane__tool or mcp__plane-foo__tool + return name.startswith("mcp__plane__") or name.startswith("mcp__plane-") + + +def normalize_tool_call(name: str, args: Any) -> dict[str, Any]: + """Tag a tool call as Plane or client-owned.""" + raw = str(name or "") + if not isinstance(args, dict): + args = {"_raw": args} + if is_plane_mcp_tool(raw): + return { + "tool": strip_mcp_prefix(raw), + "args": args, + "origin": "plane", + "raw_tool": raw, + } + return { + "tool": raw, # keep built-in name as-is (ToolSearch, Bash, …) + "args": args, + "origin": "client", + "raw_tool": raw, + } + + +def split_plane_and_client_calls( + calls: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Partition tagged calls into plane vs client lists. + + Prefer explicit ``origin`` from ``normalize_tool_call``. Untagged calls + (API path) default to plane so existing harness behavior is unchanged. + """ + plane: list[dict[str, Any]] = [] + client: list[dict[str, Any]] = [] + for c in calls: + origin = c.get("origin") + if origin is None: + raw = str(c.get("raw_tool") or c.get("tool") or "") + if is_plane_mcp_tool(raw): + origin = "plane" + elif raw.startswith("mcp__"): + origin = "client" # other MCP server + else: + origin = "plane" # bare name → assume plane (API) + if origin == "client": + client.append(c) + else: + plane.append(c) + return plane, client + + +__all__ = [ + "is_plane_mcp_tool", + "normalize_tool_call", + "split_plane_and_client_calls", + "strip_mcp_prefix", +] diff --git a/evals/drivers/__init__.py b/evals/drivers/__init__.py new file mode 100644 index 0000000..1de4cbe --- /dev/null +++ b/evals/drivers/__init__.py @@ -0,0 +1,50 @@ +"""Agent drivers: run one task against a tool surface and return an ``AgentRun``. + +The ``api`` driver owns a provider-neutral loop; CLI drivers spawn locally installed +agent CLIs on the user's own subscription. Probed CLI details live with each vendor. + +Only the registry lives here, and each driver is imported inside the branch that returns +it. This file used to re-export forty names, most of them vendor internals reached only by +tests — and because Python runs a package's ``__init__`` before any submodule, that wall +made *every* consumer load all five agent CLIs. Importing the API backend, which shares +nothing with them, pulled in the whole CLI tree. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from evals.drivers.api.driver import ApiDriver + from evals.drivers.cli.base import CliDriver + +KNOWN_DRIVERS = frozenset({"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"}) + + +def get_driver(name: str, **kwargs: Any) -> ApiDriver | CliDriver: + """Return a driver instance, loading only the surface it names.""" + key = (name or "api").strip().lower() + if key == "api": + from evals.drivers.api.driver import ApiDriver + + return ApiDriver(**kwargs) + if key == "claude-cli": + from evals.drivers.cli.claude import ClaudeCliDriver + + return ClaudeCliDriver(**kwargs) + if key == "codex-cli": + from evals.drivers.cli.codex import CodexCliDriver + + return CodexCliDriver(**kwargs) + if key == "antigravity-cli": + from evals.drivers.cli.antigravity import AntigravityCliDriver + + return AntigravityCliDriver(**kwargs) + if key == "opencode-cli": + from evals.drivers.cli.opencode import OpencodeCliDriver + + return OpencodeCliDriver(**kwargs) + raise ValueError(f"unknown driver {name!r}; expected one of {sorted(KNOWN_DRIVERS)}") + + +__all__ = ["KNOWN_DRIVERS", "get_driver"] diff --git a/evals/drivers/api/__init__.py b/evals/drivers/api/__init__.py new file mode 100644 index 0000000..68ac54f --- /dev/null +++ b/evals/drivers/api/__init__.py @@ -0,0 +1,51 @@ +"""Provider-generic API driver and registered backend translations.""" + +# Import built-in adapters for their registrations. Each adapter owns its SDK +# translation and keeps its optional SDK import lazy until construction. +from evals.drivers.api.anthropic import AnthropicBackend +from evals.drivers.api.base import ( + BACKEND_REGISTRY, + KNOWN_API_PROVIDERS, + MODEL_TIERS, + BackendFactory, + BackendRegistration, + BackendRegistry, + ModelBackend, + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + UnmappedModelTierError, + Usage, + backend_model_aliases, + create_backend, + register_backend, + resolve_backend_model, + unregister_backend, +) +from evals.drivers.api.openai import OpenAIBackend + +__all__ = [ + "BACKEND_REGISTRY", + "KNOWN_API_PROVIDERS", + "MODEL_TIERS", + "AnthropicBackend", + "BackendFactory", + "BackendRegistration", + "BackendRegistry", + "ModelBackend", + "OpenAIBackend", + "StopReason", + "ToolCall", + "ToolResult", + "ToolSpec", + "Turn", + "UnmappedModelTierError", + "Usage", + "backend_model_aliases", + "create_backend", + "register_backend", + "resolve_backend_model", + "unregister_backend", +] diff --git a/evals/drivers/api/anthropic.py b/evals/drivers/api/anthropic.py new file mode 100644 index 0000000..2d7e76f --- /dev/null +++ b/evals/drivers/api/anthropic.py @@ -0,0 +1,171 @@ +"""Anthropic Messages API translation for the provider-neutral eval loop.""" + +from __future__ import annotations + +from typing import Any + +from evals.drivers.api.base import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + Usage, + register_backend, +) + + +def _field(value: Any, name: str, default: Any = None) -> Any: + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def _normalize_usage(usage: Any) -> Usage | None: + if usage is None: + return None + return Usage( + input_tokens=int(_field(usage, "input_tokens", 0) or 0), + output_tokens=int(_field(usage, "output_tokens", 0) or 0), + cache_read_input_tokens=int(_field(usage, "cache_read_input_tokens", 0) or 0), + cache_creation_input_tokens=int(_field(usage, "cache_creation_input_tokens", 0) or 0), + ) + + +def _normalize_stop_reason(value: Any) -> tuple[StopReason, str | None]: + raw = str(value) if value is not None else None + reason = { + "end_turn": StopReason.END_TURN, + "tool_use": StopReason.TOOL_USE, + "max_tokens": StopReason.MAX_TOKENS, + "refusal": StopReason.REFUSAL, + "pause_turn": StopReason.PAUSE_TURN, + "model_context_window_exceeded": StopReason.MODEL_CONTEXT_WINDOW_EXCEEDED, + }.get(raw, StopReason.UNKNOWN) + return reason, raw + + +class AnthropicBackend: + """Stateful adapter over stable ``client.messages.create`` calls.""" + + provider = "anthropic" + # Messages reports input_tokens net of both cache fields, so the three add up. + input_tokens_include_cache = False + + def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: + if client is None: + from anthropic import Anthropic + + client = Anthropic() + self.client = client + self.model = model + self.actual_model = model + self.max_tokens = max_tokens + self.system: str | None = None + self.messages: list[dict[str, Any]] = [] + self.tools: list[dict[str, Any]] = [] + self.started = False + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.system = system + self.messages = [{"role": "user", "content": prompt}] + self.tools = [ + { + "name": tool.name, + "description": tool.description, + "input_schema": tool.input_schema, + } + for tool in tools + ] + self.started = True + + def next_turn(self) -> Turn: + if not self.started: + raise RuntimeError("AnthropicBackend.start() must be called before next_turn()") + request: dict[str, Any] = { + "model": self.model, + "max_tokens": self.max_tokens, + "messages": self.messages, + "tools": self.tools, + # Automatic caching: one top-level field, and the breakpoint advances on its own as + # the conversation grows. Anthropic caching is opt-in where OpenAI's Responses API + # caches unasked, and this loop resends the whole tool surface plus the accumulated + # transcript on every turn -- the measured OpenAI arm reads 88% of its input from + # cache, so without this an Anthropic arm pays full price for the same shape. It also + # made cost differences between the two providers read as pricing rather than as a + # missing field. A cache read is 0.1x input and a 5m write 1.25x, so this pays for + # itself on the second turn of any task. + "cache_control": {"type": "ephemeral"}, + } + if self.system is not None: + request["system"] = self.system + message = self.client.messages.create(**request) + content = _field(message, "content", None) or [] + # Replay the provider's content objects verbatim, including thinking or + # other blocks required by later Messages API turns. + self.messages.append({"role": "assistant", "content": content}) + + text_parts: list[str] = [] + calls: list[ToolCall] = [] + for block in content: + block_type = _field(block, "type") + if block_type == "text": + text = _field(block, "text") + if text: + text_parts.append(str(text)) + elif block_type == "refusal": + explanation = _field(block, "explanation") + if explanation: + text_parts.append(str(explanation)) + elif block_type == "tool_use": + args = _field(block, "input", {}) or {} + if not isinstance(args, dict): + args = {"_raw": args} + calls.append( + ToolCall( + id=str(_field(block, "id", "") or ""), + name=str(_field(block, "name", "") or ""), + args=args, + ) + ) + + response_model = _field(message, "model") + if response_model: + self.actual_model = str(response_model) + stop_reason, provider_stop_reason = _normalize_stop_reason(_field(message, "stop_reason")) + return Turn( + text="\n".join(text_parts), + tool_calls=calls, + usage=_normalize_usage(_field(message, "usage")), + stop_reason=stop_reason, + provider_stop_reason=provider_stop_reason, + ) + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.messages.append( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": result.call_id, + "content": result.text, + "is_error": result.is_error, + } + for result in results + ], + } + ) + + +register_backend( + AnthropicBackend.provider, + AnthropicBackend, + model_aliases={ + "standard": "claude-sonnet-5", + "fast": "claude-haiku-4-5", + }, +) + + +__all__ = ["AnthropicBackend"] diff --git a/evals/drivers/api/base.py b/evals/drivers/api/base.py new file mode 100644 index 0000000..2b9261e --- /dev/null +++ b/evals/drivers/api/base.py @@ -0,0 +1,252 @@ +"""Provider-neutral contracts and registry for API-backed eval loops.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping, Set +from dataclasses import dataclass +from enum import Enum +from typing import Any, Protocol + +from evals.core.results import Usage + +MODEL_TIERS = frozenset({"standard", "fast"}) + + +class UnmappedModelTierError(ValueError): + """Raised when a provider has no verified model for a harness tier.""" + + +class StopReason(str, Enum): + """Harness-owned reasons why a provider turn stopped. + + Values keep the strings the Anthropic path historically emitted so old and new rows + stay comparable; adapters keep the provider's own value on ``Turn``. REFUSAL is + terminal and prevents side effects; UNKNOWN is the explicit fallback for new values. + """ + + END_TURN = "end_turn" + TOOL_USE = "tool_use" + MAX_TOKENS = "max_tokens" + REFUSAL = "refusal" + PAUSE_TURN = "pause_turn" + MODEL_CONTEXT_WINDOW_EXCEEDED = "model_context_window_exceeded" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class ToolSpec: + """A model-facing tool definition translated from MCP ``list_tools``.""" + + name: str + description: str + input_schema: dict[str, Any] + + +@dataclass(frozen=True) +class ToolCall: + """A provider-neutral model request to invoke one MCP tool.""" + + id: str + name: str + args: dict[str, Any] + + +@dataclass(frozen=True) +class ToolResult: + """A provider-neutral MCP result paired to its model call ID.""" + + call_id: str + text: str + is_error: bool = False + kind: str = "text" + + +@dataclass(frozen=True) +class Turn: + """One normalized assistant response from a model provider.""" + + text: str + tool_calls: list[ToolCall] + usage: Usage | None + stop_reason: StopReason + provider_stop_reason: str | None = None + + +class ModelBackend(Protocol): + """Conversation-owning adapter for one model provider. + + Backends retain all provider wire state. The driver sees only normalized + turns and adds normalized tool results after executing MCP calls. + """ + + provider: str + model: str + actual_model: str + client: Any + + #: Whether this provider's ``usage.input_tokens`` already contains cached reads. + #: The two providers disagree, and the driver records the answer on every run so + #: cost analysis never has to infer it from a model name. See + #: ``evals.core.token_accounting``. + input_tokens_include_cache: bool + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: ... + + def next_turn(self) -> Turn: ... + + def add_tool_results(self, results: list[ToolResult]) -> None: ... + + +BackendFactory = Callable[..., ModelBackend] + + +@dataclass(frozen=True) +class BackendRegistration: + """A registered provider factory and the model aliases it owns.""" + + factory: BackendFactory + model_aliases: Mapping[str, str] + + +class BackendRegistry: + """Mutable registry of API providers, populated by backend modules.""" + + def __init__(self) -> None: + self.registrations: dict[str, BackendRegistration] = {} + + @staticmethod + def normalize_name(provider: str) -> str: + name = provider.strip().lower() + if not name: + raise ValueError("API provider name cannot be empty") + return name + + def register( + self, + provider: str, + factory: BackendFactory, + *, + model_aliases: Mapping[str, str] | None = None, + ) -> None: + name = self.normalize_name(provider) + if name in self.registrations: + raise ValueError(f"API provider {name!r} is already registered") + aliases = {str(alias): str(model) for alias, model in (model_aliases or {}).items()} + self.registrations[name] = BackendRegistration(factory=factory, model_aliases=aliases) + + def unregister(self, provider: str) -> None: + """Remove a provider registration, primarily for isolated tests.""" + self.registrations.pop(self.normalize_name(provider), None) + + def names(self) -> frozenset[str]: + return frozenset(self.registrations) + + def resolve(self, provider: str) -> BackendRegistration: + name = self.normalize_name(provider) + try: + return self.registrations[name] + except KeyError as exc: + raise ValueError(f"unknown API provider {name!r}; expected one of {sorted(self.registrations)}") from exc + + def create( + self, + provider: str, + model: str, + *, + max_tokens: int, + client: Any | None = None, + ) -> ModelBackend: + registration = self.resolve(provider) + return registration.factory(model, max_tokens=max_tokens, client=client) + + def resolve_model(self, provider: str, model: str) -> str: + registration = self.resolve(provider) + if model in MODEL_TIERS and model not in registration.model_aliases: + raise UnmappedModelTierError( + f"model tier {model!r} is not mapped for API provider {self.normalize_name(provider)!r}; " + "pass an explicit model ID with --model" + ) + return registration.model_aliases.get(model, model) + + def model_aliases(self, provider: str) -> dict[str, str]: + return dict(self.resolve(provider).model_aliases) + + +class RegisteredProviderNames(Set[str]): + """Live set view over the providers in a ``BackendRegistry``.""" + + def __init__(self, registry: BackendRegistry) -> None: + self.registry = registry + + def __contains__(self, value: object) -> bool: + return value in self.registry.registrations + + def __iter__(self) -> Iterator[str]: + return iter(self.registry.registrations) + + def __len__(self) -> int: + return len(self.registry.registrations) + + +BACKEND_REGISTRY = BackendRegistry() +KNOWN_API_PROVIDERS: Set[str] = RegisteredProviderNames(BACKEND_REGISTRY) + + +def register_backend( + provider: str, + factory: BackendFactory, + *, + model_aliases: Mapping[str, str] | None = None, +) -> None: + """Register a provider factory and any aliases owned by that provider.""" + BACKEND_REGISTRY.register(provider, factory, model_aliases=model_aliases) + + +def unregister_backend(provider: str) -> None: + """Remove a provider registration.""" + BACKEND_REGISTRY.unregister(provider) + + +def create_backend( + provider: str, + model: str, + *, + max_tokens: int, + client: Any | None = None, +) -> ModelBackend: + """Construct the backend registered for ``provider``.""" + return BACKEND_REGISTRY.create(provider, model, max_tokens=max_tokens, client=client) + + +def resolve_backend_model(provider: str, model: str) -> str: + """Resolve only aliases declared by the selected provider.""" + return BACKEND_REGISTRY.resolve_model(provider, model) + + +def backend_model_aliases(provider: str) -> dict[str, str]: + """Return a copy of one provider's owned alias mapping.""" + return BACKEND_REGISTRY.model_aliases(provider) + + +__all__ = [ + "BACKEND_REGISTRY", + "KNOWN_API_PROVIDERS", + "MODEL_TIERS", + "BackendFactory", + "BackendRegistration", + "BackendRegistry", + "ModelBackend", + "RegisteredProviderNames", + "StopReason", + "ToolCall", + "ToolResult", + "ToolSpec", + "Turn", + "UnmappedModelTierError", + "Usage", + "backend_model_aliases", + "create_backend", + "register_backend", + "resolve_backend_model", + "unregister_backend", +] diff --git a/evals/drivers/api/driver.py b/evals/drivers/api/driver.py new file mode 100644 index 0000000..a4dfec1 --- /dev/null +++ b/evals/drivers/api/driver.py @@ -0,0 +1,476 @@ +"""The owned model/tool loop: run one task through a registered API backend. + +Unlike a CLI driver, this driver *is* the loop — it lists the MCP tool surface, asks the +backend for a turn, executes the calls itself and records each result as it comes back. So +it needs no recording proxy and no subprocess supervision: the evidence is in hand. + +Provider differences live behind ``ModelBackend`` in ``backend.py``; nothing here knows +which vendor answered. +""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import sys +import time +from collections.abc import Callable +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +from mcp import ClientSession +from mcp.client.stdio import StdioServerParameters, stdio_client + +from evals.core.error_class import classify_error +from evals.core.evidence import ( + configured_evidence_labels, + normalize_evidence_aggregates, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_aggregate_labels, + observed_aggregates, + observed_sentinel_labels, +) +from evals.core.results import AgentRun, Usage +from evals.core.token_accounting import EXCLUSIVE, INCLUSIVE +from evals.core.token_counting import TOKEN_ESTIMATE_METHOD, estimate_result_tokens +from evals.core.tool_manifest import ToolManifestCapture, tools_page +from evals.drivers.api.base import ( + KNOWN_API_PROVIDERS, + ModelBackend, + StopReason, + ToolResult, + ToolSpec, + create_backend, +) + +DEFAULT_MAX_TOKENS = 8192 + + +def cache_semantics_for(backend: Any) -> str | None: + """Return how this backend's input_tokens treats cache, or None if it never said. + + Defaulting an undeclared backend to either answer is a guess, and a guess written + down as a declaration is worse than no record: it outranks every other signal in + token_accounting. A registered extension that omits the attribute therefore + records nothing, and inference resolves it downstream -- or refuses. + """ + declared = getattr(backend, "input_tokens_include_cache", None) + if declared is None: + return None + return INCLUSIVE if declared else EXCLUSIVE + + +McpSessionFactory = Callable[[StdioServerParameters], Any] + + +def tool_spec_from_mcp(tool: Any) -> ToolSpec: + """Translate an MCP list-tools entry into a neutral tool specification.""" + if isinstance(tool, dict): + name = tool.get("name") or "" + description = tool.get("description") or "" + schema = tool.get("inputSchema") or tool.get("input_schema") or {"type": "object"} + else: + name = getattr(tool, "name", "") or "" + description = getattr(tool, "description", "") or "" + schema = getattr(tool, "inputSchema", None) or getattr(tool, "input_schema", None) + schema = schema or {"type": "object"} + if not isinstance(schema, dict): + schema = {"type": "object"} + return ToolSpec(name=str(name), description=str(description), input_schema=schema) + + +def _dump_content_block(block: Any) -> Any: + if isinstance(block, (dict, str, int, float, bool)) or block is None: + return block + dump = getattr(block, "model_dump", None) + if callable(dump): + return dump(by_alias=True, exclude_none=True) + return str(block) + + +def _content_text_and_kind(content: Any) -> tuple[str, str]: + if content is None: + return "", "text" + if isinstance(content, str): + return content, "text" + if not isinstance(content, list): + return str(content), "text" + + text_parts: list[str] = [] + saw_non_text = False + for block in content: + block_type = block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + if block_type == "text" or block_type is None: + text = block.get("text") if isinstance(block, dict) else getattr(block, "text", None) + if text is None and isinstance(block, str): + text = block + if text is not None: + text_parts.append(str(text)) + else: + saw_non_text = True + + if not saw_non_text: + return "\n".join(text_parts), "text" + payload = json.dumps([_dump_content_block(block) for block in content], default=str, separators=(",", ":")) + return payload, "mixed" if text_parts else "image" + + +def tool_result_from_mcp(call_id: str, raw_result: Any) -> ToolResult: + """Translate an MCP call result, preserving an injected result ID for tests.""" + if isinstance(raw_result, ToolResult): + return raw_result + if isinstance(raw_result, dict): + content = raw_result.get("content") + is_error = bool(raw_result.get("isError") or raw_result.get("is_error")) + else: + content = getattr(raw_result, "content", None) + is_error = bool(getattr(raw_result, "isError", False) or getattr(raw_result, "is_error", False)) + text, kind = _content_text_and_kind(content) + return ToolResult(call_id=call_id, text=text, is_error=is_error, kind=kind) + + +class ApiDriver: + """Run an owned model/tool loop through a registered API backend.""" + + name = "api" + + def __init__( + self, + *, + provider: str = "anthropic", + client: Any | None = None, + backend_factory: Callable[[str, int], ModelBackend] | None = None, + mcp_session_factory: McpSessionFactory | None = None, + server_command: list[str] | None = None, + python_bin: str | None = None, + max_tokens: int = DEFAULT_MAX_TOKENS, + ) -> None: + provider = provider.strip().lower() + if provider not in KNOWN_API_PROVIDERS: + raise ValueError(f"unknown API provider {provider!r}; expected one of {sorted(KNOWN_API_PROVIDERS)}") + if server_command is not None and not server_command: + raise ValueError("server_command cannot be empty") + self.provider = provider + self.client = client + self.backend_factory = backend_factory + self.mcp_session_factory = mcp_session_factory + self.server_command = list(server_command) if server_command is not None else None + self.python_bin = python_bin or sys.executable + self.max_tokens = max_tokens + + def _make_backend(self, model: str) -> ModelBackend: + if self.backend_factory is not None: + return self.backend_factory(model, self.max_tokens) + backend = create_backend( + self.provider, + model, + max_tokens=self.max_tokens, + client=self.client, + ) + # Delay credential-dependent client creation until the first non-skipped + # task, then reuse the provider's connection pool across the battery. + if self.client is None: + self.client = backend.client + return backend + + def _server_params(self, mcp_env: dict[str, str], cwd: Path | None) -> StdioServerParameters: + command = self.server_command or [self.python_bin, "-m", "plane_mcp", "stdio"] + return StdioServerParameters( + command=command[0], + args=command[1:], + env=mcp_env, + cwd=cwd, + ) + + @asynccontextmanager + async def _mcp_session( + self, + params: StdioServerParameters, + *, + manifest_state: dict[str, bool], + ): + if self.mcp_session_factory is not None: + context = self.mcp_session_factory(params) + if inspect.isawaitable(context): + context = await context + if hasattr(context, "__aenter__"): + async with context as session: + yield session + else: + yield context + return + + async with stdio_client(params) as (read, write): + + async def message_handler(message: Any) -> None: + notification = getattr(message, "root", message) + if getattr(notification, "method", None) == "notifications/tools/list_changed": + manifest_state["stale"] = True + + async with ClientSession(read, write, message_handler=message_handler) as session: + yield session + + @staticmethod + async def _list_all_tools(mcp_client: Any) -> tuple[list[Any], str | None]: + """Aggregate every tools/list page and fingerprint the complete snapshot.""" + capture = ToolManifestCapture() + tools: list[Any] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + while True: + page = await mcp_client.list_tools(cursor=cursor) if cursor is not None else await mcp_client.list_tools() + capture.observe_page(page, request_cursor=cursor) + page_tools, next_cursor = tools_page(page) + tools.extend(page_tools) + if next_cursor is None: + return tools, capture.fingerprint + if next_cursor in seen_cursors: + raise RuntimeError(f"tools/list pagination repeated cursor {next_cursor!r}") + seen_cursors.add(next_cursor) + cursor = next_cursor + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, + artifact_dir: Path | None = None, + ) -> AgentRun: + del artifact_dir + if not model: + raise ValueError("the API driver requires a model ID") + if max_turns < 1: + raise ValueError("max_turns must be at least 1") + return asyncio.run( + self._run_task( + prompt=prompt, + mcp_env=mcp_env, + model=model, + max_turns=max_turns, + system=system, + cwd=cwd, + evidence_sentinels=evidence_sentinels, + evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, + ) + ) + + async def _run_task( + self, + *, + prompt: str, + mcp_env: dict[str, str], + model: str, + max_turns: int, + system: str | None, + cwd: Path | None, + evidence_sentinels: dict[str, Any] | None, + evidence_targets: dict[str, Any] | None, + evidence_aggregates: dict[str, Any] | None, + ) -> AgentRun: + backend = self._make_backend(model) + evidence = normalize_evidence_sentinels(evidence_sentinels) + targets = normalize_evidence_targets(evidence_targets) + aggregates = normalize_evidence_aggregates(evidence_aggregates) + evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) + calls: list[dict[str, Any]] = [] + pending_results: list[tuple[int, str]] = [] + usage_per_iteration: list[Usage] = [] + total_input_tokens = 0 + total_output_tokens = 0 + total_cache_read_input_tokens = 0 + total_cache_creation_input_tokens = 0 + result_pair_mismatch = False + hit_max_iterations = False + iterations = 0 + final_text = "" + stop_reason: StopReason | None = None + provider_stop_reason: str | None = None + + params = self._server_params(mcp_env, cwd) + manifest_state = {"stale": False} + tool_manifest_fingerprint: str | None = None + async with self._mcp_session(params, manifest_state=manifest_state) as mcp_client: + await mcp_client.initialize() + raw_tools, tool_manifest_fingerprint = await self._list_all_tools(mcp_client) + backend.start(system, prompt, [tool_spec_from_mcp(tool) for tool in raw_tools]) + + # Match the historical metric: model/tool loop only, after list_tools. + started_at = time.perf_counter() + try: + while iterations < max_turns: + turn = backend.next_turn() + iterations += 1 + final_text = turn.text + stop_reason = turn.stop_reason + provider_stop_reason = turn.provider_stop_reason + if turn.usage is not None: + usage_per_iteration.append(turn.usage) + total_input_tokens += turn.usage.input_tokens + total_output_tokens += turn.usage.output_tokens + total_cache_read_input_tokens += turn.usage.cache_read_input_tokens + total_cache_creation_input_tokens += turn.usage.cache_creation_input_tokens + + call_indices: dict[str, int] = {} + for tool_call in turn.tool_calls: + idx = len(calls) + calls.append( + { + "tool": tool_call.name, + "args": tool_call.args, + "result_tokens": None, + "result_chars": 0, + "result_kind": "text", + "is_error": False, + } + ) + if not tool_call.id or tool_call.id in call_indices: + result_pair_mismatch = True + else: + call_indices[tool_call.id] = idx + + # Record the model's calls, but never execute side effects on + # a refusal-terminated response. + if stop_reason is StopReason.REFUSAL: + break + + if not turn.tool_calls: + if stop_reason is StopReason.PAUSE_TURN and iterations < max_turns: + continue + break + + executed: list[tuple[ToolResult, float]] = [] + for tool_call in turn.tool_calls: + call_started = time.perf_counter() + raw_result = await mcp_client.call_tool(tool_call.name, arguments=tool_call.args) + duration_ms = round((time.perf_counter() - call_started) * 1000, 3) + executed.append((tool_result_from_mcp(tool_call.id, raw_result), duration_ms)) + + matched_ids: set[str] = set() + tool_results: list[ToolResult] = [] + for result, duration_ms in executed: + tool_results.append(result) + idx = call_indices.get(result.call_id) + if idx is None or result.call_id in matched_ids: + result_pair_mismatch = True + continue + matched_ids.add(result.call_id) + calls[idx]["result_chars"] = len(result.text) + calls[idx]["result_kind"] = result.kind + calls[idx]["is_error"] = result.is_error + calls[idx]["duration_ms"] = duration_ms + if result.is_error: + # Same classification the proxy applies to CLI runs, so the + # two driver families produce comparable rows. + calls[idx]["error_class"] = classify_error(result.text) + if evidence_active: + aggregate_observations = observed_aggregates( + result.text, + aggregates, + request_args=calls[idx]["args"], + evidence_targets=targets, + ) + calls[idx]["observed_aggregates"] = aggregate_observations + calls[idx]["observed_sentinels"] = sorted( + set(observed_sentinel_labels(result.text, evidence)) + | set(observed_aggregate_labels(aggregate_observations, aggregates)) + ) + pending_results.append((idx, result.text)) + if matched_ids != set(call_indices) or len(call_indices) != len(turn.tool_calls): + result_pair_mismatch = True + + backend.add_tool_results(tool_results) + if iterations >= max_turns: + hit_max_iterations = stop_reason not in ( + StopReason.END_TURN, + StopReason.MAX_TOKENS, + ) + break + if stop_reason is not StopReason.TOOL_USE: + break + finally: + wall_time_s = time.perf_counter() - started_at + + # Token sizing is intentionally outside wall_time. A backend may offer + # a local/exact counter; absence or failure falls back to recorded text. + counter = getattr(backend, "count_tokens", None) + token_count_failures = 0 + result_tokens_estimated = False + for idx, result_text in pending_results: + counted: int | None = None + count_estimated = False + if callable(counter): + try: + raw_count = counter(result_text) + if inspect.isawaitable(raw_count): + raw_count = await raw_count + counted = int(raw_count) + except Exception: + token_count_failures += 1 + if counted is None: + counted = estimate_result_tokens(len(result_text)) + count_estimated = True + result_tokens_estimated = True + calls[idx]["result_tokens"] = counted + calls[idx]["result_tokens_estimated"] = count_estimated + calls[idx]["result_token_count_method"] = TOKEN_ESTIMATE_METHOD if count_estimated else "backend" + + usage_total = { + "input_tokens": total_input_tokens, + "output_tokens": total_output_tokens, + "cache_read_input_tokens": total_cache_read_input_tokens, + "cache_creation_input_tokens": total_cache_creation_input_tokens, + "source": "iterations", + # Anthropic and OpenAI disagree about whether input_tokens already contains + # the cached reads, and both arrive here as source "iterations". Recording + # which one this was is the difference between pricing a run and guessing + # at it from the model name. + } + semantics = cache_semantics_for(backend) + if semantics is not None: + usage_total["cache_semantics"] = semantics + if manifest_state["stale"]: + tool_manifest_fingerprint = None + return AgentRun( + calls=calls, + final_text=final_text, + usage=usage_per_iteration[-1] if usage_per_iteration else None, + usage_total=usage_total, + usage_scope="iteration", + stopped_reason=(stop_reason or StopReason.UNKNOWN).value, + provider_stop_reason=provider_stop_reason, + call_source="api", + hit_max_turns=hit_max_iterations, + wall_time_s=round(wall_time_s, 3), + usage_per_iteration=usage_per_iteration, + cum_input_tokens=total_input_tokens, + result_pair_mismatch=result_pair_mismatch, + trace_integrity=not result_pair_mismatch, + trace_integrity_reason="result_pair_mismatch" if result_pair_mismatch else None, + tool_manifest_fingerprint=tool_manifest_fingerprint, + token_count_failures=token_count_failures, + result_tokens_estimated=result_tokens_estimated, + evidence_trace_available=evidence_active, + provider=str(backend.provider), + model=str(backend.actual_model), + requested_model=model, + ) + + +__all__ = [ + "DEFAULT_MAX_TOKENS", + "ApiDriver", + "McpSessionFactory", + "tool_result_from_mcp", + "tool_spec_from_mcp", +] diff --git a/evals/drivers/api/openai.py b/evals/drivers/api/openai.py new file mode 100644 index 0000000..4adda09 --- /dev/null +++ b/evals/drivers/api/openai.py @@ -0,0 +1,256 @@ +"""OpenAI Responses translation for the provider-neutral eval loop. + +Responses rather than Chat Completions because Chat Completions cannot run this harness at +all on current models: ``gpt-5.6-luna`` answers a request carrying function tools with +``400 — Function tools with reasoning_effort are not supported for gpt-5.6-luna in +/v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to +'none'``. The harness always sends tools, so every turn failed. + +Setting ``reasoning_effort='none'`` would also have satisfied that error, and was rejected: +it would silence reasoning on this path while the vendor CLI drivers keep it, so an arm run +here could not be compared against a CLI arm of the same model — which is the one question +an API arm exists to answer. +""" + +from __future__ import annotations + +import json +from typing import Any + +from evals.drivers.api.base import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + Usage, + register_backend, +) + + +def _field(value: Any, name: str, default: Any = None) -> Any: + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def _normalize_usage(usage: Any) -> Usage | None: + """Map Responses usage onto the neutral shape. + + Responses names these ``input_tokens``/``output_tokens``, where Chat Completions said + ``prompt_tokens``/``completion_tokens``; both spellings are read so an injected fake or a + future field rename does not silently report zero. + """ + if usage is None: + return None + input_details = _field(usage, "input_tokens_details") or _field(usage, "prompt_tokens_details") + input_tokens = _field(usage, "input_tokens") + if input_tokens is None: + input_tokens = _field(usage, "prompt_tokens", 0) + output_tokens = _field(usage, "output_tokens") + if output_tokens is None: + output_tokens = _field(usage, "completion_tokens", 0) + return Usage( + input_tokens=int(input_tokens or 0), + output_tokens=int(output_tokens or 0), + cache_read_input_tokens=int(_field(input_details, "cached_tokens", 0) or 0), + ) + + +def _text_from_content(content: Any) -> str: + """Concatenate the text parts of one output message's content list.""" + if isinstance(content, str): + return content + parts: list[str] = [] + for part in content or (): + kind = str(_field(part, "type", "") or "") + if kind in ("output_text", "text"): + parts.append(str(_field(part, "text", "") or "")) + elif kind == "refusal": + parts.append(str(_field(part, "refusal", "") or "")) + return "".join(parts) + + +def _parse_arguments(raw_args: Any) -> tuple[dict[str, Any], str]: + """Return (args dict, wire string). Malformed JSON is preserved, never dropped.""" + if isinstance(raw_args, str): + try: + parsed = json.loads(raw_args or "{}") + except json.JSONDecodeError: + return {"_raw": raw_args}, raw_args + if not isinstance(parsed, dict): + return {"_raw": parsed}, raw_args + return parsed, raw_args + if isinstance(raw_args, dict): + return raw_args, json.dumps(raw_args, separators=(",", ":")) + return {"_raw": raw_args}, json.dumps(raw_args, default=str) + + +class OpenAIBackend: + """Stateful adapter over ``client.responses.create``. + + ``openai`` is deliberately imported only when no client was injected, so + importing this module and all offline tests work without that package. + """ + + provider = "openai" + # Responses counts cached reads inside input_tokens; cached_tokens is a subset of it. + input_tokens_include_cache = True + + def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: + if client is None: + try: + from openai import OpenAI + except ImportError as exc: + raise RuntimeError( + "the OpenAI API provider requires the optional 'openai' package; " + "install it with the 'evals-openai' extra" + ) from exc + + client = OpenAI() + self.client = client + self.model = model + self.actual_model = model + self.max_tokens = max_tokens + # Responses calls this the input list; it carries user/assistant items plus + # function_call and function_call_output items, not role-tagged tool messages. + self.input_items: list[Any] = [] + self.instructions: str | None = None + self.tools: list[dict[str, Any]] = [] + self.started = False + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + # The system prompt is a top-level field here rather than a message in the list. + self.instructions = system + self.input_items = [{"role": "user", "content": prompt}] + # Responses declares a function tool flat; Chat Completions nested it under "function". + self.tools = [ + { + "type": "function", + "name": tool.name, + "description": tool.description, + "parameters": tool.input_schema, + } + for tool in tools + ] + self.started = True + + def next_turn(self) -> Turn: + if not self.started: + raise RuntimeError("OpenAIBackend.start() must be called before next_turn()") + request: dict[str, Any] = { + "model": self.model, + "max_output_tokens": self.max_tokens, + "input": self.input_items, + } + if self.instructions is not None: + request["instructions"] = self.instructions + if self.tools: + request["tools"] = self.tools + response = self.client.responses.create(**request) + + output = _field(response, "output", None) or [] + calls: list[ToolCall] = [] + texts: list[str] = [] + refusal_text = "" + for item in output: + kind = str(_field(item, "type", "") or "") + if kind == "function_call": + args, wire_args = _parse_arguments(_field(item, "arguments", "{}")) + # call_id is the identifier a function_call_output must echo back; id is the + # item's own handle. Only call_id closes the loop. + call_id = str(_field(item, "call_id", "") or _field(item, "id", "") or "") + calls.append(ToolCall(id=call_id, name=str(_field(item, "name", "") or ""), args=args)) + # Echo the model's own item back verbatim so the provider sees the history it + # produced, rather than a reconstruction of it. + self.input_items.append(item) + elif kind in ("message", ""): + content = _field(item, "content") + texts.append(_text_from_content(content)) + self.input_items.append(item) + else: + # Reasoning and any future item type: replayed untouched. Dropping a reasoning + # item breaks the chain these models expect on the next request. + self.input_items.append(item) + + for item in output: + if str(_field(item, "type", "") or "") == "message": + for part in _field(item, "content") or (): + if str(_field(part, "type", "") or "") == "refusal": + refusal_text = str(_field(part, "refusal", "") or "") + + response_model = _field(response, "model") + if response_model: + self.actual_model = str(response_model) + + status = str(_field(response, "status", "") or "") + incomplete = _field(response, "incomplete_details") + incomplete_reason = str(_field(incomplete, "reason", "") or "") if incomplete else "" + stop_reason, provider_stop_reason = self._stop_reason( + status=status, + incomplete_reason=incomplete_reason, + has_calls=bool(calls), + refusal=refusal_text, + ) + text = "".join(texts) or refusal_text + if not text: + # output_text is the SDK's own concatenation; only consulted as a fallback so a + # shape this adapter does not model yet still yields the answer. + text = str(_field(response, "output_text", "") or "") + return Turn( + text=text, + tool_calls=calls, + usage=_normalize_usage(_field(response, "usage")), + stop_reason=stop_reason, + provider_stop_reason=provider_stop_reason, + ) + + @staticmethod + def _stop_reason( + *, + status: str, + incomplete_reason: str, + has_calls: bool, + refusal: str, + ) -> tuple[StopReason, str | None]: + """Derive the neutral stop reason. + + Responses has no finish_reason: a turn's outcome is its status plus what it emitted. + Tool calls win over status because a completed response carrying calls is the loop's + continue signal, which is what TOOL_USE means to the driver. + """ + raw = incomplete_reason or status or None + if refusal: + return StopReason.REFUSAL, raw + if has_calls: + return StopReason.TOOL_USE, raw + if incomplete_reason == "max_output_tokens": + return StopReason.MAX_TOKENS, raw + if incomplete_reason == "content_filter": + return StopReason.REFUSAL, raw + if status == "completed": + return StopReason.END_TURN, raw + return StopReason.UNKNOWN, raw + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.input_items.extend( + { + "type": "function_call_output", + "call_id": result.call_id, + "output": result.text, + } + for result in results + ) + + +register_backend( + OpenAIBackend.provider, + OpenAIBackend, + model_aliases={ + "standard": "gpt-5.6-sol", + "fast": "gpt-5.6-luna", + }, +) + + +__all__ = ["OpenAIBackend"] diff --git a/evals/drivers/cli/__init__.py b/evals/drivers/cli/__init__.py new file mode 100644 index 0000000..4447270 --- /dev/null +++ b/evals/drivers/cli/__init__.py @@ -0,0 +1 @@ +"""CLI vendor drivers and their subprocess support.""" diff --git a/evals/drivers/cli/antigravity.py b/evals/drivers/cli/antigravity.py new file mode 100644 index 0000000..e70e204 --- /dev/null +++ b/evals/drivers/cli/antigravity.py @@ -0,0 +1,201 @@ +"""Antigravity CLI (agy) driver — proxy-first measurement.""" + +from __future__ import annotations + +import json +import os +import subprocess +from collections.abc import Callable +from pathlib import Path + +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput + +# Antigravity CLI (agy) — proxy-first +# --------------------------------------------------------------------------- + + +def write_antigravity_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write ``mcpServers`` map JSON (Antigravity / agy mcp_config shape).""" + cfg = { + "mcpServers": { + server_name: { + "command": command, + "args": args, + "env": env, + } + } + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def prepare_antigravity_gemini_dir( + gemini_dir: Path, + *, + command: str, + args: list[str], + env: dict[str, str], +) -> None: + """Build an isolated ``--gemini_dir`` tree for agy holding only our MCP server. + + Writes mcp_config.json to both paths agy reads under its gemini dir (``config/`` + and ``antigravity-cli/``) since which one wins is unsettled; agy creates an empty + ``config/mcp_config.json`` itself when none is present. + + This replaces an isolated HOME. agy keeps its OAuth token in the macOS login + keychain, which Security resolves through ``$HOME/Library/Keychains`` — so + overriding HOME made the keychain unfindable ("A keychain cannot be found to store + \"antigravity\"") and every run failed unauthenticated. ``--gemini_dir`` moves only + agy's own state, leaving HOME real and the credential reachable. + """ + for rel in ( + Path("config") / "mcp_config.json", + Path("antigravity-cli") / "mcp_config.json", + ): + write_antigravity_mcp_config( + gemini_dir / rel, + command=command, + args=args, + env=env, + server_name="plane", + ) + + +class AntigravityCliDriver(CliDriver): + """Run tasks via Google Antigravity CLI (``agy``). + + Probed 2026-08-12: -p headless, --output-format text|json|stream-json, --model, + --dangerously-skip-permissions. No turn-cap flag, so hit_max_turns=False plus a note. + Tool calls come from the proxy sidecar, not from parsing agy stdout. + + MCP config is not a flag, so the config must be planted somewhere agy will read. + Re-probed 2026-08-19 on 1.1.15: the undocumented ``--gemini_dir`` relocates agy's + whole state tree, which isolates the config without touching HOME. It must be an + absolute path — agy logs "must be an absolute path" and silently falls back to the + real one otherwise, which would hand the agent the user's own servers. + + Antigravity CLI has no MCP or effective-config introspection command, so server + exclusivity still cannot be proven by real-binary readback: it rests on the isolated + gemini dir plus inspection of the generated files. + """ + + name = "antigravity-cli" + run_notes = ("no_turn_cap",) + temp_dir_prefix = "plane-eval-antigravity-" + exit_note_prefix = "agy" + include_stderr_in_exit_note = True + + def __init__( + self, + *, + agy_bin: str = "agy", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.agy_bin = agy_bin + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + # Absolute, because agy ignores a relative --gemini_dir and falls back to the + # real one; temp_dir is already absolute but resolve() makes that a guarantee + # rather than a caller's promise. + gemini_dir = (temp_dir / "gemini").resolve() + prepare_antigravity_gemini_dir( + gemini_dir, + command=server_command[0], + args=server_command[1:], + env=child_env, + ) + run_env = None + if "PATH" in child_env: + run_env = {**os.environ, "PATH": child_env["PATH"]} + return CliLaunch( + cwd=task_cwd, + config_args=[f"--gemini_dir={gemini_dir}"], + env=run_env, + ) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del max_turns + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + # Every string flag takes the ``--flag=value`` form. agy parses with Go's flag + # package, where a string flag consumes the next argv entry: written as + # ``-p `` with other flags after it, ``-p`` ate ``--output-format``, the + # real prompt became a stray positional that ended flag parsing, and + # --dangerously-skip-permissions never took effect. agy then answered a question + # about its own CLI and denied its own tool calls. Keeping value and flag in one + # argv entry makes the ordering irrelevant. + command = [ + self.agy_bin, + # --gemini_dir chooses the state tree agy reads everything else out of. + *launch.config_args, + "--output-format=json", + "--dangerously-skip-permissions", + ] + if model: + command.append(f"--model={model}") + # Last, so nothing can be mistaken for its value. + command.append(f"--print={full_prompt}") + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del launch, task_cwd, max_turns + final_text = (proc.stdout or "").strip() + try: + if final_text.lstrip().startswith("{"): + blob = json.loads(final_text) + if isinstance(blob, dict): + final_text = str(blob.get("result") or blob.get("text") or blob.get("response") or final_text) + except json.JSONDecodeError: + pass + + return CliOutput( + final_text=final_text, + stopped_reason="error" if proc.returncode else "end_turn", + ) + + +__all__ = [ + "AntigravityCliDriver", + "prepare_antigravity_gemini_dir", + "write_antigravity_mcp_config", +] diff --git a/evals/drivers/cli/base.py b/evals/drivers/cli/base.py new file mode 100644 index 0000000..fed7c72 --- /dev/null +++ b/evals/drivers/cli/base.py @@ -0,0 +1,387 @@ +"""The subprocess template every CLI vendor driver fills in. + +A CLI driver runs the agent the user already pays for, in a process the harness does not +control. So unlike the API driver it cannot see the tool calls happen: it wraps the MCP +server with the recording proxy (``sidecar``), supervises the process group +(``process``), and reconciles what the proxy captured against what the vendor reported. + +Vendors supply four things — the MCP config file, the command, the output parse, and any +post-reconciliation handling. Everything shared about the run lives here. +""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import time +from abc import ABC, abstractmethod +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from evals import REPO_ROOT +from evals.core.evidence import ( + configured_evidence_labels, + normalize_evidence_aggregates, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_aggregate_labels, + write_evidence_config, +) +from evals.core.results import AgentRun +from evals.drivers.cli.process import note_timeout_kill, run_cli_subprocess +from evals.drivers.cli.sidecar import ( + ProxySidecarResult, + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + proxy_wrap_server_command, +) + + +@dataclass +class CliLaunch: + """Vendor-prepared CLI launch details.""" + + cwd: Path + config_args: list[str] = field(default_factory=list) + env: dict[str, str] | None = None + artifact_dir: Path | None = None + + +@dataclass +class CliOutput: + """Normalized vendor output consumed by the shared ``AgentRun`` assembly.""" + + final_text: str + calls: list[dict[str, Any]] = field(default_factory=list) + client_tool_calls: list[dict[str, Any]] = field(default_factory=list) + usage: dict[str, Any] | None = None + usage_total: dict[str, Any] | None = None + stopped_reason: str = "end_turn" + raw_ref: str | None = None + call_source: str = "json" + hit_max_turns: bool = False + + +class CliOutputError(RuntimeError): + """Signal that vendor output could not produce a valid ``AgentRun``.""" + + +class CliRunError(RuntimeError): + """CLI failure retaining typed sidecar observations for the result row.""" + + def __init__(self, message: str, sidecar: ProxySidecarResult | None = None) -> None: + super().__init__(message) + self.trace_integrity = sidecar.trace_integrity if sidecar is not None else True + self.trace_integrity_reason = sidecar.trace_integrity_reason if sidecar is not None else None + self.tool_manifest_fingerprint = sidecar.tool_manifest_fingerprint if sidecar is not None else None + + +class CliDriver(ABC): + """Template for CLI drivers that run one MCP-backed subprocess task.""" + + name: str + experimental = False + default_call_source = "json" + run_notes: tuple[str, ...] = () + temp_dir_prefix = "plane-eval-cli-" + temp_dir_in_cwd = False + exit_note_prefix: str | None = None + include_stderr_in_exit_note = False + + def __init__( + self, + *, + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.python_bin = python_bin or sys.executable + self._runner = runner or run_cli_subprocess + self.server_command = list(server_command) if server_command else None + self.use_proxy = use_proxy + self.record_result_payloads = record_result_payloads + + def validate_run(self) -> None: + """Reject a launch before any temporary state is created, if needed.""" + return None + + @abstractmethod + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + """Write vendor MCP configuration and return launch settings.""" + + @abstractmethod + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + """Build the vendor CLI command.""" + + def invoke_cli( + self, + command: list[str], + *, + launch: CliLaunch, + timeout_s: int, + ) -> subprocess.CompletedProcess[str]: + """Invoke the configured runner with the shared subprocess contract.""" + kwargs: dict[str, Any] = { + "cwd": str(launch.cwd), + "capture_output": True, + "text": True, + "timeout": timeout_s, + } + if launch.env is not None: + kwargs["env"] = launch.env + return self._runner(command, **kwargs) + + @abstractmethod + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + """Parse vendor output into the normalized CLI result shape.""" + + def finalize_run( + self, + proc: subprocess.CompletedProcess[str], + *, + output: CliOutput, + notes: list[str], + ) -> None: + """Apply vendor handling that must occur after proxy reconciliation.""" + del output + if proc.returncode != 0 and self.exit_note_prefix: + notes.append(f"{self.exit_note_prefix}_exit={proc.returncode}") + stderr = proc.stderr or "" + if self.include_stderr_in_exit_note and stderr.strip(): + notes.append(stderr.strip()[:500]) + + def run_task( + self, + prompt: str, + mcp_env: dict[str, str], + model: str | None, + max_turns: int, + *, + system: str | None = None, + cwd: Path | None = None, + evidence_sentinels: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, + artifact_dir: Path | None = None, + ) -> AgentRun: + """Run one CLI task using the shared configuration/proxy/timeout flow.""" + task_cwd = (cwd or REPO_ROOT).resolve() + notes = list(self.run_notes) + self.validate_run() + temp_parent = str(task_cwd) if self.temp_dir_in_cwd else None + + with ( + tempfile.TemporaryDirectory(prefix=self.temp_dir_prefix, dir=temp_parent) as td, + tempfile.TemporaryDirectory(prefix="plane-eval-evidence-") as evidence_td, + ): + temp_dir = Path(td) + sidecar = temp_dir / "proxy-sidecar.jsonl" + child_env = { + key: value for key, value in mcp_env.items() if key.startswith("PLANE_") or key in ("PATH", "HOME") + } + evidence = normalize_evidence_sentinels(evidence_sentinels) + targets = normalize_evidence_targets(evidence_targets) + aggregates = normalize_evidence_aggregates(evidence_aggregates) + evidence_active = bool(configured_evidence_labels(evidence, targets, aggregates)) + + def verify_aggregate_observations(calls: list[dict[str, Any]]) -> None: + """Turn observed proxy values into labels using harness-held seed truth.""" + for call in calls: + labels = set(call.get("observed_sentinels") or []) + labels.update(observed_aggregate_labels(call.get("observed_aggregates"), aggregates)) + if evidence_active: + call["observed_sentinels"] = sorted(labels) + + real_command = ( + list(self.server_command) if self.server_command else [self.python_bin, "-m", "plane_mcp", "stdio"] + ) + server_command = real_command + if self.use_proxy: + evidence_path = None + if evidence_active: + evidence_path = Path(evidence_td) / "proxy-evidence.json" + write_evidence_config(evidence_path, evidence, targets, aggregates) + server_command = proxy_wrap_server_command( + real_command, + sidecar_path=sidecar, + python_bin=self.python_bin, + record_result_payloads=self.record_result_payloads, + evidence_path=evidence_path, + ) + child_env = ensure_proxy_pythonpath(child_env) + + launch = self.write_mcp_config( + temp_dir, + task_cwd=task_cwd, + server_command=server_command, + child_env=child_env, + ) + launch.artifact_dir = ( + artifact_dir + if artifact_dir is not None + else task_cwd / "evals" / "output" / "driver-artifacts" / self.name + ).resolve() + command = self.build_command( + prompt, + model=model, + max_turns=max_turns, + system=system, + launch=launch, + ) + timeout_s = max(120, max_turns * 60) + # Persisted schema v1 defines wall time as the CLI invocation only. + started_at = time.perf_counter() + + try: + proc = self.invoke_cli(command, launch=launch, timeout_s=timeout_s) + except subprocess.TimeoutExpired as exc: + wall = time.perf_counter() - started_at + notes.append(f"timeout after {timeout_s}s") + note_timeout_kill(notes, exc) + calls: list[dict[str, Any]] = [] + client_calls: list[dict[str, Any]] = [] + call_source = self.default_call_source + trace_integrity = True + trace_integrity_reason = None + tool_manifest_fingerprint = None + if self.use_proxy: + sidecar_result = harvest_proxy_after_cli_timeout( + calls, + client_calls, + sidecar, + notes, + ) + calls, client_calls, call_source = sidecar_result + verify_aggregate_observations(calls) + trace_integrity = sidecar_result.trace_integrity + trace_integrity_reason = sidecar_result.trace_integrity_reason + tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint + evidence_available = False + if evidence_active and call_source == "proxy": + _proxy_calls, status = load_proxy_sidecar(sidecar) + evidence_available = bool( + status.get("state") == "complete" and status.get("evidence_trace_available") + ) + if not evidence_available: + notes.append("proxy_response_evidence_unavailable") + return AgentRun( + calls=calls, + client_tool_calls=client_calls, + final_text="", + usage=None, + stopped_reason="timeout", + raw_ref=None, + usage_scope="run", + call_source=call_source, + hit_max_turns=False, + wall_time_s=round(wall, 3), + evidence_trace_available=evidence_available, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=tool_manifest_fingerprint, + experimental=self.experimental, + notes=notes, + ) + + wall = time.perf_counter() - started_at + try: + output = self.parse_output( + proc, + launch=launch, + task_cwd=task_cwd, + max_turns=max_turns, + notes=notes, + ) + except CliOutputError as exc: + sidecar_result = None + if self.use_proxy: + sidecar_result = apply_proxy_sidecar([], [], sidecar, notes) + detail = "; ".join(notes) + raise CliRunError(f"{exc}: {detail}", sidecar_result) from None + + trace_integrity = True + trace_integrity_reason = None + tool_manifest_fingerprint = None + if self.use_proxy: + sidecar_result = apply_proxy_sidecar( + output.calls, + output.client_tool_calls, + sidecar, + notes, + ) + calls, client_calls, proxy_source = sidecar_result + output.calls = calls + output.client_tool_calls = client_calls + verify_aggregate_observations(output.calls) + trace_integrity = sidecar_result.trace_integrity + trace_integrity_reason = sidecar_result.trace_integrity_reason + tool_manifest_fingerprint = sidecar_result.tool_manifest_fingerprint + if proxy_source == "proxy": + output.call_source = "proxy" + + evidence_available = False + if evidence_active and output.call_source == "proxy": + _proxy_calls, status = load_proxy_sidecar(sidecar) + evidence_available = bool(status.get("state") == "complete" and status.get("evidence_trace_available")) + if not evidence_available: + notes.append("proxy_response_evidence_unavailable") + + self.finalize_run(proc, output=output, notes=notes) + return AgentRun( + calls=output.calls, + client_tool_calls=output.client_tool_calls, + final_text=output.final_text, + usage=output.usage, + usage_total=output.usage_total, + stopped_reason=output.stopped_reason, + raw_ref=output.raw_ref, + usage_scope="run", + call_source=output.call_source, + hit_max_turns=output.hit_max_turns, + wall_time_s=round(wall, 3), + evidence_trace_available=evidence_available, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=tool_manifest_fingerprint, + experimental=self.experimental, + notes=notes, + ) + + +__all__ = [ + "CliDriver", + "CliLaunch", + "CliOutput", + "CliOutputError", + "CliRunError", +] diff --git a/evals/drivers/cli/claude.py b/evals/drivers/cli/claude.py new file mode 100644 index 0000000..8be8261 --- /dev/null +++ b/evals/drivers/cli/claude.py @@ -0,0 +1,532 @@ +"""Claude Code CLI driver and transcript/JSON parsers. + +Probed (claude v2.1.232): -p headless; --mcp-config (repeatable) + --strict-mcp-config; +--output-format json|text|stream-json; --max-turns (present but hidden from --help); +--model; --permission-mode; transcript at +/projects//.jsonl, assistant rows +carrying tool_use blocks; MCP tools appear as mcp____. The CLI's own help +defines --strict-mcp-config as ignoring every MCP source except --mcp-config. The harness +passes the same isolated .claude.json to that option and to real-binary ``claude mcp list`` +readback, which observes only ``plane``. That readback supports the configuration claim; +exclusion during the evaluated ``claude -p`` invocation rests on the documented strict flag, +not behavioral observation of that invocation. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.core.tool_names import normalize_tool_call, split_plane_and_client_calls +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError + + +def normalize_claude_usage(data: dict[str, Any]) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Parse Claude print-mode usage into (raw_usage, usage_total). + + The envelope splits input across ``input_tokens`` (uncached new input only), + ``cache_creation_input_tokens`` and ``cache_read_input_tokens``, mirrored in + ``modelUsage.`` as camelCase plus ``costUSD``/``total_cost_usd``. + Bare ``input_tokens`` is the trap: live multi-turn rows read 8-10 while + cache_read was 180k+, so callers must never copy it into ``cum_input_tokens``. + """ + usage = data.get("usage") + if usage is not None and not isinstance(usage, dict): + usage = None + model_usage = data.get("modelUsage") or data.get("model_usage") + if model_usage is not None and not isinstance(model_usage, dict): + model_usage = None + cost = data.get("total_cost_usd") + if cost is None and isinstance(usage, dict): + cost = usage.get("total_cost_usd") + + if usage is None and model_usage is None and cost is None: + return None, None + + raw = dict(usage or {}) + if cost is not None: + raw["total_cost_usd"] = cost + if model_usage is not None: + raw["modelUsage"] = model_usage + + # Prefer summing modelUsage (per-model run totals) when present + sum_in = sum_out = sum_cr = sum_cc = sum_cost = 0.0 + used_model_usage = False + if model_usage: + for _mid, mu in model_usage.items(): + if not isinstance(mu, dict): + continue + used_model_usage = True + sum_in += float(mu.get("inputTokens") or mu.get("input_tokens") or 0) + sum_out += float(mu.get("outputTokens") or mu.get("output_tokens") or 0) + sum_cr += float(mu.get("cacheReadInputTokens") or mu.get("cache_read_input_tokens") or 0) + sum_cc += float(mu.get("cacheCreationInputTokens") or mu.get("cache_creation_input_tokens") or 0) + sum_cost += float(mu.get("costUSD") or mu.get("cost_usd") or 0) + + if used_model_usage: + uncached_in = int(sum_in) + out_tok = int(sum_out) + cache_read = int(sum_cr) + cache_write = int(sum_cc) + total_cost = float(sum_cost) if sum_cost else cost + else: + uncached_in = int(raw.get("input_tokens") or 0) + out_tok = int(raw.get("output_tokens") or 0) + cache_read = int(raw.get("cache_read_input_tokens") or 0) + cache_write = int(raw.get("cache_creation_input_tokens") or 0) + total_cost = cost + + usage_total: dict[str, Any] = { + "input_tokens": uncached_in, # uncached / new tokens only + "output_tokens": out_tok, + "cache_read_input_tokens": cache_read, + "cache_creation_input_tokens": cache_write, + "total_input_tokens_including_cache": uncached_in + cache_read + cache_write, + "total_cost_usd": total_cost, + "modelUsage": model_usage, + "source": "modelUsage" if used_model_usage else "usage", + } + return raw, usage_total + + +def _claude_project_dir(cwd: Path, *, config_dir: Path | None = None) -> Path: + """Map a cwd to Claude's ``projects/`` transcript directory.""" + munged = str(cwd.resolve()).replace("/", "-") + root = config_dir or Path.home() / ".claude" + return root / "projects" / munged + + +def parse_claude_json_result(payload: dict[str, Any] | str) -> dict[str, Any]: + """Extract final text, usage, session id, num_turns from ``claude -p --output-format json``. + + The print-mode JSON envelope is a single object (``type=result``) with + ``result``, ``session_id``, ``num_turns``, ``total_cost_usd``, ``usage``, + and ``modelUsage``. Per-call tool detail is usually **absent** — callers + should fall back to the session transcript. + """ + if isinstance(payload, str): + payload = json.loads(payload) + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object from claude, got {type(payload)}") + + data = payload + + final = data.get("result") + if final is None: + final = data.get("final_text") or data.get("text") or "" + if not isinstance(final, str): + final = json.dumps(final, default=str) + + usage, usage_total = normalize_claude_usage(data) + + session_id = data.get("session_id") or data.get("sessionId") + num_turns = data.get("num_turns") + if num_turns is None: + num_turns = data.get("numTurns") + is_error = bool(data.get("is_error") or data.get("isError")) + subtype = data.get("subtype") or "" + stop_reason = data.get("stop_reason") or data.get("terminal_reason") or "" + + # Tool calls rarely present in the result envelope; collect if present. + calls: list[dict[str, Any]] = [] + for key in ("tool_calls", "tools", "calls"): + raw = data.get(key) + if isinstance(raw, list): + for item in raw: + if not isinstance(item, dict): + continue + name = item.get("name") or item.get("tool") or "" + args = item.get("input") or item.get("arguments") or item.get("args") or {} + if isinstance(args, str): + try: + args = json.loads(args) + except json.JSONDecodeError: + args = {"_raw": args} + calls.append(normalize_tool_call(str(name), args)) + + # Preserve Claude error subtypes (e.g. error_during_execution, error_max_turns). + # is_error alone collapses to "error" and loses the subtype run.py uses for infra_cli. + if is_error and subtype and str(subtype) not in ("success", ""): + stopped = str(subtype) + elif is_error: + stopped = "error" + else: + stopped = str(stop_reason) if stop_reason else "end_turn" + if subtype and subtype not in ("success", "") and stopped == "end_turn": + stopped = str(subtype) + + plane_calls, client_calls = split_plane_and_client_calls(calls) + + return { + "final_text": final, + "usage": usage, + "usage_total": usage_total, + "session_id": session_id, + "num_turns": int(num_turns) if num_turns is not None else None, + "calls": plane_calls, + "client_tool_calls": client_calls, + "stopped_reason": stopped, + "raw": data, + } + + +def parse_claude_transcript_calls(transcript_path: Path) -> list[dict[str, Any]]: + """Parse ``tool_use`` blocks from a Claude Code session JSONL transcript. + + Returns tagged calls (``origin`` plane|client). Use + ``split_plane_and_client_calls`` before counting. + """ + calls: list[dict[str, Any]] = [] + if not transcript_path.is_file(): + return calls + with transcript_path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + msg = row.get("message") if isinstance(row, dict) else None + if not isinstance(msg, dict): + if row.get("type") == "assistant" and isinstance(row.get("content"), list): + content = row["content"] + else: + continue + else: + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") != "tool_use": + continue + name = str(block.get("name") or "") + args = block.get("input") or {} + if not isinstance(args, dict): + args = {"_raw": args} + calls.append(normalize_tool_call(name, args)) + return calls + + +def find_claude_transcript( + session_id: str | None, + cwd: Path, + *, + config_dir: Path | None = None, +) -> Path | None: + """Locate ``/projects//.jsonl``.""" + if not session_id: + return None + candidate = _claude_project_dir(cwd, config_dir=config_dir) / f"{session_id}.jsonl" + if candidate.is_file(): + return candidate + # Fallback: scan project dir for a file containing the session id + proj = _claude_project_dir(cwd, config_dir=config_dir) + if not proj.is_dir(): + return None + direct = proj / f"{session_id}.jsonl" + if direct.is_file(): + return direct + for p in proj.glob("*.jsonl"): + if session_id in p.name: + return p + return None + + +def write_claude_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write a Claude-compatible mcp-config JSON file.""" + cfg = { + "mcpServers": { + server_name: { + "command": command, + "args": args, + "env": env, + } + } + } + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def persist_claude_transcript( + transcript: Path, + *, + artifact_dir: Path, + session_id: str, +) -> Path: + """Copy a per-task transcript out of disposable Claude state for row forensics.""" + artifact_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + safe_session = "".join(character for character in session_id if character.isalnum() or character in "-_") + destination = artifact_dir / f"{safe_session or 'session'}-{uuid.uuid4().hex}.jsonl" + shutil.copy2(transcript, destination) + destination.chmod(0o600) + return destination + + +def prepare_claude_isolated_environment( + temp_dir: Path, + *, + real_config_dir: Path | None = None, +) -> dict[str, str]: + """Return isolated HOME/config/XDG roots with only Claude's login artifact copied.""" + fake_home = temp_dir / "home" + claude_config = temp_dir / "claude-config" + xdg_roots = { + name: temp_dir / name.lower().replace("_home", "") + for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME") + } + for directory in (fake_home, claude_config, *xdg_roots.values()): + directory.mkdir(parents=True, exist_ok=True) + + source_config = real_config_dir or Path(os.environ.get("CLAUDE_CONFIG_DIR") or Path.home() / ".claude") + source_credentials = source_config / ".credentials.json" + if source_credentials.is_file(): + try: + shutil.copy2(source_credentials, claude_config / ".credentials.json") + except OSError as exc: + raise RuntimeError( + f"failed to copy Claude credentials into isolated config from {source_credentials}: {exc}" + ) from exc + + return { + **os.environ, + "HOME": str(fake_home), + "CLAUDE_CONFIG_DIR": str(claude_config), + **{name: str(directory) for name, directory in xdg_roots.items()}, + } + + +# --------------------------------------------------------------------------- +# Claude CLI driver +# --------------------------------------------------------------------------- + + +class ClaudeCliDriver(CliDriver): + """Run Claude Code with isolated state and readback-supported strict MCP config. + + ``--strict-mcp-config`` makes the launch-scoped file exclusive by the Claude CLI's + documented contract, not a behavioral probe of the evaluated ``claude -p`` invocation. + HOME, CLAUDE_CONFIG_DIR, and every XDG root are isolated so ambient user-home state is + not inherited. A real ``claude mcp list`` reads the same temporary .claude.json and + observes exactly the ``plane`` server. + + Known limitation: a refreshed file-based credential is discarded with the per-task + config. It is not copied into user state because doing so would mutate the user's auth + and introduce cross-task/concurrent refresh races; later tasks re-copy the durable source. + """ + + name = "claude-cli" + run_notes = ( + "known_limitation:claude_file_credentials_refresh_discarded:per-task config is deleted; " + "refresh is not copied to user auth to avoid mutation and cross-task races", + ) + temp_dir_prefix = "plane-eval-claude-" + + def __init__( + self, + *, + claude_bin: str = "claude", + python_bin: str | None = None, + permission_mode: str = "bypassPermissions", + strict_mcp: bool = True, + builtin_tools: str | None = "", + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.claude_bin = claude_bin + self.permission_mode = permission_mode + self.strict_mcp = strict_mcp + # Which of Claude Code's own tools the agent keeps. Empty means none, which is + # what an eval of a *tool surface* wants: with Bash and Read in hand a model that + # cannot work the surface out reads the repo it is standing in, harvests the API + # key and calls Plane's REST API directly — measured, not hypothesised. Removing + # the built-ins also drops the total tool count under the threshold that defers + # MCP tools behind ToolSearch, so the surface arrives directly, as it does for + # every other driver. None keeps Claude Code's default set. + self.builtin_tools = builtin_tools + # Full replacement for the MCP server launch (external surfaces under + # benchmark): [command, *args]. None → this repo's `-m plane_mcp stdio`. + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + run_env = prepare_claude_isolated_environment(temp_dir) + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + transcript_config_dir = Path(run_env["CLAUDE_CONFIG_DIR"]) + # Claude's management readback consumes this location, while the session receives + # the exact same physical file through --mcp-config. + mcp_cfg = transcript_config_dir / ".claude.json" + write_claude_mcp_config( + mcp_cfg, + command=server_command[0], + args=server_command[1:], + env=child_env, + server_name="plane", + ) + return CliLaunch( + cwd=task_cwd, + config_args=["--mcp-config", str(mcp_cfg)], + env=run_env, + ) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + command = [ + self.claude_bin, + "-p", + "--output-format", + "json", + *launch.config_args, + "--permission-mode", + self.permission_mode, + "--max-turns", + str(max_turns), + ] + if self.strict_mcp: + command.append("--strict-mcp-config") + if self.builtin_tools is not None: + # `=` form: --tools is variadic and would otherwise swallow the trailing prompt. + command.append(f"--tools={self.builtin_tools}") + if model: + command.extend(["--model", model]) + if system: + command.extend(["--append-system-prompt", system]) + # --allowedTools is variadic and would swallow the trailing prompt. + command.extend(["--allowedTools=mcp__plane__*", prompt]) + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + stdout = proc.stdout or "" + stderr = proc.stderr or "" + parsed: dict[str, Any] | None = None + parse_err: str | None = None + # JSON may be the whole stdout or the last JSON object line. + for candidate in (stdout.strip(), *(reversed(stdout.strip().splitlines()) if stdout else [])): + if not candidate or not candidate.lstrip().startswith("{"): + continue + try: + parsed = parse_claude_json_result(candidate) + break + except (json.JSONDecodeError, ValueError, TypeError) as exc: + parse_err = str(exc) + + if parsed is None: + notes.append(f"json_parse_failed: {parse_err or 'no JSON object in stdout'}") + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + raise CliOutputError("claude cli failed") + + # Parseable JSON can still be a hard CLI failure (exit 1 + is_error subtype). + if proc.returncode != 0: + notes.append(f"claude_exit={proc.returncode}") + if stderr.strip(): + notes.append(stderr.strip()[:500]) + + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "json" + session_id = parsed.get("session_id") + config_value = (launch.env or {}).get("CLAUDE_CONFIG_DIR") + config_dir = Path(config_value) if config_value else None + transcript = find_claude_transcript( + session_id, + task_cwd, + config_dir=config_dir, + ) + if transcript is not None: + if launch.artifact_dir is None: + raise CliOutputError("Claude transcript found without a durable artifact directory") + try: + transcript = persist_claude_transcript( + transcript, + artifact_dir=launch.artifact_dir, + session_id=str(session_id or transcript.stem), + ) + except OSError as exc: + raise CliOutputError(f"failed to persist Claude transcript: {exc}") from exc + tagged = parse_claude_transcript_calls(transcript) + transcript_plane, transcript_client = split_plane_and_client_calls(tagged) + if transcript_plane or transcript_client: + calls, client_calls = transcript_plane, transcript_client + call_source = "transcript" + notes.append(f"calls_from_transcript:{transcript}") + if not calls and not client_calls: + notes.append("no_tool_calls_in_json_or_transcript") + + num_turns = parsed.get("num_turns") + hit_max = bool(num_turns is not None and int(num_turns) >= max_turns) + stopped = parsed["stopped_reason"] + if hit_max and stopped in ("end_turn", "completed", ""): + stopped = "max_turns" + + raw_ref = str(transcript) if transcript else (f"session:{session_id}" if session_id else None) + return CliOutput( + calls=calls, + final_text=parsed["final_text"], + client_tool_calls=client_calls, + usage=parsed.get("usage"), + usage_total=parsed.get("usage_total"), + stopped_reason=stopped, + raw_ref=raw_ref, + call_source=call_source, + hit_max_turns=hit_max, + ) + + +__all__ = [ + "ClaudeCliDriver", + "find_claude_transcript", + "normalize_claude_usage", + "parse_claude_json_result", + "parse_claude_transcript_calls", + "persist_claude_transcript", + "prepare_claude_isolated_environment", + "write_claude_mcp_config", +] diff --git a/evals/drivers/cli/codex.py b/evals/drivers/cli/codex.py new file mode 100644 index 0000000..5cafdd0 --- /dev/null +++ b/evals/drivers/cli/codex.py @@ -0,0 +1,457 @@ +"""Codex CLI driver and JSONL/rollout parsers. + +Probed: codex exec --json emits JSONL on stdout; -c key=value overrides config.toml +(including mcp_servers); -m selects the model; rollouts at +~/.codex/sessions/**/rollout-*.jsonl carry response_item/function_call payloads. +Experimental — live runs are opt-in because the quota is metered. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.core.tool_names import normalize_tool_call, split_plane_and_client_calls +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput +from evals.drivers.cli.process import run_cli_subprocess + + +def _codex_parse_tool_args(raw_args: Any) -> dict[str, Any]: + if isinstance(raw_args, str): + try: + args = json.loads(raw_args) + except json.JSONDecodeError: + return {"_raw": raw_args} + return args if isinstance(args, dict) else {"_raw": args} + if isinstance(raw_args, dict): + return raw_args + return {"_raw": raw_args} + + +def parse_codex_jsonl_events(lines: list[str] | str) -> dict[str, Any]: + """Parse ``codex exec --json`` stdout (JSONL) for function_call + final text + usage. + + Supports both schemas: + - **v0.147+ streamable**: ``thread.started`` / ``item.completed`` / ``turn.completed`` + (``thread_id`` matches rollout filename suffix). + - **Legacy**: ``session_meta`` / ``response_item`` / ``event_msg`` payloads. + New keys are tried first; legacy handling is retained. + """ + if isinstance(lines, str): + lines = lines.splitlines() + calls: list[dict[str, Any]] = [] + final_parts: list[str] = [] + usage: dict[str, Any] | None = None + stopped = "end_turn" + session_id: str | None = None + + for line in lines: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + rtype = row.get("type") + + # --- New schema (codex exec --json v0.147+): try first --- + if rtype == "thread.started": + session_id = row.get("thread_id") or session_id + continue + if rtype == "turn.started": + continue + if rtype == "item.completed": + item = row.get("item") if isinstance(row.get("item"), dict) else {} + itype = item.get("type") + if itype == "agent_message": + text = item.get("text") + if text: + final_parts.append(str(text)) + elif itype in ( + "function_call", + "tool_call", + "mcp_tool_call", + "command_execution", + "file_change", + ): + # Proxy sidecar is primary for plane calls; harvest best-effort names. + name = str(item.get("name") or item.get("tool") or item.get("command") or itype) + args = item.get("arguments") or item.get("args") or item.get("input") or {} + calls.append(normalize_tool_call(name, _codex_parse_tool_args(args))) + continue + if rtype == "turn.completed": + u = row.get("usage") if isinstance(row.get("usage"), dict) else {} + if u: + usage = { + "input_tokens": u.get("input_tokens", 0) or 0, + "output_tokens": u.get("output_tokens", 0) or 0, + "cache_read_input_tokens": u.get("cached_input_tokens", 0) or 0, + "cache_creation_input_tokens": u.get("cache_write_input_tokens", 0) or 0, + "total_tokens": u.get("total_tokens"), + } + continue + + # --- Legacy schema --- + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + + if rtype == "session_meta": + session_id = payload.get("id") or session_id + continue + + if rtype == "response_item": + pt = payload.get("type") + if pt == "function_call": + name = str(payload.get("name") or "") + calls.append(normalize_tool_call(name, _codex_parse_tool_args(payload.get("arguments") or "{}"))) + elif pt == "message": + # Assistant final-ish content + role = payload.get("role") + content = payload.get("content") + if role == "assistant" and isinstance(content, list): + for c in content: + if isinstance(c, dict) and c.get("type") in ("output_text", "text"): + t = c.get("text") or c.get("output_text") + if t: + final_parts.append(str(t)) + continue + + if rtype == "event_msg": + pt = payload.get("type") + if pt == "agent_message": + msg = payload.get("message") or payload.get("text") + if msg: + final_parts.append(str(msg)) + elif pt == "token_count": + info = payload.get("info") or {} + total = info.get("total_token_usage") or info.get("last_token_usage") or {} + if isinstance(total, dict): + usage = { + "input_tokens": total.get("input_tokens", 0) or 0, + "output_tokens": total.get("output_tokens", 0) or 0, + "cache_read_input_tokens": total.get("cached_input_tokens", 0) or 0, + "cache_creation_input_tokens": total.get("cache_write_input_tokens", 0) or 0, + "total_tokens": total.get("total_tokens"), + } + elif pt == "task_complete": + stopped = "end_turn" + elif pt == "turn_aborted": + stopped = "aborted" + continue + + plane_calls, client_calls = split_plane_and_client_calls(calls) + return { + "calls": plane_calls, + "client_tool_calls": client_calls, + "final_text": "\n".join(final_parts).strip(), + "usage": usage, + "stopped_reason": stopped, + "session_id": session_id, + } + + +def parse_codex_rollout_calls(rollout_path: Path) -> list[dict[str, Any]]: + """Parse function_call records from a Codex session rollout JSONL (plane only).""" + if not rollout_path.is_file(): + return [] + lines = rollout_path.read_text(encoding="utf-8").splitlines() + return parse_codex_jsonl_events(lines)["calls"] + + +def find_codex_rollout(session_id: str | None, *, after_ts: float | None = None) -> Path | None: + """Find the rollout JSONL under ~/.codex/sessions matching *session_id* exactly. + + Matches the filename (which ends with thread_id) or a first-line session_meta id. + Deliberately no newest-after-ts fallback: under parallel runs that picks another + task's rollout and corrupts final_text. Callers note codex_rollout_unmatched on None. + ``after_ts`` is accepted for API compatibility and ignored. + """ + del after_ts # intentionally unused — see docstring + if not session_id: + return None + root = Path.home() / ".codex" / "sessions" + if not root.is_dir(): + return None + sid = str(session_id) + for p in root.rglob("*.jsonl"): + if sid in p.name: + return p + for p in root.rglob("rollout-*.jsonl"): + try: + with p.open(encoding="utf-8") as fh: + first = fh.readline() + row = json.loads(first) + except Exception: + continue + # Legacy session_meta + payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} + if payload.get("id") == sid: + return p + # New thread.started on first line (unusual but possible) + if row.get("type") == "thread.started" and row.get("thread_id") == sid: + return p + if row.get("thread_id") == sid or row.get("session_id") == sid: + return p + return None + + +def write_codex_mcp_override_args( + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> list[str]: + """Build ``codex exec -c ...`` overrides for a stdio MCP server. + + Codex stores MCP under ``[mcp_servers.]`` with ``command``, ``args``, + and ``env`` (see user config.toml). Overrides use dotted ``-c`` paths. + """ + out: list[str] = [ + "-c", + f"mcp_servers.{server_name}.command={json.dumps(command)}", + "-c", + f"mcp_servers.{server_name}.args={json.dumps(args)}", + ] + # env table — pass each key + for k, v in env.items(): + out.extend(["-c", f"mcp_servers.{server_name}.env.{k}={json.dumps(v)}"]) + return out + + +def write_codex_mcp_config( + path: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write the complete MCP config for an isolated Codex home. + + ``approvals_reviewer`` is required, not cosmetic. ``codex exec`` is non-interactive, so an + MCP call that raises an approval request has nobody to answer it and Codex cancels its own + call with ``user cancelled MCP tool call``. The agent then answers from nothing: a live run + recorded zero calls on every task while still emitting confident answers. Routing approvals + through automatic review is what the developer config already does; an isolated home has to + say so itself, because isolation is exactly what stops it being inherited. + """ + lines = [ + 'approvals_reviewer = "auto_review"', + f"[mcp_servers.{json.dumps(server_name)}]", + f"command = {json.dumps(command)}", + f"args = {json.dumps(args)}", + ] + if env: + lines.append(f"[mcp_servers.{json.dumps(server_name)}.env]") + lines.extend(f"{json.dumps(key)} = {json.dumps(value)}" for key, value in env.items()) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def prepare_codex_home( + codex_home: Path, + *, + command: str, + args: list[str], + env: dict[str, str], + real_codex_home: Path | None = None, +) -> None: + """Create an exclusive config root while copying only the CLI login artifact.""" + write_codex_mcp_config( + codex_home / "config.toml", + command=command, + args=args, + env=env, + ) + source_home = real_codex_home or Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") + source_auth = source_home / "auth.json" + if source_auth.is_file(): + try: + shutil.copy2(source_auth, codex_home / "auth.json") + except OSError: + pass + + +# Codex CLI driver (experimental — do not spend live quota from CI) +# --------------------------------------------------------------------------- + + +class CodexCliDriver(CliDriver): + """Run tasks via ``codex exec`` (experimental; metered quota). + + Live invocation is supported for the interface, but the eval harness should + only exercise this driver when the team explicitly opts in. Offline tests + inject a fake runner and never touch the real binary. + """ + + name = "codex-cli" + experimental = True + default_call_source = "stream" + run_notes = ("experimental:codex-cli",) + temp_dir_prefix = "plane-eval-codex-" + exit_note_prefix = "codex" + + def __init__( + self, + *, + codex_bin: str = "codex", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + allow_live: bool = False, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.codex_bin = codex_bin + self.allow_live = allow_live + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def validate_run(self) -> None: + if self._runner is run_cli_subprocess and not self.allow_live: + raise RuntimeError( + "CodexCliDriver refuses live runs by default (metered weekly quota). " + "Pass allow_live=True or inject a fake runner for tests." + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + codex_home = temp_dir / "codex-home" + prepare_codex_home( + codex_home, + command=server_command[0], + args=server_command[1:], + env=child_env, + ) + run_env = {**os.environ, "CODEX_HOME": str(codex_home)} + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + return CliLaunch(cwd=task_cwd, env=run_env) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del max_turns + command = [ + self.codex_bin, + "exec", + "--json", + "--skip-git-repo-check", + *launch.config_args, + ] + if model: + command.extend(["-m", model]) + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + command.append(full_prompt) + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del launch + del task_cwd, max_turns + parsed = parse_codex_jsonl_events(proc.stdout or "") + calls = list(parsed.get("calls") or []) + client_calls = list(parsed.get("client_tool_calls") or []) + call_source = "stream" + session_id = parsed.get("session_id") + + # Exact-match rollout only — never steal another parallel task's file. + need_rollout = (not calls and not client_calls) or not parsed.get("final_text") + if need_rollout and session_id: + rollout = find_codex_rollout(session_id) + if rollout is not None: + full = parse_codex_jsonl_events(rollout.read_text(encoding="utf-8").splitlines()) + if not calls and not client_calls: + calls = list(full.get("calls") or []) + client_calls = list(full.get("client_tool_calls") or []) + if calls or client_calls: + call_source = "transcript" + notes.append(f"calls_from_rollout:{rollout}") + if full.get("final_text") and not parsed.get("final_text"): + parsed["final_text"] = full["final_text"] + notes.append(f"final_text_from_rollout:{rollout}") + if full.get("usage") and not parsed.get("usage"): + parsed["usage"] = full["usage"] + else: + notes.append("codex_rollout_unmatched") + elif need_rollout and not session_id: + notes.append("codex_rollout_unmatched") + + usage = parsed.get("usage") + usage_total = None + if isinstance(usage, dict): + usage_total = { + "input_tokens": usage.get("input_tokens"), + "output_tokens": usage.get("output_tokens"), + "cache_read_input_tokens": usage.get("cache_read_input_tokens"), + "cache_creation_input_tokens": usage.get("cache_creation_input_tokens"), + "total_input_tokens_including_cache": ( + int(usage.get("input_tokens") or 0) + + int(usage.get("cache_read_input_tokens") or 0) + + int(usage.get("cache_creation_input_tokens") or 0) + ), + "source": "codex_token_count", + } + + raw_ref = f"session:{session_id}" if session_id else None + # A nonzero exit means the process failed — authentication, network, a crash — so the + # transcript is not a finished attempt and must not be scored as one. opencode and + # antigravity already say so; codex defaulting to end_turn charged its own process + # failures to the model's success rate and biased every driver comparison against + # the other two. + stopped_reason = parsed.get("stopped_reason") or "end_turn" + if proc.returncode: + notes.append(f"codex_exit={proc.returncode}") + stopped_reason = "error" + return CliOutput( + calls=calls, + final_text=parsed.get("final_text") or "", + client_tool_calls=client_calls, + usage=usage, + usage_total=usage_total, + stopped_reason=stopped_reason, + raw_ref=raw_ref, + call_source=call_source, + hit_max_turns=False, # codex exec has no max-turns flag in --help + ) + + +__all__ = [ + "CodexCliDriver", + "find_codex_rollout", + "parse_codex_jsonl_events", + "parse_codex_rollout_calls", + "prepare_codex_home", + "write_codex_mcp_config", + "write_codex_mcp_override_args", +] diff --git a/evals/drivers/cli/opencode.py b/evals/drivers/cli/opencode.py new file mode 100644 index 0000000..af60f3f --- /dev/null +++ b/evals/drivers/cli/opencode.py @@ -0,0 +1,189 @@ +"""OpenCode CLI driver — proxy-first measurement.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from collections.abc import Callable +from pathlib import Path + +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput + +# OpenCode CLI — proxy-first +# --------------------------------------------------------------------------- + + +def write_opencode_mcp_config( + path: Path, + *, + command: list[str], + env: dict[str, str], + server_name: str = "plane", +) -> None: + """Write project ``opencode.json`` with a local MCP server entry. + + Schema (opencode.ai docs / probed binary strings, 2026-08-12):: + + {"mcp": {"plane": {"type": "local", "command": [...], "environment": {...}}}} + """ + cfg = { + "$schema": "https://opencode.ai/config.json", + "mcp": { + server_name: { + "type": "local", + "command": list(command), + "environment": env, + "enabled": True, + } + }, + } + path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def prepare_opencode_isolated_environment(temp_dir: Path) -> dict[str, str]: + """Return an environment whose HOME/XDG roots cannot load user MCP config.""" + fake_home = temp_dir / "home" + xdg_config = temp_dir / "xdg-config" + xdg_data = temp_dir / "xdg-data" + xdg_cache = temp_dir / "xdg-cache" + xdg_state = temp_dir / "xdg-state" + for directory in (fake_home, xdg_config, xdg_data, xdg_cache, xdg_state): + directory.mkdir(parents=True, exist_ok=True) + + real_data_root = Path(os.environ.get("XDG_DATA_HOME") or Path.home() / ".local" / "share") + source_auth = real_data_root / "opencode" / "auth.json" + if source_auth.is_file(): + destination = xdg_data / "opencode" / "auth.json" + destination.parent.mkdir(parents=True, exist_ok=True) + try: + shutil.copy2(source_auth, destination) + except OSError: + pass + + return { + **os.environ, + "HOME": str(fake_home), + "XDG_CONFIG_HOME": str(xdg_config), + "XDG_DATA_HOME": str(xdg_data), + "XDG_CACHE_HOME": str(xdg_cache), + "XDG_STATE_HOME": str(xdg_state), + } + + +class OpencodeCliDriver(CliDriver): + """Run tasks via ``opencode run`` (proxy-first call recording). + + Probed 2026-08-12: ``opencode run [message..]`` non-interactive, --format json|default, + -m/--model. MCP comes from an ``opencode.json`` mcp section written into the task cwd; + no turn-cap flag, so hit_max_turns=False plus a ``no_turn_cap`` note. + """ + + name = "opencode-cli" + run_notes = ("no_turn_cap",) + temp_dir_prefix = "plane-eval-opencode-" + temp_dir_in_cwd = True + exit_note_prefix = "opencode" + include_stderr_in_exit_note = True + + def __init__( + self, + *, + opencode_bin: str = "opencode", + python_bin: str | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] | None = None, + server_command: list[str] | None = None, + use_proxy: bool = True, + record_result_payloads: bool = False, + ) -> None: + self.opencode_bin = opencode_bin + super().__init__( + python_bin=python_bin, + runner=runner, + server_command=server_command, + use_proxy=use_proxy, + record_result_payloads=record_result_payloads, + ) + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del task_cwd + # Project-local config avoids polluting the user's global config. + write_opencode_mcp_config( + temp_dir / "opencode.json", + command=server_command, + env=child_env, + server_name="plane", + ) + run_env = prepare_opencode_isolated_environment(temp_dir) + if "PATH" in child_env: + run_env["PATH"] = child_env["PATH"] + return CliLaunch(cwd=temp_dir, env=run_env) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del max_turns, launch + full_prompt = prompt if not system else f"{system}\n\n{prompt}" + command = [self.opencode_bin, "run", "--format", "json"] + if model: + command.extend(["-m", model]) + command.append(full_prompt) + return command + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del launch, task_cwd, max_turns + final_text = (proc.stdout or "").strip() + # JSONL events: concatenate text-ish fields best-effort. + if final_text and "\n" in final_text: + parts: list[str] = [] + for line in final_text.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(row, dict): + for key in ("text", "message", "part", "delta"): + value = row.get(key) + if isinstance(value, str) and value.strip(): + parts.append(value) + if row.get("type") in ("text", "message") and isinstance(row.get("content"), str): + parts.append(row["content"]) + if parts: + final_text = "\n".join(parts) + + return CliOutput( + final_text=final_text, + stopped_reason="error" if proc.returncode else "end_turn", + ) + + +__all__ = [ + "OpencodeCliDriver", + "prepare_opencode_isolated_environment", + "write_opencode_mcp_config", +] diff --git a/evals/drivers/cli/process.py b/evals/drivers/cli/process.py new file mode 100644 index 0000000..4299550 --- /dev/null +++ b/evals/drivers/cli/process.py @@ -0,0 +1,124 @@ +"""CLI subprocess lifecycle: process-group launch, timeout kill, bounded reap.""" + +from __future__ import annotations + +import os +import signal +import subprocess +from typing import Any + +# Bounded drain after process-group kill so communicate() never hangs forever +# when a grandchild still holds the pipe open. +_CLI_TIMEOUT_DRAIN_S = 2.0 + + +def kill_process_group(proc: subprocess.Popen[Any]) -> bool: + """SIGKILL the process group whose leader is ``proc``. + + With start_new_session, pgid == proc.pid even after the leader is reaped, so killpg on + that pid directly — killing only the leader leaves grandchildren alive. Returns True if + the signal was delivered, False if the group was already gone. + """ + if proc.pid is None: + return False + try: + # Do NOT use getpgid: if the leader is already reaped, getpgid fails and + # a proc.kill() fallback would recreate the original orphan bug. + os.killpg(proc.pid, signal.SIGKILL) + return True + except ProcessLookupError: + # No process left in the group — fully gone. + return False + + +def _decode_pipe(data: str | bytes | None, *, text: bool) -> str | bytes | None: + if data is None or not text or isinstance(data, str): + return data + return data.decode("utf-8", errors="replace") + + +def _close_pipes_and_reap(proc: subprocess.Popen[Any], *, drain_s: float = _CLI_TIMEOUT_DRAIN_S) -> tuple[Any, Any]: + """Bounded drain / close after a group kill. Never hangs unbounded.""" + try: + return proc.communicate(timeout=drain_s) + except (subprocess.TimeoutExpired, ValueError, OSError): + for stream in (proc.stdout, proc.stderr): + if stream is not None: + try: + stream.close() + except Exception: + pass + try: + proc.wait(timeout=1.0) + except subprocess.TimeoutExpired: + pass + return None, None + + +def run_cli_subprocess( + cmd: list[str], + *, + timeout: float | None = None, + cwd: str | None = None, + env: dict[str, str] | None = None, + capture_output: bool = True, + text: bool = True, + **_kwargs: Any, +) -> subprocess.CompletedProcess[Any]: + """Run a CLI in its own process group; kill the **whole group** on timeout/interrupt. + + Node wrappers like ``codex`` spawn native grandchildren, and plain subprocess.run kills + only the parent — the grandchild holds stdout open and communicate() hangs forever. So: + start_new_session, killpg on any exception, then a bounded second communicate to drain. + Raises TimeoutExpired with ``killed_process_group=True`` only when the signal landed. + """ + popen_kwargs: dict[str, Any] = { + "cwd": cwd, + "start_new_session": True, + "stdout": subprocess.PIPE if capture_output else None, + "stderr": subprocess.PIPE if capture_output else None, + "text": text, + } + if env is not None: + popen_kwargs["env"] = env + + proc = subprocess.Popen(cmd, **popen_kwargs) # noqa: S603 — eval harness launches user CLIs + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + killed = kill_process_group(proc) + out, err = _close_pipes_and_reap(proc) + if out is None and err is None: + stdout = _decode_pipe(exc.stdout, text=text) or ("" if text else b"") + stderr = _decode_pipe(exc.stderr, text=text) or ("" if text else b"") + else: + stdout, stderr = out, err + te = subprocess.TimeoutExpired( + cmd=cmd, + timeout=timeout if timeout is not None else 0, + output=stdout, + stderr=stderr, + ) + te.killed_process_group = killed # type: ignore[attr-defined] + raise te from None + except BaseException: + # KeyboardInterrupt / SystemExit / etc. — do not leave the CLI tree running. + # start_new_session means SIGINT no longer reaches the group automatically. + kill_process_group(proc) + _close_pipes_and_reap(proc) + raise + + return subprocess.CompletedProcess(cmd, proc.returncode if proc.returncode is not None else 0, stdout, stderr) + + +def note_timeout_kill(notes: list[str], exc: BaseException) -> None: + """Append process-group kill note when killpg actually delivered the signal.""" + if getattr(exc, "killed_process_group", False): + notes.append("timeout_killed_process_group") + + +__all__ = [ + "kill_process_group", + "note_timeout_kill", + "run_cli_subprocess", +] diff --git a/evals/drivers/cli/sidecar.py b/evals/drivers/cli/sidecar.py new file mode 100644 index 0000000..3807abb --- /dev/null +++ b/evals/drivers/cli/sidecar.py @@ -0,0 +1,630 @@ +"""Recording-proxy glue: wrap commands, PYTHONPATH, load/harvest sidecar JSONL.""" + +from __future__ import annotations + +import json +import os +import sys +import time +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from evals import REPO_ROOT +from evals.core.results import TraceIntegrityReason + +ProxyMetaWaitOutcome = Literal["meta_present", "proxy_exited", "proxy_not_observed", "timeout"] + + +@dataclass(slots=True) +class ProxySidecarResult: + """Harvested calls plus typed integrity and manifest observations.""" + + calls: list[dict[str, Any]] + client_calls: list[dict[str, Any]] + call_source: str + trace_integrity: bool + trace_integrity_reason: TraceIntegrityReason | None + tool_manifest_fingerprint: str | None + status: dict[str, Any] + + def __iter__(self) -> Iterator[Any]: + """Retain the established three-value unpacking API.""" + yield self.calls + yield self.client_calls + yield self.call_source + + +def _pumps_blocking(meta: dict[str, Any] | None) -> bool: + """Whether a still-running pump means the recording may be short. + + Not every live pump is evidence of loss. A CLI that signals its MCP servers on exit + leaves the proxy's stdin read parked on a client that will never write again — the + trace is complete with respect to everything that client actually sent, and an + in-flight request that never got its answer is already counted as an unmatched + response. What does imply loss is a live *output* pump, because the server may have + been mid-reply when the deadline expired. + + Sidecars written before the proxy recorded per-stream detail carry only the boolean, + so they keep the old, stricter reading rather than being reinterpreted after the fact. + """ + if meta is None: + return False + if not meta.get("pumps_alive"): + return False + streams = meta.get("pumps_alive_streams") + if not isinstance(streams, list): + return True + if {"stdout", "stderr"} & {str(name) for name in streams}: + return True + return str(meta.get("finalization_reason") or "") not in ("signal", "child_exit") + + +def _nonnegative_int(value: Any) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def proxy_wrap_server_command( + real_command: list[str], + *, + sidecar_path: Path, + python_bin: str | None = None, + record_result_payloads: bool = False, + evidence_path: Path | None = None, +) -> list[str]: + """Return ``[python, -m, evals.proxy, --log, sidecar, --, *real_command]``.""" + py = python_bin or sys.executable + command = [py, "-m", "evals.proxy", "--log", str(sidecar_path)] + if record_result_payloads: + command.append("--record-result-payloads") + if evidence_path is not None: + command.extend(["--evidence-file", str(evidence_path)]) + return [*command, "--", *real_command] + + +def ensure_proxy_pythonpath(env: dict[str, str]) -> dict[str, str]: + """Inject the repo root into PYTHONPATH so ``python -m evals.proxy`` works from any cwd. + + ``evals`` is not an installed package (pyproject excludes it); the MCP child + is often launched from a foreign temp dir (OpenCode project dir, etc.). + """ + root = str(REPO_ROOT) + out = dict(env) + existing = out.get("PYTHONPATH", "") + parts = [p for p in existing.split(os.pathsep) if p] + if root not in parts: + out["PYTHONPATH"] = root + (os.pathsep + existing if existing else "") + return out + + +def proxy_session_paths(path: Path) -> list[Path]: + """Discover the legacy base file and every per-process session derived from it.""" + discovered = list(path.parent.glob(f"{path.name}.*.jsonl")) + if path.is_file(): + discovered.append(path) + return sorted(set(discovered), key=lambda item: item.name) + + +def load_proxy_sidecar(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Load, validate, and merge exactly one proxy session from each sidecar file.""" + status: dict[str, Any] = { + "state": "missing", + "torn_line": False, + "skipped_rows": 0, + "meta": None, + "metas": [], + "sessions": [], + "session_count": 0, + "session_file_count": 0, + "session_files": [], + "all_sessions_finalized": False, + "unfinalized_sessions": 0, + "pending_left": None, + "non_tool_pending_left": None, + "proxy_meta_count": 0, + "proxy_meta_not_final": False, + "invalid_seq": 0, + "duplicate_seq": 0, + "missing_seq": 0, + "unexpected_seq": 0, + "invalid_meta_fields": 0, + "tool_manifest_disagreement": False, + "tool_manifest_fingerprints": [], + "tool_manifest_missing_sessions": 0, + } + session_paths = proxy_session_paths(path) + if not session_paths: + return [], status + status["session_file_count"] = len(session_paths) + status["session_files"] = [str(session_path) for session_path in session_paths] + + raw_sessions: list[dict[str, Any]] = [] + for session_path in session_paths: + raw_session: dict[str, Any] = { + "path": session_path, + "calls": [], + "metas": [], + "torn_line": False, + "skipped_rows": 0, + "proxy_meta_not_final": False, + } + try: + raw = session_path.read_bytes() + except OSError: + raw_sessions.append(raw_session) + continue + lines = raw.decode("utf-8", errors="replace").splitlines() + last_row_kind: str | None = None + for index, line in enumerate(lines): + text = line.strip() + if not text: + continue + try: + row = json.loads(text) + except json.JSONDecodeError: + if index == len(lines) - 1: + raw_session["torn_line"] = True + raw_session["proxy_meta_not_final"] = bool(raw_session["metas"]) + break + raw_session["skipped_rows"] += 1 + last_row_kind = "invalid" + continue + if not isinstance(row, dict): + raw_session["skipped_rows"] += 1 + last_row_kind = "invalid" + continue + if row.get("row_type") == "proxy_meta": + raw_session["metas"].append(row) + last_row_kind = "meta" + continue + tool = row.get("tool") + if not tool: + raw_session["skipped_rows"] += 1 + last_row_kind = "invalid" + continue + call = { + "tool": str(tool), + "args": row.get("args") if isinstance(row.get("args"), dict) else (row.get("args") or {}), + "origin": "plane", + "is_error": bool(row.get("is_error")), + "result_chars": int(row.get("result_chars") or 0), + "duration_ms": row.get("duration_ms"), + "seq": row.get("seq"), + } + if isinstance(row.get("error_class"), str): + # The proxy classifies while the payload still exists; this reader is the + # only path from the sidecar to a result row, so a key absent here is a + # key that never reaches the report. + call["error_class"] = row["error_class"] + if isinstance(row.get("result_text"), str): + call["result_text"] = row["result_text"] + if isinstance(row.get("observed_sentinels"), list): + call["observed_sentinels"] = [str(value) for value in row["observed_sentinels"]] + if isinstance(row.get("observed_aggregates"), list): + call["observed_aggregates"] = [value for value in row["observed_aggregates"] if isinstance(value, dict)] + raw_session["calls"].append(call) + last_row_kind = "call" + raw_session["proxy_meta_not_final"] = raw_session["proxy_meta_not_final"] or bool( + raw_session["metas"] and last_row_kind != "meta" + ) + raw_sessions.append(raw_session) + + counter_keys = ( + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "non_json_lines", + "malformed_jsonrpc", + "recorder_errors", + "undelivered_lines", + ) + fatal_counts = ( + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "recorder_errors", + "undelivered_lines", + "invalid_seq", + "duplicate_seq", + "missing_seq", + "unexpected_seq", + "invalid_meta_fields", + ) + calls: list[dict[str, Any]] = [] + session_statuses: list[dict[str, Any]] = [] + manifests: list[str] = [] + for index, raw_session in enumerate(raw_sessions): + session_calls = raw_session["calls"] + meta_rows = raw_session["metas"] + meta = meta_rows[-1] if meta_rows else None + valid_sequences = [call["seq"] for call in session_calls if _nonnegative_int(call.get("seq")) not in (None, 0)] + last_seq = _nonnegative_int(meta.get("last_seq")) if meta is not None else None + tool_request_count = _nonnegative_int(meta.get("tool_request_count")) if meta is not None else None + segment: dict[str, Any] = { + "index": index, + "path": str(raw_session["path"]), + "meta": meta, + "meta_count": len(meta_rows), + "call_count": len(session_calls), + "torn_line": bool(raw_session["torn_line"]), + "skipped_rows": int(raw_session["skipped_rows"]), + "proxy_meta_not_final": bool(raw_session["proxy_meta_not_final"]), + "finalized": len(meta_rows) == 1 and not raw_session["proxy_meta_not_final"], + "invalid_seq": len(session_calls) - len(valid_sequences), + "duplicate_seq": len(valid_sequences) - len(set(valid_sequences)), + "missing_seq": 0, + "unexpected_seq": 0, + "invalid_meta_fields": 0, + "pumps_alive": bool(meta.get("pumps_alive")) if meta is not None else False, + "pumps_blocking": _pumps_blocking(meta), + "last_seq": last_seq, + "tool_request_count": tool_request_count, + } + if meta is not None: + segment["invalid_meta_fields"] = ( + abs(len(meta_rows) - 1) + int(last_seq is None) + int(tool_request_count is None) + ) + if last_seq is not None and tool_request_count is not None and tool_request_count != last_seq: + segment["invalid_meta_fields"] += 1 + expected_sequences = set(range(1, last_seq + 1)) if last_seq is not None else set() + observed_sequences = set(valid_sequences) + segment["missing_seq"] = len(expected_sequences - observed_sequences) + segment["unexpected_seq"] = len(observed_sequences - expected_sequences) if last_seq is not None else 0 + for key in counter_keys: + value = _nonnegative_int(meta.get(key)) + if meta.get(key) is not None and value is None: + segment["invalid_meta_fields"] += 1 + segment[key] = value + fingerprint = meta.get("tool_manifest_fingerprint") + if isinstance(fingerprint, str): + manifests.append(fingerprint) + else: + for key in counter_keys: + segment[key] = None + + segment["state"] = ( + "incomplete" + if meta is None + or not segment["finalized"] + or segment["torn_line"] + or segment["skipped_rows"] > 0 + or any((segment.get(key) or 0) > 0 for key in fatal_counts) + or segment["pumps_blocking"] + else "complete" + ) + session_calls.sort( + key=lambda call: ( + _nonnegative_int(call.get("seq")) in (None, 0), + _nonnegative_int(call.get("seq")) or 0, + ) + ) + calls.extend(session_calls) + session_statuses.append(segment) + + metas = [segment["meta"] for segment in session_statuses if segment["meta"] is not None] + unfinalized_sessions = sum(not segment["finalized"] for segment in session_statuses) + status["sessions"] = session_statuses + status["session_count"] = len(session_statuses) + status["metas"] = metas + status["meta"] = metas[-1] if metas else None + status["proxy_meta_count"] = sum(segment["meta_count"] for segment in session_statuses) + status["unfinalized_sessions"] = unfinalized_sessions + status["all_sessions_finalized"] = bool(session_statuses) and unfinalized_sessions == 0 + status["proxy_meta_not_final"] = any(segment["proxy_meta_not_final"] for segment in session_statuses) + status["torn_line"] = any(segment["torn_line"] for segment in session_statuses) + status["pumps_alive"] = any(segment["pumps_alive"] for segment in session_statuses) + status["pumps_blocking"] = any(segment["pumps_blocking"] for segment in session_statuses) + aggregate_keys = ( + "skipped_rows", + *counter_keys, + "invalid_seq", + "duplicate_seq", + "missing_seq", + "unexpected_seq", + "invalid_meta_fields", + ) + for key in aggregate_keys: + status[key] = sum((segment.get(key) or 0) for segment in session_statuses) + + unique_manifests = sorted(set(manifests)) + missing_manifests = len(metas) - len(manifests) + status["tool_manifest_fingerprints"] = unique_manifests + status["tool_manifest_missing_sessions"] = missing_manifests + # A session that never called tools/list has no opinion about the manifest, and silence + # is not contradiction. Claude Code splits the work — one session lists the tools, a + # second makes the calls and never lists — so counting the quiet one as a dissenter + # discarded the fingerprint on almost every row and left the reporter unable to + # establish that a file's rows hit the same surface. Real disagreement is two sessions + # reporting different fingerprints, which len(unique) > 1 already catches. + status["tool_manifest_disagreement"] = len(unique_manifests) > 1 + status["tool_manifest_fingerprint"] = unique_manifests[0] if len(unique_manifests) == 1 else None + status["evidence_trace_available"] = ( + bool(metas) + and len(metas) == len(session_statuses) + and all(bool(meta.get("evidence_trace_available")) for meta in metas) + ) + if status["meta"] is not None: + status["finalization_reason"] = status["meta"].get("finalization_reason") + status["finalization_signal"] = status["meta"].get("finalization_signal") + if all(segment["call_count"] == 0 and segment["meta_count"] == 0 for segment in session_statuses) and not ( + status["torn_line"] or status["skipped_rows"] + ): + status["state"] = "empty" + else: + status["state"] = ( + "incomplete" if any(segment["state"] == "incomplete" for segment in session_statuses) else "complete" + ) + return calls, status + + +def trace_integrity_from_status( + status: dict[str, Any], +) -> tuple[bool, TraceIntegrityReason | None]: + """Map sidecar status to the typed result-row integrity fields.""" + if status.get("state") == "complete": + return True, None + if (status.get("unparsed_lines") or 0) > 0: + return False, "protocol_violation" + return False, "recorder_loss" + + +def _incompleteness_note(status: dict[str, Any]) -> str: + parts = ["proxy_sidecar_incomplete"] + if status.get("torn_line"): + parts.append("torn_line=1") + if status.get("meta") is None: + parts.append("no_meta=1") + else: + if status.get("proxy_meta_not_final"): + parts.append("proxy_meta_not_final=1") + if status.get("unfinalized_sessions"): + parts.append(f"unfinalized_sessions={int(status['unfinalized_sessions'])}") + for key in ( + "skipped_rows", + "pending_left", + "non_tool_pending_left", + "unmatched_responses", + "unparsed_lines", + "non_json_lines", + "malformed_jsonrpc", + "recorder_errors", + "undelivered_lines", + "invalid_seq", + "duplicate_seq", + "missing_seq", + "unexpected_seq", + "invalid_meta_fields", + ): + value = status.get(key) + if value and not (key == "proxy_meta_count" and value == 1): + parts.append(f"{key}={int(value)}") + if status.get("pumps_blocking"): + parts.append("pumps_alive=1") + return ":".join(parts) + + +def load_proxy_sidecar_calls(path: Path) -> list[dict[str, Any]]: + """Convenience: call rows only (sorted by seq).""" + calls, _status = load_proxy_sidecar(path) + return calls + + +def proxy_pid_path(sidecar_path: Path) -> Path: + """Return the companion lifecycle file written by the recording proxy.""" + return sidecar_path.with_name(f"{sidecar_path.name}.pid") + + +def _read_proxy_pid(sidecar_path: Path) -> int | None: + pids = _read_proxy_pids(sidecar_path) + return pids[-1] if pids else None + + +def _read_proxy_pids(sidecar_path: Path) -> list[int]: + lifecycle_paths = [proxy_pid_path(path) for path in proxy_session_paths(sidecar_path)] + lifecycle_paths.extend(sidecar_path.parent.glob(f"{sidecar_path.name}.*.jsonl.pid")) + lifecycle_paths.append(proxy_pid_path(sidecar_path)) + pids: set[int] = set() + for lifecycle_path in set(lifecycle_paths): + try: + pid = int(lifecycle_path.read_text(encoding="ascii").strip()) + except (OSError, UnicodeError, ValueError): + continue + if pid > 0: + pids.add(pid) + return sorted(pids) + + +def _process_is_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def _wait_for_proxy_meta_outcome( + sidecar_path: Path, + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, +) -> ProxyMetaWaitOutcome: + """Wait for final metadata and retain why an absent row cannot still arrive.""" + # Local import keeps drivers import-light for non-proxy unit tests. + from evals.proxy import SHUTDOWN_DEADLINE_S + + if max_wait_s is None: + max_wait_s = SHUTDOWN_DEADLINE_S + 2.0 + + # A completed CLI cannot launch a proxy after the fact. No trace and no + # lifecycle file therefore means the proxy was never observed, rather than + # a finalizer that could benefit from waiting for the whole deadline. + pids = _read_proxy_pids(sidecar_path) + if not proxy_session_paths(sidecar_path) and not pids: + return "proxy_not_observed" + + deadline = time.monotonic() + max(0.0, max_wait_s) + while True: + _, status = load_proxy_sidecar(sidecar_path) + if status.get("all_sessions_finalized"): + return "meta_present" + pids = _read_proxy_pids(sidecar_path) + if pids and not any(_process_is_alive(pid) for pid in pids): + return "proxy_exited" + rem = deadline - time.monotonic() + if rem <= 0: + break + time.sleep(min(max(0.001, poll_s), rem)) + + # Close the boundary races in both directions: metadata may have landed on + # the final sleep, or the proxy may have exited without writing it. + _, status = load_proxy_sidecar(sidecar_path) + if status.get("all_sessions_finalized"): + return "meta_present" + pids = _read_proxy_pids(sidecar_path) + if pids and not any(_process_is_alive(pid) for pid in pids): + return "proxy_exited" + return "timeout" + + +def _note_proxy_meta_wait(outcome: ProxyMetaWaitOutcome, sidecar_path: Path, notes: list[str]) -> None: + if outcome == "proxy_exited": + notes.append("proxy_meta_missing_after_proxy_exit") + elif outcome == "proxy_not_observed": + notes.append("proxy_meta_missing:proxy_not_observed") + elif outcome == "timeout": + pid = _read_proxy_pid(sidecar_path) + state = "proxy_alive=1" if pid is not None and _process_is_alive(pid) else "proxy_state=unknown" + notes.append(f"proxy_meta_wait_timeout:{state}") + + +def apply_proxy_sidecar( + calls: list[dict[str, Any]], + client_calls: list[dict[str, Any]], + sidecar_path: Path, + notes: list[str], + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, +) -> ProxySidecarResult: + """Wait for and prefer a complete sidecar; fall back when incomplete/empty. + + Incomplete sidecar (torn/skipped row, missing meta, pending_left>0) yields + to the CLI trace when the CLI has *more* plane calls. Returns + ``(plane_calls, client_calls, call_source)``. + """ + wait_outcome = _wait_for_proxy_meta_outcome( + sidecar_path, + poll_s=poll_s, + max_wait_s=max_wait_s, + ) + _note_proxy_meta_wait(wait_outcome, sidecar_path, notes) + proxy_calls, status = load_proxy_sidecar(sidecar_path) + state = status.get("state") + trace_integrity, trace_integrity_reason = trace_integrity_from_status(status) + fingerprint = status.get("tool_manifest_fingerprint") if trace_integrity else None + if status.get("tool_manifest_disagreement"): + manifest_values = list(status.get("tool_manifest_fingerprints") or []) + if status.get("tool_manifest_missing_sessions"): + manifest_values.append("") + manifests = ",".join(manifest_values) + notes.append(f"proxy_tool_manifest_disagreement:{manifests}") + + def result( + selected_calls: list[dict[str, Any]], + selected_client_calls: list[dict[str, Any]], + source: str, + ) -> ProxySidecarResult: + return ProxySidecarResult( + calls=selected_calls, + client_calls=selected_client_calls, + call_source=source, + trace_integrity=trace_integrity, + trace_integrity_reason=trace_integrity_reason, + tool_manifest_fingerprint=str(fingerprint) if isinstance(fingerprint, str) else None, + status=status, + ) + + if state in ("missing", "empty"): + notes.append("proxy_sidecar_empty") + return result(calls, client_calls, "json") + if state == "incomplete": + notes.append(_incompleteness_note(status)) + if len(calls) > len(proxy_calls): + notes.append("proxy_sidecar_deferred_to_cli_trace") + return result(calls, client_calls, "json") + if proxy_calls: + notes.append(f"calls_from_proxy:{sidecar_path}") + return result(proxy_calls, client_calls, "proxy") + return result(calls, client_calls, "json") + # complete + notes.append(f"calls_from_proxy:{sidecar_path}") + return result(proxy_calls, client_calls, "proxy") + + +def wait_for_proxy_meta( + sidecar_path: Path, + *, + poll_s: float = 0.2, + max_wait_s: float | None = None, +) -> bool: + """Poll until the sidecar gains a ``proxy_meta`` row, returning True if it appears. + + The proxy is a separate process: after the driver kills the CLI it only then sees stdin + EOF and needs up to SHUTDOWN_DEADLINE_S to flush. Call before harvesting so the temp + directory is not deleted mid-finalization. + """ + return ( + _wait_for_proxy_meta_outcome( + sidecar_path, + poll_s=poll_s, + max_wait_s=max_wait_s, + ) + == "meta_present" + ) + + +def harvest_proxy_after_cli_timeout( + calls: list[dict[str, Any]], + client_calls: list[dict[str, Any]], + sidecar_path: Path, + notes: list[str], + *, + max_wait_s: float | None = None, +) -> ProxySidecarResult: + """Wait for proxy finalization after CLI kill, then harvest the sidecar. + + If meta never appears within the wait window, harvest anyway (incomplete + note from ``apply_proxy_sidecar``). ``max_wait_s`` defaults to + ``SHUTDOWN_DEADLINE_S + 2`` (see ``wait_for_proxy_meta``). + """ + return apply_proxy_sidecar( + calls, + client_calls, + sidecar_path, + notes, + max_wait_s=max_wait_s, + ) + + +__all__ = [ + "apply_proxy_sidecar", + "ensure_proxy_pythonpath", + "harvest_proxy_after_cli_timeout", + "load_proxy_sidecar", + "load_proxy_sidecar_calls", + "proxy_pid_path", + "proxy_session_paths", + "proxy_wrap_server_command", + "ProxySidecarResult", + "trace_integrity_from_status", + "wait_for_proxy_meta", +] diff --git a/evals/listing.py b/evals/listing.py new file mode 100644 index 0000000..fa1d249 --- /dev/null +++ b/evals/listing.py @@ -0,0 +1,224 @@ +"""Measure MCP tool listing size: tool count and cl100k tokens. + +``python -m evals.listing [--label local | --server-cmd '' --server-env KEY=VAL]`` +Reports wire tokens (with outputSchema), model-facing tokens (without), and the top-10 +tools by size. Needs EVAL_PLANE_* credentials; tiktoken is a dev optional dependency. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import shlex +import sys +from dataclasses import dataclass +from typing import Any + +from evals.core.server_env import stdio_server_env + + +@dataclass +class ToolTokenRow: + name: str + wire_tokens: int + model_facing_tokens: int + has_output_schema: bool + + +def tool_payload_wire(tool: Any) -> dict[str, Any]: + """Full wire-shaped tool dict for token counting (includes outputSchema when present).""" + name = getattr(tool, "name", None) or (tool.get("name") if isinstance(tool, dict) else "") or "" + description = (getattr(tool, "description", None) if not isinstance(tool, dict) else tool.get("description")) or "" + input_schema = ( + getattr(tool, "inputSchema", None) + if not isinstance(tool, dict) + else (tool.get("inputSchema") or tool.get("input_schema")) + ) or {} + output_schema = ( + getattr(tool, "outputSchema", None) + if not isinstance(tool, dict) + else (tool.get("outputSchema") or tool.get("output_schema")) + ) + d: dict[str, Any] = { + "name": name, + "description": description, + "input_schema": input_schema, + } + if output_schema is not None: + d["output_schema"] = output_schema + return d + + +def tool_payload_model_facing(tool: Any) -> dict[str, Any]: + """Model-facing payload: name + description + input_schema only (no outputSchema).""" + wire = tool_payload_wire(tool) + return { + "name": wire["name"], + "description": wire["description"], + "input_schema": wire["input_schema"], + } + + +def count_tool_tokens( + tools: list[Any], + *, + encode: Any | None = None, +) -> tuple[list[ToolTokenRow], int, int]: + """Count cl100k tokens per tool for wire and model-facing serializations. + + ``encode`` is a callable ``str -> list[int]`` (tiktoken Encoding.encode). When + None, imports tiktoken cl100k_base. Returns (per-tool rows sorted by wire + tokens desc, total_wire, total_model_facing). + """ + if encode is None: + import tiktoken + + enc = tiktoken.get_encoding("cl100k_base") + encode = enc.encode + + rows: list[ToolTokenRow] = [] + total_wire = 0 + total_model = 0 + for t in tools: + wire = tool_payload_wire(t) + model = tool_payload_model_facing(t) + w_tok = len(encode(json.dumps(wire, separators=(",", ":"), ensure_ascii=False))) + m_tok = len(encode(json.dumps(model, separators=(",", ":"), ensure_ascii=False))) + has_out = "output_schema" in wire and wire["output_schema"] is not None + rows.append( + ToolTokenRow( + name=str(wire["name"]), + wire_tokens=w_tok, + model_facing_tokens=m_tok, + has_output_schema=has_out, + ) + ) + total_wire += w_tok + total_model += m_tok + rows.sort(key=lambda r: r.wire_tokens, reverse=True) + return rows, total_wire, total_model + + +def _listing_stdio_env(*, extra: dict[str, str] | None = None) -> dict[str, str]: + """Build MCP stdio env from EVAL_* credentials via the shared runner helper.""" + if not os.environ.get("EVAL_PLANE_API_KEY") or not os.environ.get("EVAL_PLANE_WORKSPACE_SLUG"): + raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for listing measurement") + return stdio_server_env(extra=extra) + + +async def list_tools_from_stdio( + command: str, + args: list[str], + env: dict[str, str], +) -> list[Any]: + """Connect to a stdio MCP server and return all tools (paginated).""" + from mcp import ClientSession + from mcp.client.stdio import StdioServerParameters, stdio_client + + params = StdioServerParameters(command=command, args=args, env=env) + tools: list[Any] = [] + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + cursor = None + while True: + page = await session.list_tools(cursor=cursor) if cursor else await session.list_tools() + tools.extend(page.tools or []) + cursor = getattr(page, "nextCursor", None) or getattr(page, "next_cursor", None) + if not cursor: + break + return tools + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Measure MCP tool listing tokens (cl100k)") + p.add_argument( + "--label", + type=str, + default="local", + help="Label printed with the listing measurement (default: local).", + ) + p.add_argument( + "--server-cmd", + type=str, + default=None, + help="External MCP stdio launch command (shlex-split)", + ) + p.add_argument( + "--server-env", + action="append", + default=[], + metavar="KEY=VAL", + help="Extra env for the MCP server child; repeatable", + ) + p.add_argument("--top", type=int, default=10, help="Top-N tools by wire tokens (default 10)") + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + extra: dict[str, str] = {} + for pair in args.server_env: + key, sep, val = pair.partition("=") + if not sep or not key: + print(f"error: --server-env expects KEY=VAL, got {pair!r}", file=sys.stderr) + return 2 + extra[key] = val + + if args.server_cmd: + parts = shlex.split(args.server_cmd) + if not parts: + print("error: --server-cmd is empty", file=sys.stderr) + return 2 + command, cmd_args = parts[0], parts[1:] + label = (args.label or "local").strip() or "local" + try: + env = _listing_stdio_env(extra=extra or None) + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + else: + command = sys.executable + cmd_args = ["-m", "plane_mcp", "stdio"] + label = (args.label or "local").strip() or "local" + try: + env = _listing_stdio_env(extra=extra or None) + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + try: + tools = asyncio.run(list_tools_from_stdio(command, cmd_args, env)) + except Exception as exc: + print(f"error: failed to list tools: {exc}", file=sys.stderr) + return 1 + + try: + rows, total_wire, total_model = count_tool_tokens(tools) + except ImportError: + print( + "error: tiktoken is required (install with: uv pip install '.[dev]')", + file=sys.stderr, + ) + return 1 + + with_out = sum(1 for r in rows if r.has_output_schema) + print( + f"label={label} tools={len(rows)} " + f"listing_tokens_cl100k={total_wire} " + f"model_facing(no_outputSchema)={total_model} " + f"tools_with_outputSchema={with_out}" + ) + top_n = max(0, int(args.top)) + if top_n and rows: + print(f"top {min(top_n, len(rows))} by wire tokens:") + for r in rows[:top_n]: + flag = " +out" if r.has_output_schema else "" + print(f" {r.wire_tokens:6d} {r.name}{flag}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/proxy.py b/evals/proxy.py new file mode 100644 index 0000000..0e32e10 --- /dev/null +++ b/evals/proxy.py @@ -0,0 +1,878 @@ +"""Stdio MCP recording proxy — byte-faithful JSON-RPC relay with a sidecar call log. + +``python -m evals.proxy --log SIDECAR.jsonl [--record-result-payloads] -- `` +Relays raw bytes both ways and parses a *copy* to log tools/call pairs. Uses ``os.read`` on +raw fds, never select + buffered readline: partial lines hang and prefetch stalls multi-line +clients. Child stderr is forwarded; exit code matches the child (signals as 128+signum). +""" + +from __future__ import annotations + +import argparse +import json +import os +import select +import signal +import subprocess +import sys +import threading +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from evals.core.error_class import classify_error, detect_refusal +from evals.core.evidence import ( + EVIDENCE_SENTINELS_ENV, + consume_evidence_config, + fingerprint_evidence_sentinels, + normalize_evidence_aggregate_shapes, + normalize_evidence_fingerprints, + normalize_evidence_sentinels, + normalize_evidence_targets, + observed_aggregates, + observed_fingerprint_labels, +) +from evals.core.tool_manifest import ToolManifestCapture + +# Single post-EOF / child-exit deadline for the whole shutdown sequence. +SHUTDOWN_DEADLINE_S = 10.0 +READ_CHUNK = 65536 + +# Repo root for PYTHONPATH scrubbing (parent of evals/). Deliberately computed +# here rather than imported from ``evals``: this module runs inside the MCP +# server's process tree with the repo scrubbed off PYTHONPATH, so it cannot +# import its own package. +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def proxy_session_log_path(configured_path: Path, *, pid: int | None = None) -> Path: + """Derive the one sidecar owned by this proxy process from the configured base.""" + process_id = os.getpid() if pid is None else pid + return configured_path.with_name(f"{configured_path.name}.{process_id}.jsonl") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Stdio MCP recording proxy (tools/call → sidecar JSONL)", + ) + p.add_argument( + "--log", + required=True, + type=Path, + help="Sidecar JSONL path for recorded tool calls + proxy_meta summary", + ) + p.add_argument( + "--record-result-payloads", + action="store_true", + help="Also store serialized tool-result text (off by default; may contain workspace data)", + ) + p.add_argument( + "--evidence-file", + type=Path, + help="Run-scoped target-evidence configuration loaded before the MCP child starts", + ) + p.add_argument( + "command", + nargs=argparse.REMAINDER, + help="Target MCP server command after --", + ) + args = p.parse_args(argv) + cmd = list(args.command or []) + if cmd and cmd[0] == "--": + cmd = cmd[1:] + if not cmd: + p.error("target command required after --") + args.command = cmd + return args + + +def map_child_returncode(rc: int | None) -> int: + """Map subprocess returncode to a conventional shell exit status. + + Negative codes mean killed by signal N (``-N``); return ``128 + N``. + """ + if rc is None: + return 1 + if rc < 0: + return 128 + (-rc) + return int(rc) + + +def write_all_fd(fd: int, data: bytes) -> None: + """Write ``data`` fully to a raw fd, looping on short writes.""" + view = memoryview(data) + offset = 0 + while offset < len(view): + n = os.write(fd, view[offset:]) + if n == 0: + raise BrokenPipeError("os.write returned 0") + offset += n + + +def scrub_child_pythonpath(env: dict[str, str] | None = None) -> dict[str, str]: + """Return a copy of ``env`` with this repo's root removed from PYTHONPATH. + + The proxy may be launched via ``python -m evals.proxy`` with PYTHONPATH set + to the monorepo root so ``evals`` is importable from a foreign cwd. That + entry must not leak into the *real* MCP server child (which may resolve + ``plane_mcp`` from its own venv). + """ + base = dict(env if env is not None else os.environ) + # Matching configuration belongs only to the recorder. The real Plane MCP server + # neither needs nor receives hidden sentinel values. + base.pop(EVIDENCE_SENTINELS_ENV, None) + root = str(REPO_ROOT) + raw = base.get("PYTHONPATH", "") + if not raw: + return base + parts = [p for p in raw.split(os.pathsep) if p and Path(p).resolve() != REPO_ROOT.resolve()] + # Also drop exact string matches that may not resolve the same way. + parts = [p for p in parts if p != root] + if parts: + base["PYTHONPATH"] = os.pathsep.join(parts) + else: + base.pop("PYTHONPATH", None) + return base + + +class SidecarRecorder: + """Thread-safe recorder for tools/call pairs into a JSONL sidecar. + + Finalization is atomic under ``_lock``: once ``write_meta`` sets + ``finalized``, further row appends no-op so ``proxy_meta`` is always the + last sidecar line even if daemon pumps keep running briefly. + """ + + def __init__( + self, + log_path: Path, + *, + record_result_payloads: bool = False, + evidence_sentinels: dict[str, Any] | None = None, + evidence_fingerprints: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, + ) -> None: + self.log_path = log_path + self.record_result_payloads = record_result_payloads + raw_sentinels = normalize_evidence_sentinels(evidence_sentinels) + self.evidence_fingerprints = normalize_evidence_fingerprints(evidence_fingerprints) + if not self.evidence_fingerprints and raw_sentinels: + self.evidence_fingerprints = fingerprint_evidence_sentinels(raw_sentinels) + self.evidence_targets = normalize_evidence_targets(evidence_targets) + self.evidence_aggregates = normalize_evidence_aggregate_shapes(evidence_aggregates) + self.evidence_active = bool( + self.evidence_fingerprints or (self.evidence_aggregates.keys() & self.evidence_targets.keys()) + ) + self._lock = threading.Lock() + self._error_lock = threading.Lock() + self._pending: dict[Any, dict[str, Any]] = {} + self._non_tool_pending: dict[Any, dict[str, Any]] = {} + self._tool_manifest = ToolManifestCapture() + self._seq = 0 + self.relayed_lines = 0 + self.unparsed_lines = 0 + self.non_json_lines = 0 + self.malformed_jsonrpc = 0 + self.recorder_errors = 0 + self.undelivered_lines = 0 + self.unmatched_responses = 0 + self.non_tool_responses = 0 + self.notifications = 0 + self.server_requests = 0 + self.child_killed = False + self.pumps_alive = False + # Which streams were still pumping at finalization. A bare boolean cannot + # distinguish "the server may still have been talking to us" from "the client + # went away while our stdin read was parked", which are different facts. + self.pumps_alive_streams: set[str] = set() + self.finalization_reason = "direct" + self.finalization_signal: str | None = None + self.finalized = False + # Post-finalize append attempts (not written; for tests / diagnostics). + self.post_finalize_appends = 0 + self.log_path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(self.log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + os.close(descriptor) + + def _append(self, row: dict[str, Any]) -> None: + line = json.dumps(row, default=str, ensure_ascii=False) + "\n" + with self._lock: + if self.finalized: + self.post_finalize_appends += 1 + return + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(line) + + def note_relayed(self) -> None: + with self._lock: + self.relayed_lines += 1 + + def note_unparsed(self, *, malformed_jsonrpc: bool = False) -> None: + with self._lock: + self.unparsed_lines += 1 + if malformed_jsonrpc: + self.malformed_jsonrpc += 1 + else: + self.non_json_lines += 1 + self.relayed_lines += 1 + + def note_recorder_error(self) -> None: + """Count a swallowed callback failure without depending on recorder state.""" + with self._error_lock: + self.recorder_errors += 1 + + def note_undelivered(self) -> None: + """Count a line recorded here that never reached the other endpoint. + + Recording happens before forwarding so a fast child cannot race an unregistered + pending id. The cost is that a broken pipe leaves a match in the sidecar for a + response the agent never saw, which would prove surface use it never had. Counting + it makes the sidecar non-authoritative instead of quietly wrong. + """ + with self._error_lock: + self.undelivered_lines += 1 + + def on_client_message(self, obj: dict[str, Any]) -> None: + """Handle a parsed JSON-RPC message from the client (parent → child).""" + has_method = "method" in obj + has_id = "id" in obj + if has_method and not has_id: + with self._lock: + self.notifications += 1 + return + if not has_method: + return + method = obj.get("method") + req_id = obj.get("id") + if method != "tools/call": + params = obj.get("params") + cursor = params.get("cursor") if isinstance(params, dict) else None + with self._lock: + self._non_tool_pending[req_id] = { + "method": str(method), + "cursor": str(cursor) if cursor is not None else None, + } + return + params = obj.get("params") or {} + if not isinstance(params, dict): + params = {} + name = params.get("name") or "" + arguments = params.get("arguments") + if arguments is None: + arguments = {} + with self._lock: + self._seq += 1 + self._pending[req_id] = { + "tool": str(name), + "args": arguments, + "t_start": time.perf_counter(), + "seq": self._seq, + } + + def on_server_message(self, obj: dict[str, Any]) -> None: + """Handle a parsed JSON-RPC message from the server (child → parent).""" + has_method = "method" in obj + has_id = "id" in obj + + if has_method and not has_id: + with self._lock: + self.notifications += 1 + if obj.get("method") == "notifications/tools/list_changed": + self._tool_manifest.invalidate() + return + if has_method and has_id: + with self._lock: + self.server_requests += 1 + return + if not has_id: + return + + req_id = obj.get("id") + with self._lock: + pending = self._pending.pop(req_id, None) + non_tool_pending = self._non_tool_pending.pop(req_id, None) if pending is None else None + if non_tool_pending is not None: + with self._lock: + self.non_tool_responses += 1 + if non_tool_pending["method"] == "tools/list" and isinstance(obj.get("result"), dict): + self._tool_manifest.observe_page( + obj["result"], + request_cursor=non_tool_pending["cursor"], + ) + return + if pending is None: + with self._lock: + self.unmatched_responses += 1 + return + duration_ms = int(round((time.perf_counter() - pending["t_start"]) * 1000)) + if "error" in obj: + is_error = True + result_payload = obj.get("error") + else: + result = obj.get("result") + is_error = False + if isinstance(result, dict): + is_error = bool(result.get("isError") or result.get("is_error")) + result_payload = result + try: + result_text = json.dumps(result_payload, default=str, ensure_ascii=False) + except Exception: + result_text = str(result_payload) + row = { + "tool": pending["tool"], + "args": pending["args"], + "is_error": is_error, + "result_chars": len(result_text), + "duration_ms": duration_ms, + "seq": pending["seq"], + } + # Classified here because this is the last place the payload exists: rows keep + # result_chars, not the text. Only the category is stored. A refusal the server + # reports as a successful result is classified too, so the metric is not blind + # to it -- see detect_refusal. + error_class = classify_error(result_text) if is_error else detect_refusal(result_text) + if error_class is not None: + row["error_class"] = error_class + if self.evidence_active: + # Persist only labels matched from non-enumerable sentinels and + # target-bound aggregate values the agent already received. The + # expected aggregate truth and complete result body never enter + # the proxy process. + row["observed_sentinels"] = observed_fingerprint_labels(result_text, self.evidence_fingerprints) + row["observed_aggregates"] = observed_aggregates( + result_text, + self.evidence_aggregates, + request_args=pending["args"], + evidence_targets=self.evidence_targets, + ) + if self.record_result_payloads: + row["result_text"] = result_text + self._append(row) + + def write_meta(self) -> None: + """Write proxy_meta as the last row and seal the sidecar (atomic under lock).""" + with self._lock: + if self.finalized: + self.post_finalize_appends += 1 + return + with self._error_lock: + recorder_errors = self.recorder_errors + undelivered_lines = self.undelivered_lines + row = { + "row_type": "proxy_meta", + "relayed_lines": self.relayed_lines, + "unparsed_lines": self.unparsed_lines, + "non_json_lines": self.non_json_lines, + "malformed_jsonrpc": self.malformed_jsonrpc, + "recorder_errors": recorder_errors, + "undelivered_lines": undelivered_lines, + "unmatched_responses": self.unmatched_responses, + "non_tool_responses": self.non_tool_responses, + "notifications": self.notifications, + "server_requests": self.server_requests, + "pending_left": len(self._pending), + "non_tool_pending_left": len(self._non_tool_pending), + "last_seq": self._seq, + "tool_request_count": self._seq, + "child_killed": self.child_killed, + "pumps_alive": self.pumps_alive, + "pumps_alive_streams": sorted(self.pumps_alive_streams), + "finalization_reason": self.finalization_reason, + "finalization_signal": self.finalization_signal, + "evidence_trace_available": self.evidence_active, + "tool_manifest_fingerprint": self._tool_manifest.fingerprint, + } + line = json.dumps(row, default=str, ensure_ascii=False) + "\n" + with self.log_path.open("a", encoding="utf-8") as fh: + fh.write(line) + self.finalized = True + + +def _valid_jsonrpc_object(obj: dict[str, Any]) -> bool: + """Validate the JSON-RPC 2.0 message envelope used by MCP stdio.""" + if obj.get("jsonrpc") != "2.0": + return False + has_method = "method" in obj + has_id = "id" in obj + if has_id and (isinstance(obj.get("id"), bool) or not isinstance(obj.get("id"), (str, int, float, type(None)))): + return False + if has_method: + if not isinstance(obj.get("method"), str) or "result" in obj or "error" in obj: + return False + params = obj.get("params") + return params is None or isinstance(params, (dict, list)) + if not has_id or ("result" in obj) == ("error" in obj): + return False + if "error" not in obj: + return True + error = obj.get("error") + return bool( + isinstance(error, dict) + and isinstance(error.get("code"), int) + and not isinstance(error.get("code"), bool) + and isinstance(error.get("message"), str) + ) + + +def try_parse_json_line(line: bytes) -> dict[str, Any] | None: + """Parse a valid JSON-RPC object line; return None on failure (never raises).""" + try: + text = line.decode("utf-8").strip() + except UnicodeDecodeError: + return None + if not text or not text.startswith("{"): + return None + try: + obj = json.loads(text) + except json.JSONDecodeError: + return None + return obj if isinstance(obj, dict) and _valid_jsonrpc_object(obj) else None + + +def classify_jsonrpc_line(line: bytes) -> tuple[str, dict[str, Any] | None]: + """Classify a framed line as blank, non-JSON, malformed JSON-RPC, or valid.""" + try: + text = line.decode("utf-8").strip() + except UnicodeDecodeError: + return "non_json", None + if not text: + return "blank", None + try: + obj = json.loads(text) + except json.JSONDecodeError: + return "non_json", None + if not isinstance(obj, dict) or not _valid_jsonrpc_object(obj): + return "malformed_jsonrpc", None + return "valid", obj + + +def process_buffer_lines( + buf: bytearray, + *, + forward_fd: int, + recorder: SidecarRecorder | None, + is_client: bool, + record_jsonrpc: bool, +) -> None: + """Split complete lines from ``buf``, record then forward, leave incomplete tail. + + **Record-before-forward**: for JSON-RPC directions, update the sidecar / + pending map *before* the line becomes visible to the opposite endpoint. + A fast child responding on stdout must never race past an unregistered + pending tools/call id; a failed parent write must not lose a completed + response that was already matched. + """ + while True: + idx = buf.find(b"\n") + if idx < 0: + break + line = bytes(buf[: idx + 1]) + del buf[: idx + 1] + if record_jsonrpc and recorder is not None: + classification, obj = classify_jsonrpc_line(line) + if classification == "blank": + recorder.note_relayed() + elif obj is None: + recorder.note_unparsed(malformed_jsonrpc=classification == "malformed_jsonrpc") + else: + recorder.note_relayed() + try: + if is_client: + recorder.on_client_message(obj) + else: + recorder.on_server_message(obj) + except Exception: + recorder.note_recorder_error() + # Forward only after recording so the opposite endpoint cannot race. + try: + write_all_fd(forward_fd, line) + except (BrokenPipeError, OSError): + if recorder is not None: + recorder.note_undelivered() + raise + + +def pump_raw( + *, + read_fd: int, + write_fd: int, + recorder: SidecarRecorder | None, + is_client: bool, + record_jsonrpc: bool, + cancel: threading.Event | None, + done: threading.Event, +) -> None: + """Byte-faithful pump: ``os.read`` + line buffer; optional cancel for stdin only. + + Stdout/stderr pumps pass ``cancel=None`` and drain until ``os.read`` returns + ``b""`` (pipe EOF) so final responses after child exit are not dropped. + Never uses buffered TextIO wrappers with select. + """ + buf = bytearray() + try: + while True: + if cancel is not None and cancel.is_set(): + break + try: + ready, _, _ = select.select([read_fd], [], [], 0.2) + except (ValueError, OSError): + break + if not ready: + continue + try: + chunk = os.read(read_fd, READ_CHUNK) + except OSError: + break + if not chunk: + break + buf.extend(chunk) + try: + process_buffer_lines( + buf, + forward_fd=write_fd, + recorder=recorder, + is_client=is_client, + record_jsonrpc=record_jsonrpc, + ) + except (BrokenPipeError, OSError): + break + # Flush remaining complete lines, then any partial tail (byte-faithful). + try: + process_buffer_lines( + buf, + forward_fd=write_fd, + recorder=recorder, + is_client=is_client, + record_jsonrpc=record_jsonrpc, + ) + except (BrokenPipeError, OSError): + pass + if buf: + try: + write_all_fd(write_fd, bytes(buf)) + except (BrokenPipeError, OSError): + pass + if record_jsonrpc and recorder is not None: + recorder.note_unparsed(malformed_jsonrpc=True) + buf.clear() + finally: + done.set() + + +def _remaining(deadline_at: float) -> float: + """Seconds left until ``deadline_at`` (never negative).""" + return max(0.0, deadline_at - time.monotonic()) + + +def reap_timeout(deadline_at: float | None, floor: float = 0.1) -> float: + """Timeout for kill/reap waits: remaining budget, never below ``floor``. + + When the overall deadline is exhausted, still allow a short reap so kill + is not skipped entirely. + """ + if deadline_at is None: + return floor + return max(floor, _remaining(deadline_at)) + + +def _signal_name(signum: int) -> str: + try: + return signal.Signals(signum).name + except ValueError: + return str(signum) + + +def _record_signal_finalization(recorder: SidecarRecorder, signum: int) -> None: + recorder.finalization_reason = "signal" + recorder.finalization_signal = _signal_name(signum) + + +def run_proxy( + command: list[str], + log_path: Path, + *, + record_result_payloads: bool = False, + evidence_sentinels: dict[str, Any] | None = None, + evidence_fingerprints: dict[str, Any] | None = None, + evidence_targets: dict[str, Any] | None = None, + evidence_aggregates: dict[str, Any] | None = None, + termination_signal: Callable[[], int | None] | None = None, +) -> int: + """Spawn ``command`` as the real MCP server and relay with recording. + + Returns the child's exit code (or 1 on spawn failure). Guarantees + ``proxy_meta`` is the last sidecar row and the child is reaped even on + crash paths. Pump threads are daemon so a blocked write cannot hold the + process past the shutdown deadline. + """ + recorder = SidecarRecorder( + log_path, + record_result_payloads=record_result_payloads, + evidence_sentinels=evidence_sentinels, + evidence_fingerprints=evidence_fingerprints, + evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, + ) + recorder.finalization_reason = "running" + # The CLI driver does not own this detached process, so leave a companion + # lifecycle file that lets it distinguish a slow finalizer from a proxy + # that exited before writing proxy_meta. The temp directory owns cleanup. + try: + log_path.with_name(f"{log_path.name}.pid").write_text(str(os.getpid()), encoding="ascii") + except OSError: + # Metadata remains authoritative. A missing lifecycle file merely + # leaves timeout diagnostics with an unknown process state. + pass + child: subprocess.Popen[bytes] | None = None + # Scrub repo PYTHONPATH so the real server does not import from this tree. + child_env = scrub_child_pythonpath() + t_in = t_out = t_err = None + stdin_done = stdout_done = stderr_done = None + deadline_at: float | None = None + try: + try: + child = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + env=child_env, + ) + except OSError as exc: + recorder.finalization_reason = "spawn_failure" + print(f"evals.proxy: failed to spawn {command!r}: {exc}", file=sys.stderr) + return 1 + + assert child.stdin is not None and child.stdout is not None and child.stderr is not None + # Raw fds — never use the buffered TextIO wrappers with select. + child_stdin_fd = child.stdin.fileno() + child_stdout_fd = child.stdout.fileno() + child_stderr_fd = child.stderr.fileno() + parent_stdin_fd = sys.stdin.fileno() + parent_stdout_fd = sys.stdout.fileno() + parent_stderr_fd = sys.stderr.fileno() + + cancel_stdin = threading.Event() + stdin_done = threading.Event() + stdout_done = threading.Event() + stderr_done = threading.Event() + + # daemon=True: a pump blocked writing to an undrained parent cannot + # keep the process alive past the shutdown deadline. + t_in = threading.Thread( + target=pump_raw, + kwargs={ + "read_fd": parent_stdin_fd, + "write_fd": child_stdin_fd, + "recorder": recorder, + "is_client": True, + "record_jsonrpc": True, + "cancel": cancel_stdin, + "done": stdin_done, + }, + name="proxy-stdin", + daemon=True, + ) + t_out = threading.Thread( + target=pump_raw, + kwargs={ + "read_fd": child_stdout_fd, + "write_fd": parent_stdout_fd, + "recorder": recorder, + "is_client": False, + "record_jsonrpc": True, + "cancel": None, # drain until pipe EOF — never gate on cancel + "done": stdout_done, + }, + name="proxy-stdout", + daemon=True, + ) + t_err = threading.Thread( + target=pump_raw, + kwargs={ + "read_fd": child_stderr_fd, + "write_fd": parent_stderr_fd, + "recorder": None, + "is_client": False, + "record_jsonrpc": False, # stderr is not JSON-RPC + "cancel": None, + "done": stderr_done, + }, + name="proxy-stderr", + daemon=True, + ) + t_in.start() + t_out.start() + t_err.start() + + # Phase 1: run until client stdin EOF or child exits. + # Child exit cancels the *stdin* pump only — stdout must drain to pipe EOF. + while ( + child.poll() is None + and not stdin_done.is_set() + and (termination_signal is None or termination_signal() is None) + ): + time.sleep(0.05) + + requested_signal = termination_signal() if termination_signal is not None else None + if requested_signal is not None: + _record_signal_finalization(recorder, requested_signal) + elif child.poll() is not None: + recorder.finalization_reason = "child_exit" + else: + recorder.finalization_reason = "normal_eof" + + # One deadline for the entire post-EOF / post-child-exit shutdown. + deadline_at = time.monotonic() + SHUTDOWN_DEADLINE_S + cancel_stdin.set() + try: + os.close(child_stdin_fd) + except OSError: + pass + + # Phase 2: wait for pumps (remaining time only — no stacked fixed timeouts). + while _remaining(deadline_at) > 0: + if stdout_done.is_set() and stderr_done.is_set() and stdin_done.is_set(): + break + if child.poll() is not None and stdout_done.is_set() and stderr_done.is_set(): + break + time.sleep(min(0.05, max(0.01, _remaining(deadline_at)))) + + rem = _remaining(deadline_at) + if rem > 0: + t_in.join(timeout=rem) + rem = _remaining(deadline_at) + if rem > 0: + t_out.join(timeout=rem) + rem = _remaining(deadline_at) + if rem > 0: + t_err.join(timeout=rem) + + if child.poll() is None: + rem = _remaining(deadline_at) + if rem > 0: + try: + child.wait(timeout=rem) + except subprocess.TimeoutExpired: + recorder.child_killed = True + child.kill() + # Bounded wait after kill — remaining budget with floor. + try: + child.wait(timeout=reap_timeout(deadline_at)) + except subprocess.TimeoutExpired: + pass + else: + recorder.child_killed = True + child.kill() + try: + child.wait(timeout=reap_timeout(deadline_at)) + except subprocess.TimeoutExpired: + pass + + # After kill/exit, join stdout/stderr again (bounded) so meta is last. + rem = _remaining(deadline_at) + if rem > 0 and t_out is not None: + t_out.join(timeout=rem) + rem = _remaining(deadline_at) + if rem > 0 and t_err is not None: + t_err.join(timeout=rem) + + for name, thread, done in ( + ("stdin", t_in, stdin_done), + ("stdout", t_out, stdout_done), + ("stderr", t_err, stderr_done), + ): + if (thread is not None and thread.is_alive()) or (done is not None and not done.is_set()): + recorder.pumps_alive_streams.add(name) + recorder.pumps_alive = bool(recorder.pumps_alive_streams) + return map_child_returncode(child.returncode) + except KeyboardInterrupt: + # Preserve Python's existing SIGINT behaviour: unwind through the + # finalizer, then let KeyboardInterrupt retain the signal exit status. + _record_signal_finalization(recorder, signal.SIGINT) + raise + except BaseException: + recorder.finalization_reason = "exception" + raise + finally: + if child is not None and child.poll() is None: + try: + recorder.child_killed = True + child.kill() + try: + child.wait(timeout=reap_timeout(deadline_at)) + except subprocess.TimeoutExpired: + pass + except Exception: + pass + # If pumps are still alive at deadline, note it; meta is still last row + # (finalized flag drops any further appends from daemon pumps). + for name, thread in (("stdin", t_in), ("stdout", t_out), ("stderr", t_err)): + if thread is not None and thread.is_alive(): + recorder.pumps_alive_streams.add(name) + if recorder.pumps_alive_streams: + recorder.pumps_alive = True + requested_signal = termination_signal() if termination_signal is not None else None + if requested_signal is not None: + _record_signal_finalization(recorder, requested_signal) + try: + recorder.write_meta() + except Exception as exc: + # Safe to continue: no meta marks the sidecar incomplete, so the parent + # driver rejects it as authoritative and falls back to the CLI trace. + print(f"evals.proxy: failed to write proxy_meta: {exc}", file=sys.stderr) + + +def main(argv: list[str] | None = None) -> int: + # Detach from the CLI's process group so a harness timeout killpg on the + # agent CLI does not SIGKILL this proxy (+ MCP child). After setsid we are + # our own session/group leader; CLI group kill leaves us alive to see stdin + # EOF, flush rows, and write proxy_meta within the shutdown deadline. + try: + os.setsid() + except OSError: + # Already a session leader, or platform forbids setsid — continue. + pass + args = parse_args(argv) + evidence_fingerprints, evidence_targets, evidence_aggregates = consume_evidence_config(args.evidence_file) + received_signal: list[int | None] = [None] + + def request_termination(signum: int, _frame: Any) -> None: + # Do not finalize in the handler: it can interrupt code holding the + # recorder lock. The relay loop observes this state and drains first. + if received_signal[0] is None: + received_signal[0] = signum + + previous_handlers: dict[int, Any] = {} + for signum in (signal.SIGTERM, signal.SIGHUP): + previous_handlers[signum] = signal.getsignal(signum) + signal.signal(signum, request_termination) + try: + returncode = run_proxy( + list(args.command), + proxy_session_log_path(Path(args.log)), + record_result_payloads=bool(args.record_result_payloads), + evidence_fingerprints=evidence_fingerprints, + evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, + termination_signal=lambda: received_signal[0], + ) + finally: + for signum, previous_handler in previous_handlers.items(): + signal.signal(signum, previous_handler) + + signum = received_signal[0] + if signum is not None: + # Metadata and child cleanup are complete. Re-deliver with the + # default disposition so subprocess/shell status encodes the signal. + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/report/__init__.py b/evals/report/__init__.py new file mode 100644 index 0000000..a341a8a --- /dev/null +++ b/evals/report/__init__.py @@ -0,0 +1,147 @@ +"""Evaluation result reports.""" + +from .command import main +from .compare import ab_compare, print_ab_report +from .identity import ( + ComparabilityError, + FileIdentity, + IdentityReport, + format_refusal, + identity_header_lines, + parse_varied_dimensions, + validate_persisted_identity, +) +from .load import ( + DedupeMode, + ResultRow, + RunExpectation, + RunKeyValidation, + dedupe_rows_latest, + is_infra_error_row, + is_meta_row, + load_rows, + load_run_expectation, + load_run_expected_rows, + read_result, + validate_run_keys, +) +from .off_surface import ( + ANSWER_WITHOUT_PROVENANCE, + IMPLAUSIBLY_FEW_CALLS, + INDICATOR_LABELS, + INDICATOR_ORDER, + LOW_CALL_RULE, + OFF_SURFACE_LIMITATION, + WRITE_WITHOUT_WRITE_CALL, + ZERO_CALL_SUCCESS, + OffSurfaceMeasurement, + OffSurfaceRow, + call_plausibly_writes, + measure_off_surface, + off_surface_statement, + task_requires_mutation, +) +from .schema_friction import ( + SCHEMA_FRICTION_LIMITATION, + SchemaFrictionMeasurement, + TaskSchemaFriction, + measure_schema_friction, + schema_friction_statement, + successful_trace_rows, +) +from .statistics import iqr, median, paired_bootstrap_mean_ci, paired_permutation_pvalue, percentile, wilson_interval +from .summary import ( + ResultTokensMode, + Summary, + TaskSummary, + completeness_statement, + execution_coverage_statement, + result_tokens_mode, + summarize, +) +from .table import ( + build_multi_surface_table, + format_multi_rep_surface_cell, + format_number, + format_result_tokens, + format_surface_cell, + format_tool_distribution, + format_tool_variability, + print_table, + prompt_excerpt, + render_multi_surface_table, + result_tokens_marker, + surface_label_for_file, + task_sort_key, +) + +__all__ = [ + "DedupeMode", + "RunExpectation", + "RunKeyValidation", + "ComparabilityError", + "FileIdentity", + "IdentityReport", + "OffSurfaceMeasurement", + "OffSurfaceRow", + "ResultRow", + "ResultTokensMode", + "Summary", + "SchemaFrictionMeasurement", + "TaskSummary", + "TaskSchemaFriction", + "ANSWER_WITHOUT_PROVENANCE", + "IMPLAUSIBLY_FEW_CALLS", + "INDICATOR_LABELS", + "INDICATOR_ORDER", + "LOW_CALL_RULE", + "OFF_SURFACE_LIMITATION", + "WRITE_WITHOUT_WRITE_CALL", + "ZERO_CALL_SUCCESS", + "SCHEMA_FRICTION_LIMITATION", + "ab_compare", + "build_multi_surface_table", + "call_plausibly_writes", + "completeness_statement", + "dedupe_rows_latest", + "format_multi_rep_surface_cell", + "format_number", + "format_result_tokens", + "format_refusal", + "format_surface_cell", + "format_tool_distribution", + "format_tool_variability", + "execution_coverage_statement", + "iqr", + "is_infra_error_row", + "is_meta_row", + "identity_header_lines", + "load_rows", + "load_run_expectation", + "load_run_expected_rows", + "main", + "median", + "measure_off_surface", + "measure_schema_friction", + "off_surface_statement", + "paired_bootstrap_mean_ci", + "paired_permutation_pvalue", + "parse_varied_dimensions", + "percentile", + "print_ab_report", + "print_table", + "prompt_excerpt", + "read_result", + "validate_run_keys", + "render_multi_surface_table", + "result_tokens_marker", + "result_tokens_mode", + "schema_friction_statement", + "summarize", + "surface_label_for_file", + "successful_trace_rows", + "task_requires_mutation", + "task_sort_key", + "validate_persisted_identity", + "wilson_interval", +] diff --git a/evals/report/__main__.py b/evals/report/__main__.py new file mode 100644 index 0000000..615f849 --- /dev/null +++ b/evals/report/__main__.py @@ -0,0 +1,14 @@ +"""Command-line entry for evaluation reports. + +Usage: + python -m evals.report evals/output/A.jsonl + python -m evals.report A.jsonl B.jsonl # paired A/B bootstrap + permutation + python -m evals.report --vary resolved_model A.jsonl B.jsonl + python -m evals.report --table f1.jsonl f2.jsonl … # per-task × per-surface + python -m evals.report --table --markdown f1.jsonl f2.jsonl +""" + +from .command import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/report/command.py b/evals/report/command.py new file mode 100644 index 0000000..e54aee5 --- /dev/null +++ b/evals/report/command.py @@ -0,0 +1,167 @@ +"""Command-line behavior for evaluation reports.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from evals.core.results import TaskResult + +from .compare import ab_compare, print_ab_report +from .identity import ( + ComparabilityError, + format_refusal, + identity_header_lines, + parse_varied_dimensions, + validate_persisted_identity, +) +from .load import ( + DedupeMode, + RunKeyValidation, + load_rows, + load_run_expectation, + load_run_expected_rows, + validate_run_keys, +) +from .summary import summarize +from .table import ( + build_multi_surface_table, + print_table, + render_multi_surface_table, + surface_label_for_file, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Summarize eval JSONL results") + parser.add_argument( + "files", + nargs="*", + help="JSONL file(s): one for summary, two for A/B, N with --table", + ) + parser.add_argument( + "--table", + action="store_true", + help="Multi-surface per-task table (one column per file, using its run label)", + ) + parser.add_argument( + "--markdown", + action="store_true", + help="With --table, emit a GitHub-flavored markdown table", + ) + parser.add_argument( + "--vary", + action="append", + default=[], + metavar="DIM[,DIM]", + help="Declare treatment dimensions (resolved_model, provider, driver, or server)", + ) + parser.add_argument( + "--no-dedupe", + action="store_true", + help="Keep all rows (forensics); default is latest-wins per (task_id,rep,label)", + ) + arguments = parser.parse_args(argv) + try: + varied_dimensions = parse_varied_dimensions(arguments.vary) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + dedupe: DedupeMode = "none" if arguments.no_dedupe else "latest" + + if not arguments.files: + parser.print_help() + return 2 + + paths = [Path(file_name) for file_name in arguments.files] + for path in paths: + if not path.exists(): + print(f"error: file not found: {path}", file=sys.stderr) + return 2 + + if not arguments.table and len(paths) not in {1, 2}: + print( + "error: pass one JSONL (summary), two (A/B delta), or use --table with N files", + file=sys.stderr, + ) + return 2 + + try: + identity = validate_persisted_identity(paths, varied_dimensions=varied_dimensions) + run_keys_by_path: dict[Path, RunKeyValidation | None] = {} + for path in paths: + expectation = load_run_expectation(path) + run_keys_by_path[path] = ( + validate_run_keys(load_rows(path, dedupe="none"), expectation) if expectation is not None else None + ) + except ComparabilityError as exc: + print(format_refusal(exc), file=sys.stderr) + return 2 + except ValueError as exc: + print(f"error: invalid run expectation: {exc}", file=sys.stderr) + return 2 + header_lines = identity_header_lines( + identity, + warn_missing_manifest=arguments.table or len(paths) == 2, + ) + for line in header_lines: + print(line) + if header_lines and arguments.table and arguments.markdown: + print() + + if arguments.table: + if len(paths) < 1: + print("error: --table requires at least one JSONL", file=sys.stderr) + return 2 + labeled: list[tuple[str, list[TaskResult]]] = [] + expected_by_label: dict[str, int | None] = {} + run_keys_by_label: dict[str, RunKeyValidation | None] = {} + used_labels: set[str] = set() + for path in paths: + rows = load_rows(path, dedupe=dedupe) + label = surface_label_for_file(path, rows) + # Disambiguate duplicate run labels (e.g. two external files). + label_root = label + number = 2 + while label in used_labels: + label = f"{label_root}-{number}" + number += 1 + used_labels.add(label) + labeled.append((label, rows)) + expected_by_label[label] = load_run_expected_rows(path) + run_keys_by_label[label] = run_keys_by_path[path] + table = build_multi_surface_table( + labeled, + expected_rows_by_column=expected_by_label, + run_keys_by_column=run_keys_by_label, + ) + sys.stdout.write(render_multi_surface_table(table, markdown=arguments.markdown)) + return 0 if all(values["complete"] for values in table["footer"].values()) else 1 + + if len(paths) == 1: + path = paths[0] + rows = load_rows(path, dedupe=dedupe) + summary = summarize( + rows, + expected_rows=load_run_expected_rows(path), + run_keys=run_keys_by_path[path], + ) + print_table(summary, f"Summary: {path}") + return 0 if summary.complete else 1 + + if len(paths) == 2: + rows_a = load_rows(paths[0], dedupe=dedupe) + rows_b = load_rows(paths[1], dedupe=dedupe) + comparison = ab_compare( + rows_a, + rows_b, + expected_rows_a=load_run_expected_rows(paths[0]), + expected_rows_b=load_run_expected_rows(paths[1]), + run_keys_a=run_keys_by_path[paths[0]], + run_keys_b=run_keys_by_path[paths[1]], + ) + print_ab_report(comparison, paths[0], paths[1]) + return 0 if comparison["summary_a"].complete and comparison["summary_b"].complete else 1 + + raise AssertionError("validated report arity did not select a command path") diff --git a/evals/report/compare.py b/evals/report/compare.py new file mode 100644 index 0000000..da069d1 --- /dev/null +++ b/evals/report/compare.py @@ -0,0 +1,363 @@ +"""A/B comparison for evaluation result sets.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .economics import economics_statement +from .failure_kinds import failure_kind_statement +from .load import ResultRow, RunKeyValidation +from .off_surface import off_surface_statement +from .power import power_statement +from .schema_friction import measure_schema_friction, schema_friction_statement, successful_trace_rows +from .statistics import median, paired_bootstrap_mean_ci, paired_permutation_pvalue +from .summary import completeness_statement, execution_coverage_statement, summarize +from .table import format_number + + +def _paired_metric( + economics_a: Any, + economics_b: Any, + shared: list[str], + attribute: str, +) -> dict[str, Any]: + """Pair one per-task resource metric across arms, skipping tasks either side lacks. + + Same shape as the call delta -- mean, median and a paired bootstrap -- so the + resource lines read against it directly rather than in different units. + """ + deltas: list[float] = [] + for task_id in shared: + task_a = economics_a.tasks.get(task_id) + task_b = economics_b.tasks.get(task_id) + if task_a is None or task_b is None: + continue + value_a = getattr(task_a, attribute) + value_b = getattr(task_b, attribute) + if value_a is None or value_b is None: + continue + deltas.append(float(value_b) - float(value_a)) + return { + "n": len(deltas), + # Missingness that correlates with expensive tasks can bias a delta, so the + # count of shared tasks this metric could not use travels with it. + "dropped": len(shared) - len(deltas), + "mean_delta": sum(deltas) / len(deltas) if deltas else None, + "median_delta": median(deltas), + "ci": paired_bootstrap_mean_ci(deltas), + } + + +def ab_compare( + rows_a: list[ResultRow], + rows_b: list[ResultRow], + *, + expected_rows_a: int | None = None, + expected_rows_b: int | None = None, + run_keys_a: RunKeyValidation | None = None, + run_keys_b: RunKeyValidation | None = None, +) -> dict[str, Any]: + """Compare two result sets with task-paired call and success deltas. + + Paired call deltas only include tasks with at least one successful repetition + in both A and B. Calls are the median across successful repetitions; this is + identical to the historical behavior for single-rep files. + + Success-rate differences pair each task's completed-repetition rate across + labels, then bootstrap whole task pairs. This treats tasks as independent + sampling units and assumes the labels cover comparable task instances. + """ + summary_a = summarize(rows_a, expected_rows=expected_rows_a, run_keys=run_keys_a) + summary_b = summarize(rows_b, expected_rows=expected_rows_b, run_keys=run_keys_b) + + def successful_calls_by_task(rows: list[ResultRow]) -> dict[str, list[float]]: + output: dict[str, list[float]] = {} + for row in successful_trace_rows(rows): + output.setdefault(row.task_id, []).append(float(row.num_calls)) + return output + + calls_a = successful_calls_by_task(rows_a) + calls_b = successful_calls_by_task(rows_b) + shared = sorted(set(calls_a) & set(calls_b)) + deltas: list[float] = [] + per_task: list[dict[str, Any]] = [] + for task_id in shared: + count_a = float(median(calls_a[task_id]) or 0.0) + count_b = float(median(calls_b[task_id]) or 0.0) + delta = count_b - count_a # B − A (negative = B fewer calls = better if lower is better) + deltas.append(delta) + per_task.append( + { + "task_id": task_id, + "calls_a": count_a, + "calls_b": count_b, + "delta": delta, + } + ) + + success_shared = sorted( + task_id + for task_id in set(summary_a.tasks) & set(summary_b.tasks) + if summary_a.tasks[task_id].n and summary_b.tasks[task_id].n + ) + success_deltas = [ + (summary_b.tasks[task_id].k / summary_b.tasks[task_id].n) + - (summary_a.tasks[task_id].k / summary_a.tasks[task_id].n) + for task_id in success_shared + ] + paired_success_delta = sum(success_deltas) / len(success_deltas) if success_deltas else None + paired_success_ci = paired_bootstrap_mean_ci(success_deltas) + + friction_a = measure_schema_friction(rows_a) + friction_b = measure_schema_friction(rows_b) + paired_schema_friction: list[dict[str, Any]] = [] + errored_call_deltas: list[float] = [] + errored_call_rate_deltas: list[float] = [] + for task_id in shared: + task_a = friction_a.tasks[task_id] + task_b = friction_b.tasks[task_id] + errored_call_delta = task_b.median_errored_calls - task_a.median_errored_calls + rate_a = task_a.errored_call_rate + rate_b = task_b.errored_call_rate + rate_delta = rate_b - rate_a if rate_a is not None and rate_b is not None else None + errored_call_deltas.append(errored_call_delta) + if rate_delta is not None: + errored_call_rate_deltas.append(rate_delta) + paired_schema_friction.append( + { + "task_id": task_id, + "errored_calls_a": task_a.median_errored_calls, + "errored_calls_b": task_b.median_errored_calls, + "errored_call_delta": errored_call_delta, + "errored_call_rate_a": rate_a, + "errored_call_rate_b": rate_b, + "errored_call_rate_delta": rate_delta, + "raw_errored_calls_a": task_a.errored_calls, + "raw_total_calls_a": task_a.total_calls, + "raw_errored_calls_b": task_b.errored_calls, + "raw_total_calls_b": task_b.total_calls, + } + ) + + # Resource deltas over the same shared tasks as the call delta. Fewer calls and + # less spend are different virtues, and reporting one without the other is what + # let a 32%-fewer-calls arm read as the cheaper one while burning 3.1x the input. + economics_a, economics_b = summary_a.economics, summary_b.economics + paired_resources = { + name: _paired_metric(economics_a, economics_b, shared, attribute) + for name, attribute in ( + ("input", "med_total_input"), + ("result_tokens", "med_result_tokens"), + ("wall_time", "med_wall_time_s"), + ("call_latency", "med_call_latency_ms"), + ("cost", "cost_usd"), + ) + } + + return { + "summary_a": summary_a, + "summary_b": summary_b, + "paired_tasks": per_task, + "economics_a": economics_a, + "economics_b": economics_b, + "total_input_a": economics_a.total_input_tokens, + "total_input_b": economics_b.total_input_tokens, + "cost_a": economics_a.cost_usd, + "cost_b": economics_b.cost_usd, + "paired_resources": paired_resources, + "mean_delta": sum(deltas) / len(deltas) if deltas else None, + "median_delta": median(deltas), + "call_permutation_p": paired_permutation_pvalue(deltas), + "call_zero_deltas": sum(delta == 0 for delta in deltas), + "n_paired": len(deltas), + "paired_success_tasks": success_shared, + "n_paired_success": len(success_deltas), + "paired_success_delta": paired_success_delta, + "paired_success_ci": paired_success_ci, + "paired_schema_friction": paired_schema_friction, + "mean_errored_call_delta": ( + sum(errored_call_deltas) / len(errored_call_deltas) if errored_call_deltas else None + ), + "errored_call_delta_ci": paired_bootstrap_mean_ci(errored_call_deltas), + "mean_errored_call_rate_delta": ( + sum(errored_call_rate_deltas) / len(errored_call_rate_deltas) if errored_call_rate_deltas else None + ), + "errored_call_rate_delta_ci": paired_bootstrap_mean_ci(errored_call_rate_deltas), + "n_paired_errored_call_rates": len(errored_call_rate_deltas), + "multi_rep": summary_a.multi_rep or summary_b.multi_rep, + "success_a": { + "k": summary_a.aggregate_k, + "n": summary_a.aggregate_n, + "wilson": ( + summary_a.aggregate_wilson_lo, + summary_a.aggregate_wilson_hi, + ), + "task_mean": summary_a.task_mean_success, + "task_cluster": (summary_a.task_cluster_lo, summary_a.task_cluster_hi), + "task_n": sum(task.n > 0 for task in summary_a.tasks.values()), + }, + "success_b": { + "k": summary_b.aggregate_k, + "n": summary_b.aggregate_n, + "wilson": ( + summary_b.aggregate_wilson_lo, + summary_b.aggregate_wilson_hi, + ), + "task_mean": summary_b.task_mean_success, + "task_cluster": (summary_b.task_cluster_lo, summary_b.task_cluster_hi), + "task_n": sum(task.n > 0 for task in summary_b.tasks.values()), + }, + } + + +#: Label, units and precision for each paired resource delta. +_RESOURCE_LINES: tuple[tuple[str, str, str, int], ...] = ( + ("input", "input tokens", "", 0), + ("result_tokens", "result tokens", "", 0), + ("cost", "cost per successful rep", "$", 4), + ("wall_time", "wall time", "s", 1), + ("call_latency", "call latency", "ms", 0), +) + + +def _print_resource_deltas(comparison: dict[str, Any]) -> None: + """Print the resource deltas beside the call delta, in the same paired shape. + + These sit immediately after the call delta on purpose: the call delta alone says + which arm did less work, which is not the same question as which arm cost less, + and the two answers pointed opposite ways on the run that motivated this. + """ + paired = comparison.get("paired_resources") or {} + for key, label, unit, places in _RESOURCE_LINES: + metric = paired.get(key) + if not metric or not metric["n"]: + print(f" median {label} delta (B−A): n/a (no paired tasks reporting it)") + continue + dropped = metric.get("dropped") or 0 + omitted = f", {dropped} shared task(s) omitted for missing values" if dropped else "" + low, high = metric["ci"] + prefix = unit if unit == "$" else "" + suffix = "" if unit == "$" else unit + interval = ( + f" paired-bootstrap95 [{prefix}{low:+,.{places}f}{suffix},{prefix}{high:+,.{places}f}{suffix}]" + if low is not None and high is not None + else "" + ) + print( + f" median {label} delta (B−A): {prefix}{metric['median_delta']:+,.{places}f}{suffix}" + f"{interval} (n={metric['n']} tasks{omitted})" + ) + for label, key in (("A", "economics_a"), ("B", "economics_b")): + economics = comparison.get(key) + if economics is None: + continue + for line in economics_statement(economics).splitlines(): + print(f" {label} {line}") + + +def print_ab_report(comparison: dict[str, Any], path_a: Path, path_b: Path) -> None: + print(f"A/B compare: A={path_a} B={path_b}") + success_a, success_b = comparison["success_a"], comparison["success_b"] + rate_a = (success_a["k"] / success_a["n"]) if success_a["n"] else 0.0 + rate_b = (success_b["k"] / success_b["n"]) if success_b["n"] else 0.0 + for label, success, pooled_rate in (("A", success_a, rate_a), ("B", success_b, rate_b)): + task_mean = success["task_mean"] + task_lo, task_hi = success["task_cluster"] + if task_mean is None or task_lo is None or task_hi is None: + print(f" success {label} task-cluster: n/a (no evaluated tasks)") + else: + print( + f" success {label} task-cluster: {task_mean:.1%} " + f"cluster-bootstrap95 [{task_lo:.2f},{task_hi:.2f}] (n={success['task_n']} tasks)" + ) + print( + f" success {label} pooled repetitions: {success['k']}/{success['n']} ({pooled_rate:.1%}) " + f"Wilson95 [{success['wilson'][0]:.2f},{success['wilson'][1]:.2f}]" + ) + print(f" A {execution_coverage_statement(comparison['summary_a'])}") + print(f" B {execution_coverage_statement(comparison['summary_b'])}") + for label, summary in (("A", comparison["summary_a"]), ("B", comparison["summary_b"])): + for line in off_surface_statement(summary.off_surface).splitlines(): + print(f" {label} {line}") + for line in schema_friction_statement(summary.schema_friction).splitlines(): + print(f" {label} {line}") + for line in failure_kind_statement(summary.failure_kinds).splitlines(): + print(f" {label} {line}") + print(f" {label} {summary.lookup_reuse.statement()}") + for label, summary in (("A", comparison["summary_a"]), ("B", comparison["summary_b"])): + power = power_statement(summary) + if power: + print(f" {label} {power}") + print(f" A {completeness_statement(comparison['summary_a'])}") + print(f" B {completeness_statement(comparison['summary_b'])}") + print(f" success rate delta (B−A): {rate_b - rate_a:+.1%}") + paired_success_delta = comparison["paired_success_delta"] + paired_success_lo, paired_success_hi = comparison["paired_success_ci"] + if paired_success_delta is None or paired_success_lo is None or paired_success_hi is None: + print(" paired task success delta (B−A): n/a (no shared evaluated tasks)") + else: + print( + f" paired task success delta (B−A): {paired_success_delta:+.1%} " + f"paired-bootstrap95 [{paired_success_lo:+.1%},{paired_success_hi:+.1%}] " + f"(n={comparison['n_paired_success']} tasks)" + ) + print(f" paired successful tasks: {comparison['n_paired']}") + print(f" mean call delta (B−A): {format_number(comparison['mean_delta'])}") + print(f" median call delta (B−A): {format_number(comparison['median_delta'])}") + probability = comparison["call_permutation_p"] + tie_count = comparison["call_zero_deltas"] + print( + " paired permutation p-value for mean call delta (two-sided): " + f"{probability if probability is not None else 'n/a'} " + f"({tie_count} zero-delta ties retained)" + ) + errored_delta = comparison["mean_errored_call_delta"] + errored_lo, errored_hi = comparison["errored_call_delta_ci"] + if errored_delta is None or errored_lo is None or errored_hi is None: + print(" mean errored-call delta (B−A): n/a (no paired successful tasks)") + else: + print( + f" mean errored-call delta (B−A): {errored_delta:+.1f} " + f"paired-bootstrap95 [{errored_lo:+.1f},{errored_hi:+.1f}] " + f"(n={comparison['n_paired']} tasks)" + ) + rate_delta = comparison["mean_errored_call_rate_delta"] + rate_lo, rate_hi = comparison["errored_call_rate_delta_ci"] + if rate_delta is None or rate_lo is None or rate_hi is None: + print(" mean errored-call-rate delta (B−A): n/a (no paired tasks with calls on both surfaces)") + else: + print( + f" mean errored-call-rate delta (B−A): {rate_delta * 100:+.1f} percentage points " + f"paired-bootstrap95 [{rate_lo * 100:+.1f},{rate_hi * 100:+.1f}] " + f"(n={comparison['n_paired_errored_call_rates']} tasks)" + ) + _print_resource_deltas(comparison) + multiple_repetitions = bool(comparison.get("multi_rep")) + if comparison["paired_tasks"]: + print() + print(f"{'task':<6} {'calls_A':>8} {'calls_B':>8} {'delta':>8}") + print("-" * 34) + for row in comparison["paired_tasks"]: + if multiple_repetitions: + print( + f"{row['task_id']:<6} {format_number(row['calls_a']):>8} " + f"{format_number(row['calls_b']):>8} {row['delta']:>+8.1f}" + ) + else: + print(f"{row['task_id']:<6} {row['calls_a']:>8.0f} {row['calls_b']:>8.0f} {row['delta']:>+8.0f}") + if comparison["paired_schema_friction"]: + print() + print("schema friction by paired task (raw errors/calls; median errors per successful repetition):") + for row in comparison["paired_schema_friction"]: + rate_a = row["errored_call_rate_a"] + rate_b = row["errored_call_rate_b"] + rate_a_text = f"{rate_a:.1%}" if rate_a is not None else "n/a" + rate_b_text = f"{rate_b:.1%}" if rate_b is not None else "n/a" + print( + f" {row['task_id']}: " + f"A={row['raw_errored_calls_a']}/{row['raw_total_calls_a']} ({rate_a_text}), " + f"median={row['errored_calls_a']:.1f}; " + f"B={row['raw_errored_calls_b']}/{row['raw_total_calls_b']} ({rate_b_text}), " + f"median={row['errored_calls_b']:.1f}" + ) diff --git a/evals/report/economics.py b/evals/report/economics.py new file mode 100644 index 0000000..67352f9 --- /dev/null +++ b/evals/report/economics.py @@ -0,0 +1,287 @@ +"""What a run cost and how much it moved, in the view where two arms are compared. + +Every number here was already recorded. Result tokens even reached ``--table``. None +of it reached the two-file A/B view, which is where a two-arm question is actually +asked -- so an arm making 32% fewer tool calls while burning 3.1x the input tokens +read as the efficient one until someone totalled the tokens by hand. + +Two populations, deliberately: + + arm totals every executed row, because cost was incurred whether or not the + task passed. + per task the successful, trace-intact rows that call deltas already use, so a + paired delta compares like with like. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass + +from evals.core.pricing import PRICED, PRICES_AS_OF, UNMEASURED, UNPRICED, price_usage +from evals.core.results import TaskResult +from evals.core.token_accounting import has_token_counts, normalize_usage + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .schema_friction import successful_trace_rows +from .statistics import median, percentile + + +def format_usd(amount: float) -> str: + """Render dollars without rounding a real figure down to nothing. + + A genuine $0.00032 printed as $0.000 is the same "reads as free" mistake the three + cost outcomes exist to prevent, so small non-zero amounts keep enough digits to stay + visible. Used for every figure, the drift diagnostic included. + """ + magnitude = abs(amount) + if magnitude < 1e-9: + # Float noise from summing many rows, not a real fraction of a cent. + return "$0.000" + if magnitude < 0.001: + return f"{'-' if amount < 0 else ''}${magnitude:.2e}" + return f"${amount:,.3f}" if amount >= 0 else f"-${magnitude:,.3f}" + + +COST_LIMITATION = ( + "limitation: cost is computed from a static price table; a model absent from it reports " + "unpriced, and a row whose driver recorded no usage at all reports unmeasured. Neither is $0" +) + + +@dataclass(frozen=True, slots=True) +class TaskEconomics: + """One task's resource footprint across its eligible repetitions.""" + + task_id: str + repetitions: int + med_total_input: float | None + med_result_tokens: float | None + med_wall_time_s: float | None + med_call_latency_ms: float | None + cost_usd: float | None + """Mean billed cost per successful repetition -- not a total, unlike the arm figure.""" + + +@dataclass(frozen=True, slots=True) +class EconomicsMeasurement: + """Arm-level totals plus the per-task values a paired delta needs.""" + + tasks: dict[str, TaskEconomics] + total_input_tokens: int | None + total_result_tokens: int + total_wall_time_s: float + cost_usd: float | None + computed_cost_usd: float | None + vendor_cost_usd: float | None + cost_outcome: str + priced_rows: int + unpriced_rows: int + unmeasured_rows: int + missing_input_rows: int + drift_computed_usd: float | None + drift_vendor_usd: float | None + drift_rows: int + med_call_latency_ms: float | None + p95_call_latency_ms: float | None + prices_as_of: str = PRICES_AS_OF + + @property + def cost_drift_usd(self) -> float | None: + """How far the price table sits from what the vendor said it charged. + + Computed over the rows carrying *both* figures, so a row the vendor priced but the + table could not (or the reverse) adds coverage noise to neither side. None when no + row carries both, which is most runs. + """ + if self.drift_computed_usd is None or self.drift_vendor_usd is None: + return None + return self.drift_computed_usd - self.drift_vendor_usd + + @property + def cost_text(self) -> str: + """Never render an unknown cost as a number, or a real one as zero.""" + if self.cost_usd is None: + return UNMEASURED if self.cost_outcome == UNMEASURED else UNPRICED + text = format_usd(self.cost_usd) + if self.unpriced_rows or self.unmeasured_rows: + text += f" (+{self.unpriced_rows} unpriced, {self.unmeasured_rows} unmeasured rows)" + return text + + @property + def input_text(self) -> str: + """The input total, saying so when it does not cover every row.""" + if self.total_input_tokens is None: + return UNMEASURED + text = f"{self.total_input_tokens:,}" + if self.missing_input_rows: + text += f" (excludes {self.missing_input_rows} row(s) with unreadable usage)" + return text + + +def _charged_rows(rows: list[ResultRow]) -> list[TaskResult]: + """Every row whose model actually ran, including ones that later went wrong. + + A verifier crash, a contained timeout or a post-run skip happens after the tokens + were spent, so excluding those rows understates an arm and hides the spend + entirely -- it was not even counted as unmeasured. A row that carries usage is + kept regardless of how it ended; one that never ran is not. + """ + charged: list[TaskResult] = [] + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row): + continue + if row.error or row.skipped or is_infra_error_row(row): + # An infrastructure classification is applied *after* the agent run is folded + # in, so a contained CLI timeout or trace failure can carry real usage. Keep + # any row that shows the model ran; drop only ones that never started. + if not has_token_counts(row.usage_total) and not row.calls: + continue + charged.append(row) + return charged + + +def _row_input_tokens(row: TaskResult) -> int | None: + accounting = normalize_usage(row.usage_total, model=row.model) + return accounting.total_input if accounting else None + + +def _call_latencies(rows: list[TaskResult]) -> list[float]: + return [float(call.duration_ms) for row in rows for call in row.calls if call.duration_ms is not None] + + +def measure_economics(rows: list[ResultRow]) -> EconomicsMeasurement: + """Total an arm's cost and volume, and break it down per task.""" + executed = _charged_rows(rows) + + total_input = 0 + saw_input = False + missing_input = 0 + billed_total = 0.0 + saw_billed = False + computed_total = 0.0 + saw_computed = False + vendor_total = 0.0 + saw_vendor = False + drift_computed = 0.0 + drift_vendor = 0.0 + drift_rows = 0 + priced = unpriced = unmeasured = 0 + for row in executed: + tokens = _row_input_tokens(row) + if tokens is not None: + total_input += tokens + saw_input = True + else: + missing_input += 1 + cost = price_usage(row.usage_total, model=row.model) + if cost.outcome == PRICED: + priced += 1 + elif cost.outcome == UNPRICED: + unpriced += 1 + else: + unmeasured += 1 + # Billed is what to report; computed is the table's own opinion, kept apart so + # the drift check compares the table against the vendor rather than the vendor + # against itself. Billed accrues on any row that has a figure, so an + # authoritative vendor cost survives a model the table cannot price. + if cost.billed_usd is not None: + billed_total += cost.billed_usd + saw_billed = True + if cost.usd is not None: + computed_total += cost.usd + saw_computed = True + if cost.vendor_usd is not None: + vendor_total += cost.vendor_usd + saw_vendor = True + # Drift is only meaningful over rows that carry *both* figures. Summing each side + # independently would fold coverage differences into what is meant to be a + # price-table comparison. + if cost.usd is not None and cost.vendor_usd is not None: + drift_computed += cost.usd + drift_vendor += cost.vendor_usd + drift_rows += 1 + + if not saw_billed: + # Nothing to report: say which kind of nothing it is. + outcome = UNMEASURED if unpriced == 0 else UNPRICED + else: + outcome = PRICED # possibly partial; the counts travel alongside + + by_task: dict[str, list[TaskResult]] = defaultdict(list) + for row in successful_trace_rows(rows): + by_task[row.task_id].append(row) + + tasks: dict[str, TaskEconomics] = {} + for task_id in sorted(by_task): + task_rows = by_task[task_id] + inputs = [float(value) for value in (_row_input_tokens(row) for row in task_rows) if value is not None] + costs = [price_usage(row.usage_total, model=row.model).billed_usd for row in task_rows] + known_costs = [value for value in costs if value is not None] + tasks[task_id] = TaskEconomics( + task_id=task_id, + repetitions=len(task_rows), + med_total_input=median(inputs), + med_result_tokens=median([float(row.total_result_tokens) for row in task_rows]), + med_wall_time_s=median([float(row.wall_time_s) for row in task_rows]), + med_call_latency_ms=median(_call_latencies(task_rows)), + cost_usd=(sum(known_costs) / len(known_costs) if known_costs else None), + ) + + latencies = _call_latencies(executed) + return EconomicsMeasurement( + tasks=tasks, + total_input_tokens=total_input if saw_input else None, + total_result_tokens=sum(row.total_result_tokens for row in executed), + total_wall_time_s=sum(float(row.wall_time_s) for row in executed), + cost_usd=billed_total if saw_billed else None, + computed_cost_usd=computed_total if saw_computed else None, + vendor_cost_usd=vendor_total if saw_vendor else None, + missing_input_rows=missing_input, + drift_computed_usd=drift_computed if drift_rows else None, + drift_vendor_usd=drift_vendor if drift_rows else None, + drift_rows=drift_rows, + cost_outcome=outcome, + priced_rows=priced, + unpriced_rows=unpriced, + unmeasured_rows=unmeasured, + med_call_latency_ms=median(latencies), + p95_call_latency_ms=percentile(latencies, 0.95), + ) + + +def economics_statement(measurement: EconomicsMeasurement) -> str: + """One block naming cost, volume and latency, with unknowns named as unknowns.""" + input_text = measurement.input_text + latency = measurement.med_call_latency_ms + p95 = measurement.p95_call_latency_ms + latency_text = f"{latency:,.0f}ms median / {p95:,.0f}ms p95" if latency is not None and p95 is not None else "n/a" + lines = [ + f"economics: cost={measurement.cost_text} (prices as of {measurement.prices_as_of}); " + f"input tokens={input_text}; result tokens={measurement.total_result_tokens:,}", + f" wall time={measurement.total_wall_time_s:,.0f}s; call latency {latency_text}", + ] + drift = measurement.cost_drift_usd + if drift is not None: + # The only standing check that the price table has not gone stale, so it compares + # the table's own figure against the vendor's -- not the reported cost, which + # already prefers the vendor and would always agree with itself. + lines.append( + f" price-table check over {measurement.drift_rows} row(s) carrying both: " + f"vendor {format_usd(measurement.drift_vendor_usd)}, " + f"table {format_usd(measurement.drift_computed_usd)} " + f"(differs by {format_usd(drift)})" + ) + lines.append(f" {COST_LIMITATION}") + return "\n".join(lines) + + +__all__ = [ + "COST_LIMITATION", + "format_usd", + "EconomicsMeasurement", + "TaskEconomics", + "economics_statement", + "measure_economics", +] diff --git a/evals/report/failure_kinds.py b/evals/report/failure_kinds.py new file mode 100644 index 0000000..44d2afe --- /dev/null +++ b/evals/report/failure_kinds.py @@ -0,0 +1,93 @@ +"""Group a run's failures by what kind of wrong they were.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass, field + +from evals.core.failure_kind import FAILURE_KINDS, NON_DEFECT_KINDS, UNCLASSIFIED, classify_failure +from evals.core.results import TaskResult + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result + +FAILURE_KIND_LIMITATION = ( + "limitation: kinds are read from verifier note text, so they describe what the verifier " + "could see. A write that landed on the wrong entity reports as a missing or partial write, " + "because the right entity is empty either way -- distinguishing those needs call arguments" +) + + +@dataclass(frozen=True, slots=True) +class FailureKindMeasurement: + """Counts per kind, plus the task ids behind each.""" + + counts: dict[str, int] = field(default_factory=dict) + task_ids: dict[str, tuple[str, ...]] = field(default_factory=dict) + total: int = 0 + + @property + def defects(self) -> int: + """Failures attributable to the agent rather than the run or environment.""" + return sum(count for kind, count in self.counts.items() if kind not in NON_DEFECT_KINDS) + + @property + def non_defects(self) -> int: + return sum(self.counts.get(kind, 0) for kind in NON_DEFECT_KINDS) + + +def measure_failure_kinds(rows: list[ResultRow]) -> FailureKindMeasurement: + """Classify every failed row that carries a verifier note.""" + counts: dict[str, int] = dict.fromkeys(FAILURE_KINDS, 0) + tasks: dict[str, set[str]] = defaultdict(set) + total = 0 + for raw_row in rows: + row: TaskResult = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped: + continue + if row.success: + continue + kind = classify_failure( + row.verify_note, + stop_reason=row.stop_reason, + hit_max_iterations=row.hit_max_iterations, + ) + counts[kind] += 1 + tasks[kind].add(row.task_id) + total += 1 + return FailureKindMeasurement( + counts=counts, + task_ids={kind: tuple(sorted(ids)) for kind, ids in sorted(tasks.items())}, + total=total, + ) + + +def failure_kind_statement(measurement: FailureKindMeasurement) -> str: + """Name every kind that occurred, and say plainly which are not agent defects.""" + if not measurement.total: + return "failure kinds: no failed rows" + parts = [ + f"{kind}={measurement.counts[kind]}" + + (f" [{', '.join(measurement.task_ids[kind])}]" if kind in measurement.task_ids else "") + for kind in FAILURE_KINDS + if measurement.counts.get(kind) + ] + lines = [f"failure kinds ({measurement.total} failed rows): " + "; ".join(parts)] + if measurement.non_defects: + lines.append( + f" {measurement.non_defects} of {measurement.total} are not agent defects: a correct answer " + "the run could not evidence, an environment gap, or a capped run" + ) + unclassified = measurement.counts.get(UNCLASSIFIED, 0) + if unclassified: + # Never let a zero in some kind stand in for "the classifier did not recognise it". + lines.append(f" {unclassified} note(s) matched no pattern, so the split above is incomplete by that much") + lines.append(f" {FAILURE_KIND_LIMITATION}") + return "\n".join(lines) + + +__all__ = [ + "FAILURE_KIND_LIMITATION", + "FailureKindMeasurement", + "failure_kind_statement", + "measure_failure_kinds", +] diff --git a/evals/report/identity.py b/evals/report/identity.py new file mode 100644 index 0000000..2035494 --- /dev/null +++ b/evals/report/identity.py @@ -0,0 +1,236 @@ +"""Persisted run-identity validation shared by every report command path.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from evals.report.load import is_unlaunched_row + +MISSING = "" +IDENTITY_FIELDS = ("battery", "resolved_model", "provider", "driver", "server") +VARYABLE_DIMENSIONS = ("resolved_model", "provider", "driver", "server") +TOOL_MANIFEST_FIELD = "tool_manifest_fingerprint" + + +class ComparabilityError(ValueError): + """The persisted records do not establish the requested comparison.""" + + def __init__(self, details: Iterable[str]) -> None: + self.details = tuple(details) + super().__init__("; ".join(self.details)) + + +@dataclass(frozen=True, slots=True) +class FileIdentity: + """Validated canonical identity and realized-model observations for one file.""" + + path: Path + values: dict[str, str] + realized_models: tuple[str, ...] + + @property + def realized_model_changed(self) -> bool: + return len(self.realized_models) > 1 + + +@dataclass(frozen=True, slots=True) +class IdentityReport: + """Identity evidence safe to print beside report measurements.""" + + files: tuple[FileIdentity, ...] + varied_dimensions: tuple[str, ...] + + +def persisted_value(value: Any) -> str: + """Normalize absent and empty persisted values to an explicit, non-wildcard value.""" + if value is None or value == "": + return MISSING + return str(value) + + +def parse_varied_dimensions(raw_values: Iterable[str]) -> tuple[str, ...]: + """Parse repeatable comma-separated --vary declarations.""" + requested: list[str] = [] + for raw_value in raw_values: + requested.extend(part.strip() for part in raw_value.split(",")) + if not requested: + return () + if any(not dimension for dimension in requested): + raise ValueError("--vary requires a dimension name") + if "all" in requested: + raise ValueError("--vary has no 'all'; name each treatment dimension individually") + if "battery" in requested: + raise ValueError("--vary battery is not allowed; the measurement universe cannot be waived") + unknown = sorted(set(requested) - set(VARYABLE_DIMENSIONS)) + if unknown: + valid = ", ".join(VARYABLE_DIMENSIONS) + raise ValueError(f"unknown --vary dimension(s): {', '.join(unknown)}; choose from: {valid}") + requested_set = set(requested) + return tuple(dimension for dimension in VARYABLE_DIMENSIONS if dimension in requested_set) + + +def _read_records(path: Path) -> tuple[list[tuple[int, dict[str, Any]]], list[tuple[int, dict[str, Any]]]]: + headers: list[tuple[int, dict[str, Any]]] = [] + rows: list[tuple[int, dict[str, Any]]] = [] + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + record = {} + if not isinstance(record, dict): + record = {} + target = headers if record.get("row_type") == "meta" else rows + target.append((line_number, record)) + return headers, rows + + +def _record_values(records: list[tuple[int, dict[str, Any]]], field: str) -> dict[str, list[int]]: + values: dict[str, list[int]] = {} + for line_number, record in records: + values.setdefault(persisted_value(record.get(field)), []).append(line_number) + return values + + +def _format_values(values: dict[str, list[int]]) -> str: + return ", ".join(f"{value} (line(s) {','.join(str(line) for line in lines)})" for value, lines in values.items()) + + +def _validate_file(path: Path) -> tuple[FileIdentity, list[str]]: + headers, rows = _read_records(path) + issues: list[str] = [] + values: dict[str, str] = {} + + for field in IDENTITY_FIELDS: + header_values = _record_values(headers, field) + row_values = _record_values(rows, field) + if len(header_values) > 1: + issues.append(f"{path}: conflicting meta headers for {field}: {_format_values(header_values)}") + if len(row_values) > 1: + issues.append(f"{path}: rows disagree on {field}: {_format_values(row_values)}") + if header_values and row_values and set(header_values) != set(row_values): + issues.append( + f"{path}: meta header disagrees with raw rows on {field}: " + f"header={_format_values(header_values)}; rows={_format_values(row_values)}" + ) + source = row_values or header_values or {MISSING: []} + values[field] = next(iter(source)) + + # A row that ended during seeding never launched an agent, so no tools/list was ever + # observed and there is no manifest to record. Demanding one from those rows refuses + # comparisons that are perfectly sound: the first live A/B was blocked by six + # infra_seed rows that every statistic already excludes. + surface_rows = [(line, record) for line, record in rows if not is_unlaunched_row(record)] + manifest_values = _record_values(surface_rows, TOOL_MANIFEST_FIELD) + if len(manifest_values) > 1: + issues.append(f"{path}: rows disagree on {TOOL_MANIFEST_FIELD}: {_format_values(manifest_values)}") + values[TOOL_MANIFEST_FIELD] = next(iter(manifest_values), MISSING) + + realized_models = tuple(sorted(_record_values(rows, "model"))) + return FileIdentity(path=path, values=values, realized_models=realized_models), issues + + +def validate_persisted_identity( + paths: Iterable[Path], + *, + varied_dimensions: Iterable[str] = (), +) -> IdentityReport: + """Validate files internally and against each other before any dedupe or statistics.""" + varied = tuple(varied_dimensions) + identities: list[FileIdentity] = [] + issues: list[str] = [] + for path in paths: + identity, file_issues = _validate_file(path) + identities.append(identity) + issues.extend(file_issues) + + if len(identities) > 1: + for field in IDENTITY_FIELDS: + by_path = {str(identity.path): identity.values[field] for identity in identities} + if len(set(by_path.values())) <= 1: + continue + if field != "battery" and field in varied: + continue + detail = "; ".join(f"{path}={value}" for path, value in by_path.items()) + issues.append(f"{field} differs across files: {detail}") + manifest_values = {identity.values[TOOL_MANIFEST_FIELD] for identity in identities} + if MISSING in manifest_values: + unidentified = [ + str(identity.path) for identity in identities if identity.values[TOOL_MANIFEST_FIELD] == MISSING + ] + issues.append( + f"{TOOL_MANIFEST_FIELD} is missing for comparison input(s): {', '.join(unidentified)}; " + "every compared surface must be identified" + ) + + if issues: + raise ComparabilityError(issues) + return IdentityReport(files=tuple(identities), varied_dimensions=varied) + + +def format_refusal(error: ComparabilityError) -> str: + """Render an exit-2 refusal without any report measurements.""" + lines = ["error: comparability cannot be established from the persisted identity"] + lines.extend(f" - {detail}" for detail in error.details) + return "\n".join(lines) + + +def identity_header_lines(report: IdentityReport, *, warn_missing_manifest: bool = False) -> list[str]: + """Render treatment declarations and non-canonical realized-model evidence.""" + lines: list[str] = [] + varied = report.varied_dimensions + if varied: + treatment = ", ".join(varied) + if len(varied) > 1: + treatment += " — end-to-end comparison; effect not attributable to any single dimension" + elif varied == ("driver",): + treatment += " — end-to-end driver question; cannot support a surface-only claim" + lines.append(f"Treatment: {treatment}") + if "driver" in varied and len(varied) > 1: + lines.append("Driver interpretation: end-to-end driver question; cannot support a surface-only claim") + for dimension in varied: + values = "; ".join(f"{identity.path}={identity.values[dimension]}" for identity in report.files) + lines.append(f"Treatment values ({dimension}): {values}") + + evidence = [ + identity for identity in report.files if identity.realized_models and identity.realized_models != (MISSING,) + ] + if evidence: + detail = "; ".join(f"{identity.path}={','.join(identity.realized_models)}" for identity in evidence) + lines.append(f"Realized model evidence: {detail}") + for identity in evidence: + if identity.realized_model_changed: + lines.append( + f"WARNING: realized model changed within {identity.path}: {','.join(identity.realized_models)}" + ) + if any(identity.values[TOOL_MANIFEST_FIELD] != MISSING for identity in report.files): + manifests = "; ".join(f"{identity.path}={identity.values[TOOL_MANIFEST_FIELD]}" for identity in report.files) + lines.append(f"Tool manifest evidence: {manifests}") + missing_manifests = [ + str(identity.path) for identity in report.files if identity.values[TOOL_MANIFEST_FIELD] == MISSING + ] + if warn_missing_manifest and missing_manifests: + lines.append("WARNING: TOOL MANIFEST ABSENT — tool surface is unidentified for " + ", ".join(missing_manifests)) + return lines + + +__all__ = [ + "ComparabilityError", + "FileIdentity", + "IDENTITY_FIELDS", + "IdentityReport", + "MISSING", + "TOOL_MANIFEST_FIELD", + "VARYABLE_DIMENSIONS", + "format_refusal", + "identity_header_lines", + "parse_varied_dimensions", + "persisted_value", + "validate_persisted_identity", +] diff --git a/evals/report/load.py b/evals/report/load.py new file mode 100644 index 0000000..6aaf426 --- /dev/null +++ b/evals/report/load.py @@ -0,0 +1,265 @@ +"""JSONL row loading and error handling for evaluation reports.""" + +from __future__ import annotations + +import json +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from evals.core.results import TaskResult +from evals.result_lifecycle import is_terminal_result + +DedupeMode = Literal["latest", "none"] +ResultRow = TaskResult | dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class RunExpectation: + """Exact task/repetition universe declared by a result-file meta header.""" + + task_ids: tuple[str, ...] + reps: int + label: str | None = None + + @property + def expected_rows(self) -> int: + return len(self.task_ids) * self.reps + + @property + def keys(self) -> frozenset[tuple[str, int]]: + return frozenset((task_id, rep) for task_id in self.task_ids for rep in range(self.reps)) + + +@dataclass(frozen=True, slots=True) +class RunKeyValidation: + """Raw-row comparison against one exact run expectation.""" + + expectation: RunExpectation + missing: tuple[str, ...] + unexpected: tuple[str, ...] + + @property + def exact(self) -> bool: + return not self.missing and not self.unexpected + + +def _first_meta_row(path: Path) -> dict[str, Any] | None: + with path.open(encoding="utf-8") as file: + for line in file: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(row, dict) or row.get("row_type") != "meta": + return None + return row + return None + + +def load_run_expectation(path: Path) -> RunExpectation | None: + """Reconstruct the exact expected ``(task_id, rep)`` set when declared.""" + row = _first_meta_row(path) + if row is None: + return None + raw_task_ids = row.get("expected_task_ids") + raw_reps = row.get("expected_reps") + if raw_task_ids is None and raw_reps is None: + return None + if not isinstance(raw_task_ids, list) or raw_reps is None: + raise ValueError(f"{path}: meta must declare expected_task_ids and expected_reps together") + task_ids = tuple(str(task_id) for task_id in raw_task_ids) + try: + reps = int(raw_reps) + except (TypeError, ValueError) as exc: + raise ValueError(f"{path}: expected_reps must be a positive integer") from exc + if not task_ids or any(not task_id for task_id in task_ids): + raise ValueError(f"{path}: expected_task_ids must be a non-empty list of non-empty ids") + if len(set(task_ids)) != len(task_ids): + raise ValueError(f"{path}: expected_task_ids contains duplicates") + if reps < 1: + raise ValueError(f"{path}: expected_reps must be a positive integer") + expectation = RunExpectation(task_ids=task_ids, reps=reps, label=str(row["label"]) if row.get("label") else None) + declared_rows = row.get("expected_rows") + if declared_rows is not None and int(declared_rows) != expectation.expected_rows: + raise ValueError( + f"{path}: expected_rows={declared_rows} disagrees with exact expectation={expectation.expected_rows}" + ) + return expectation + + +def _format_run_key(key: tuple[str, int], count: int) -> str: + rendered = f"{key[0]}[rep={key[1]}]" + return f"{rendered} x{count}" if count > 1 else rendered + + +def validate_run_keys(rows: list[ResultRow], expectation: RunExpectation) -> RunKeyValidation: + """Validate exact keys while allowing append-only retry history. + + For an expected key, every occurrence except the last must be retryable. The final + occurrence is authoritative. A prior terminal occurrence is a genuine duplicate and + remains visible as an unexpected key. + """ + expected = Counter({key: 1 for key in expectation.keys}) + histories: dict[tuple[str, int, str | None], list[TaskResult]] = defaultdict(list) + for raw_row in rows: + row = read_result(raw_row) + if not is_meta_row(row): + row_label = row.label if expectation.label is not None else None + histories[(row.task_id, row.rep, row_label)].append(row) + expected_history_keys = {(task_id, rep, expectation.label) for task_id, rep in expectation.keys} + observed = Counter( + {(task_id, rep): len(histories.get((task_id, rep, expectation.label), ())) for task_id, rep in expectation.keys} + ) + missing_counts = expected - observed + unexpected_counts: Counter[tuple[str, int]] = Counter() + for history_key, history in histories.items(): + key = history_key[:2] + if history_key not in expected_history_keys: + unexpected_counts[key] += len(history) + continue + terminal_predecessors = sum(is_terminal_result(row) for row in history[:-1]) + if terminal_predecessors: + unexpected_counts[key] += terminal_predecessors + return RunKeyValidation( + expectation=expectation, + missing=tuple(_format_run_key(key, count) for key, count in sorted(missing_counts.items())), + unexpected=tuple(_format_run_key(key, count) for key, count in sorted(unexpected_counts.items())), + ) + + +def _invalid_result_row(path: Path, line_number: int, reason: str) -> TaskResult: + """Represent an unreadable persisted row as a completeness-visible harness error.""" + return TaskResult( + task_id=f"", + rep=line_number, + label=path.stem, + success=False, + error=f"{path}:{line_number}: {reason}", + error_class="harness_report_load", + ) + + +def load_run_expected_rows(path: Path) -> int | None: + """Read the declared run size from the JSONL meta header, when available.""" + if expectation := load_run_expectation(path): + return expectation.expected_rows + row = _first_meta_row(path) + if row is None: + return None + value = row.get("expected_rows") + return int(value) if value is not None else None + + +def read_result(row: ResultRow) -> TaskResult: + """Return one row as the declared persisted result type.""" + return row if isinstance(row, TaskResult) else TaskResult.from_row(row) + + +def is_meta_row(row: ResultRow) -> bool: + """True for run-header meta lines (or any row without a task_id).""" + result = read_result(row) + return result.row_type == "meta" or not result.task_id + + +def is_infra_error_row(row: ResultRow) -> bool: + """True when a row failed for infrastructure reasons, not task verification. + + Any ``error_class`` starting with ``infra_`` (``infra_seed``, ``infra_cli``, + ``infra_api``, …) is excluded from success-rate denominators. + """ + error_class = read_result(row).error_class + return isinstance(error_class, str) and error_class.startswith("infra_") + + +def is_unlaunched_row(row: ResultRow) -> bool: + """True when no agent ran for this row, so it observed no tool manifest. + + A seed failure and a seed-time skip both end before the agent starts. Neither can + carry a manifest fingerprint, so neither can be held to one — demanding it refused + perfectly sound comparisons: one plan-gated task made every report command exit. + """ + result = read_result(row) + return is_infra_error_row(row) or bool(result.skipped) + + +def dedupe_rows_latest(rows: list[ResultRow]) -> list[TaskResult]: + """Keep only the last row per (task_id, rep, label); preserve key insertion order.""" + latest: dict[tuple[str, int, str], TaskResult] = {} + order: list[tuple[str, int, str]] = [] + for raw_row in rows: + row = read_result(raw_row) + key = (row.task_id, row.rep, row.label) + if key not in latest: + order.append(key) + latest[key] = row + return [latest[key] for key in order] + + +def load_rows(path: Path, *, dedupe: DedupeMode = "latest") -> list[TaskResult]: + """Load JSONL data rows, representing malformed data as harness errors. + + Default ``dedupe="latest"`` keeps the last row per (task_id, rep, label) + so resume appends do not double-count. Pass ``dedupe="none"`` for forensics. + """ + rows: list[TaskResult] = [] + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: {path}:{line_number}: recording invalid JSON as a harness error ({exc})", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, f"invalid JSON ({exc})")) + continue + if not isinstance(row, dict): + print( + f"warning: {path}:{line_number}: recording non-object JSON as a harness error", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, "result row is not a JSON object")) + continue + if row.get("row_type") == "meta": + continue + try: + result = TaskResult.from_row(row) + except Exception as exc: + print( + f"warning: {path}:{line_number}: recording invalid result object as a harness error ({exc})", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, f"invalid result object ({exc})")) + continue + if not result.task_id: + print( + f"warning: {path}:{line_number}: recording result without task_id as a harness error", + file=sys.stderr, + ) + rows.append(_invalid_result_row(path, line_number, "result row has no task_id")) + continue + rows.append(result) + if dedupe == "latest": + return dedupe_rows_latest(rows) + # Forensics: warn on duplicates but keep all. + seen_keys: set[tuple[str, int, str]] = set() + for row in rows: + key = (row.task_id, row.rep, row.label) + if key in seen_keys: + print( + f"warning: {path}: duplicate (task_id, rep, label)={key} " + "(--no-dedupe keeps append-only history; validator distinguishes retries from duplicates)", + file=sys.stderr, + ) + else: + seen_keys.add(key) + return rows diff --git a/evals/report/lookup_reuse.py b/evals/report/lookup_reuse.py new file mode 100644 index 0000000..9c07030 --- /dev/null +++ b/evals/report/lookup_reuse.py @@ -0,0 +1,143 @@ +"""How often an agent goes looking for an entity it has already resolved. + +The sharpest single finding of the 2026-08-24 cross-harness pair was +``workitem.search`` 111 vs 26 against ``workitem.retrieve`` 4 vs 20. One arm carried +resolved ids across turns; the other went looking again each time. That is a +property of the **surface** as much as of the agent -- identifiers that stayed +sticky would close the gap without either agent changing -- and nothing measured it. + +The rule is deliberately narrow, because the first version was not and counted +things no reasonable reader would call redundant: + + same entity only the resource's *own* identifier counts. ``project_id`` on a + work item call says where to look, not which item is known, and + treating it as "in hand" flagged every list in a project. + not paging a lookup carrying a cursor or offset is continuing one traversal, + not starting a second. + args only ids are read from request arguments. An id that arrived in a + *result* is invisible here, so this undercounts rather than over. + +What survives is still a heuristic: retrieving item A and then searching for an +unrelated item B of the same resource is counted, because nothing in the arguments +distinguishes that from re-hunting A. Read it as an upper bound on identifier +stickiness, not as a defect count. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from dataclasses import dataclass, field + +from evals.core.results import CallRecord, TaskResult + +from .load import ResultRow, is_meta_row, read_result + +LOOKUP_REUSE_LIMITATION = ( + "limitation: counts a search/list on a resource whose own id was already an argument, " + "excluding paginated continuations. It cannot tell re-hunting the same entity from looking " + "up a different one of the same kind, and ids that arrived only in results are invisible" +) + +#: Actions that go looking for something rather than addressing it directly. +_SEARCH_ACTIONS = frozenset({"search", "list", "list_archived"}) + +#: Arguments that mean "continue the previous traversal" rather than "look again". +_PAGINATION_ARGS = frozenset({"cursor", "offset", "page", "next_cursor", "page_token"}) + + +def _args_of(call: CallRecord) -> dict | None: + if not call.args_json: + return None + try: + parsed = json.loads(call.args_json) + except (TypeError, ValueError): + return None + return parsed if isinstance(parsed, dict) else None + + +def _own_ids(resource: str, args: dict) -> set[str]: + """Identifiers of *this* resource, ignoring scope and cross-references. + + ``workitem_id`` on the ``workitem`` tool is the entity; ``project_id`` is the + scope it lives in, and ``cycle_id`` is a different resource entirely. + """ + own = {"id", f"{resource}_id"} + return {value for name, value in args.items() if name in own and isinstance(value, str) and value} + + +@dataclass(frozen=True, slots=True) +class LookupReuseMeasurement: + """Redundant lookups, and whether the question could be asked at all.""" + + total: int = 0 + by_resource: dict[str, int] = field(default_factory=dict) + rows_measured: int = 0 + rows_without_args: int = 0 + calls_without_args: int = 0 + + @property + def measurable(self) -> bool: + return self.rows_measured > 0 + + def statement(self) -> str: + if not self.measurable: + return f"redundant lookups: not measured — {self.rows_without_args} row(s) carry no recorded call arguments" + detail = ", ".join(f"{resource}={count}" for resource, count in sorted(self.by_resource.items())) + line = f"redundant lookups: {self.total}" + if detail: + line += f" [{detail}]" + extras = [] + if self.rows_without_args: + extras.append(f"{self.rows_without_args} row(s) not measured for want of arguments") + if self.calls_without_args: + extras.append(f"{self.calls_without_args} call(s) skipped inside measured rows") + if extras: + line += "; " + "; ".join(extras) + return f"{line}\n {LOOKUP_REUSE_LIMITATION}" + + +def measure_lookup_reuse(rows: list[ResultRow]) -> LookupReuseMeasurement: + """Count searches issued after the same resource's own id was already an argument. + + Scoped to one row. Each repetition is a fresh conversation, so an id learned in + one tells the agent in another nothing. + """ + total = 0 + by_resource: dict[str, int] = defaultdict(int) + measured = 0 + rows_without_args = 0 + calls_without_args = 0 + for raw_row in rows: + row: TaskResult = read_result(raw_row) + if is_meta_row(row): + continue + if not any(call.args_json for call in row.calls): + rows_without_args += 1 + continue + measured += 1 + known: dict[str, set[str]] = defaultdict(set) + for call in row.calls: + args = _args_of(call) + if args is None: + calls_without_args += 1 + continue + resource = call.tool + action = (call.action or "").lower() + paging = any(name in args for name in _PAGINATION_ARGS) + if action in _SEARCH_ACTIONS and known[resource] and not paging: + total += 1 + by_resource[resource] += 1 + # Learned after the check, so the call that first resolves an id is never + # charged for the lookup that produced it. + known[resource].update(_own_ids(resource, args)) + return LookupReuseMeasurement( + total=total, + by_resource=dict(by_resource), + rows_measured=measured, + rows_without_args=rows_without_args, + calls_without_args=calls_without_args, + ) + + +__all__ = ["LOOKUP_REUSE_LIMITATION", "LookupReuseMeasurement", "measure_lookup_reuse"] diff --git a/evals/report/off_surface.py b/evals/report/off_surface.py new file mode 100644 index 0000000..eebc0d7 --- /dev/null +++ b/evals/report/off_surface.py @@ -0,0 +1,270 @@ +"""Trace-signature indicators for possible work outside the measured MCP surface.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import TRACE_INTEGRITY_SCHEMA_VERSION, CallRecord, TaskResult +from evals.core.task_metadata import task_metadata_from_rows + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import percentile + +ZERO_CALL_SUCCESS = "zero_call_success" +WRITE_WITHOUT_WRITE_CALL = "write_without_write_call" +ANSWER_WITHOUT_PROVENANCE = "answer_without_provenance" +IMPLAUSIBLY_FEW_CALLS = "implausibly_few_calls" + +INDICATOR_ORDER = ( + ZERO_CALL_SUCCESS, + WRITE_WITHOUT_WRITE_CALL, + ANSWER_WITHOUT_PROVENANCE, + IMPLAUSIBLY_FEW_CALLS, +) +INDICATOR_LABELS = { + ZERO_CALL_SUCCESS: "zero-call success", + WRITE_WITHOUT_WRITE_CALL: "write without a write call", + ANSWER_WITHOUT_PROVENANCE: "answer without provenance", + IMPLAUSIBLY_FEW_CALLS: "implausibly few calls", +} + +LOW_CALL_MIN_REPETITIONS = 5 +LOW_CALL_IQR_MULTIPLIER = 3.0 +LOW_CALL_RULE = ( + "among at least 5 successful trace-usable repetitions for the same task, " + "calls < Q1 - 3×IQR and calls ≤ half the task median" +) +OFF_SURFACE_LIMITATION = ( + "detects off-surface work only when it leaves a trace signature; it cannot detect " + "an agent that performs the work off-surface and also makes convincing surface calls" +) + +_MUTATING_TASK_TAGS = frozenset({"setup", "write"}) +_MUTATING_VERBS = frozenset( + { + "accept", + "add", + "approve", + "archive", + "assign", + "attach", + "cancel", + "complete", + "create", + "decline", + "delete", + "detach", + "disable", + "duplicate", + "enable", + "link", + "manage", + "move", + "publish", + "reject", + "remove", + "restore", + "set", + "start", + "submit", + "transfer", + "unarchive", + "unlink", + "update", + "upload", + } +) + + +@dataclass(frozen=True, slots=True) +class OffSurfaceRow: + """The indicator set attached to one persisted result row.""" + + task_id: str + rep: int + indicators: frozenset[str] + + @property + def address(self) -> str: + return f"{self.task_id}[rep={self.rep}]" + + +@dataclass(frozen=True, slots=True) +class OffSurfaceMeasurement: + """Per-row findings and their run-level aggregate views.""" + + rows: tuple[OffSurfaceRow, ...] = () + mutation_intent_available: bool = True + """False when no run declared its task tags, so write-intent cannot be judged.""" + + @property + def flagged_rows(self) -> int: + return len(self.rows) + + @property + def indicator_hits(self) -> int: + return sum(len(row.indicators) for row in self.rows) + + def addresses(self, indicator: str) -> tuple[str, ...]: + return tuple(row.address for row in self.rows if indicator in row.indicators) + + +def _trace_usable(row: TaskResult) -> bool: + """Accept authoritative traces and legacy rows predating typed trace integrity.""" + return row.trace_integrity is True or ( + row.trace_integrity is None and row.schema_version < TRACE_INTEGRITY_SCHEMA_VERSION + ) + + +def _successful_row(row: TaskResult) -> bool: + return bool( + row.success and not row.error and not row.skipped and not is_infra_error_row(row) and _trace_usable(row) + ) + + +def task_requires_mutation(task: Mapping[str, Any] | None) -> bool: + """Derive mutation intent from persisted tags instead of a task-id allowlist.""" + tags = task.get("tags") if task is not None else () + return bool(_MUTATING_TASK_TAGS.intersection(str(tag) for tag in (tags or ()))) + + +def call_plausibly_writes(call: CallRecord) -> bool: + """Conservatively recognize successful calls with a mutating verb or action.""" + if call.is_error: + return False + candidates = (call.action, call.tool) + for candidate in candidates: + normalized = str(candidate or "").strip().casefold().replace("-", "_") + verb = normalized.split("_", 1)[0] + if verb in _MUTATING_VERBS: + return True + return False + + +def _has_target_provenance(row: TaskResult) -> bool: + return any(not call.is_error and TARGET_ENTITY_EVIDENCE in (call.observed_sentinels or ()) for call in row.calls) + + +def _answer_was_correct(row: TaskResult) -> bool: + # Provenance-enforced read rows already have success=False when their answer was + # correct but evidence was missing. ``answer_with_provenance`` persists the two + # facts separately in this stable verifier note. + return row.success or "answer_correct=true" in row.verify_note.casefold() + + +def measure_off_surface( + rows: list[ResultRow], + *, + task_catalog: Mapping[str, Mapping[str, Any]] | None = None, +) -> OffSurfaceMeasurement: + """Compute suspicion indicators without changing row success or completeness. + + ``task_catalog`` carries the run's own task facts. When absent it falls back to the + metadata persisted in the rows' meta header; a file written before that header existed + yields no mutation intent, which suppresses one indicator rather than inventing it from + a catalog that may have changed since. + """ + if task_catalog is None: + task_catalog = task_metadata_from_rows(rows) + mutation_intent_available = bool(task_catalog) + results: list[TaskResult] = [] + for raw_row in rows: + if is_meta_row(raw_row): + continue + results.append(read_result(raw_row)) + + flags_by_index: dict[int, set[str]] = defaultdict(set) + successful_by_task: dict[str, list[tuple[int, TaskResult]]] = defaultdict(list) + for index, row in enumerate(results): + successful = _successful_row(row) + if successful: + successful_by_task[row.task_id].append((index, row)) + if row.num_calls == 0 and not row.calls: + flags_by_index[index].add(ZERO_CALL_SUCCESS) + if task_requires_mutation(task_catalog.get(row.task_id)) and not any( + call_plausibly_writes(call) for call in row.calls + ): + flags_by_index[index].add(WRITE_WITHOUT_WRITE_CALL) + + if ( + _trace_usable(row) + and not row.error + and not row.skipped + and row.evidence_trace_available + and _answer_was_correct(row) + and not _has_target_provenance(row) + ): + flags_by_index[index].add(ANSWER_WITHOUT_PROVENANCE) + + for task_rows in successful_by_task.values(): + if len(task_rows) < LOW_CALL_MIN_REPETITIONS: + continue + call_counts = [float(row.num_calls) for _, row in task_rows] + first_quartile = percentile(call_counts, 0.25) + task_median = percentile(call_counts, 0.5) + third_quartile = percentile(call_counts, 0.75) + assert first_quartile is not None and task_median is not None and third_quartile is not None + lower_outer_fence = first_quartile - LOW_CALL_IQR_MULTIPLIER * (third_quartile - first_quartile) + for index, row in task_rows: + if row.num_calls < lower_outer_fence and row.num_calls <= task_median / 2.0: + flags_by_index[index].add(IMPLAUSIBLY_FEW_CALLS) + + findings = tuple( + OffSurfaceRow( + task_id=row.task_id, + rep=row.rep, + indicators=frozenset(flags_by_index[index]), + ) + for index, row in enumerate(results) + if flags_by_index[index] + ) + return OffSurfaceMeasurement(rows=findings, mutation_intent_available=mutation_intent_available) + + +def off_surface_statement(measurement: OffSurfaceMeasurement) -> str: + """Render an investigation-ready aggregate, including explicit zero results.""" + if measurement.flagged_rows: + headline = ( + f"off-surface indicators: {measurement.flagged_rows} flagged rows " + f"({measurement.indicator_hits} indicator hits)" + ) + else: + headline = "off-surface indicators: 0" + lines = [headline] + for indicator in INDICATOR_ORDER: + # An indicator that could not be evaluated says so in its own position. A bare zero + # here would read as "checked and clean", which is the opposite of unknown. + if indicator == WRITE_WITHOUT_WRITE_CALL and not measurement.mutation_intent_available: + lines.append( + f" {INDICATOR_LABELS[indicator]}: not evaluated — this file declares no task tags, " + "so mutation intent is unknown" + ) + continue + addresses = measurement.addresses(indicator) + suffix = f" [{', '.join(addresses)}]" if addresses else "" + rule = f"; rule: {LOW_CALL_RULE}" if indicator == IMPLAUSIBLY_FEW_CALLS else "" + lines.append(f" {INDICATOR_LABELS[indicator]}: {len(addresses)}{suffix}{rule}") + lines.append(f" limitation: {OFF_SURFACE_LIMITATION}") + return "\n".join(lines) + + +__all__ = [ + "ANSWER_WITHOUT_PROVENANCE", + "IMPLAUSIBLY_FEW_CALLS", + "INDICATOR_LABELS", + "INDICATOR_ORDER", + "LOW_CALL_RULE", + "OFF_SURFACE_LIMITATION", + "OffSurfaceMeasurement", + "OffSurfaceRow", + "WRITE_WITHOUT_WRITE_CALL", + "ZERO_CALL_SUCCESS", + "call_plausibly_writes", + "measure_off_surface", + "off_surface_statement", + "task_requires_mutation", +] diff --git a/evals/report/power.py b/evals/report/power.py new file mode 100644 index 0000000..ff0f9f3 --- /dev/null +++ b/evals/report/power.py @@ -0,0 +1,50 @@ +"""Say plainly when the per-task numbers cannot carry a verdict. + +A run's aggregate and its per-task rows have very different power, and the report +prints both in the same table. At 2 repetitions a task that passed once is +``1/2 UNSTABLE`` with a 95% interval of roughly [0.09, 0.91] -- compatible with +almost any true success rate -- while the paired aggregate across 35 tasks resolved +a call difference at p=0.0018. Reading a per-task row as a finding is therefore +wrong in exactly the runs where the aggregate is most convincing. + +No new machinery: ``--tasks`` already allows a focused high-rep subset, and at +roughly $0.50 an arm that is affordable. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover - import cycle guard, summary imports this module + from .summary import Summary + +#: Repetitions per task below which a per-task pass rate is not worth reading. +UNDERPOWERED_REPS = 5 + + +def power_statement(summary: Summary) -> str | None: + """Return the guardrail line, or None when at least one task is well powered. + + Scoped to the tasks that are actually shallow. An earlier version keyed off the + best-covered task and fell silent as soon as any one task was deep, which left a + mixed run's shallow tasks uncaveated -- the opposite of the intended failure. + + The threshold is a reporting heuristic, not a power calculation: 5/5 still carries a + Wilson interval of roughly [0.57, 1.00], so the line points readers at the aggregate + rather than promising that five repetitions settle anything. + """ + counts = [task.n for task in summary.tasks.values() if task.n] + if not counts: + return None + shallow = [count for count in counts if count < UNDERPOWERED_REPS] + if not shallow: + return None + return ( + f"POWER: {len(shallow)} of {len(counts)} task(s) below {UNDERPOWERED_REPS} repetitions " + f"(fewest {min(shallow)}) — their per-task pass rates and UNSTABLE flags are not verdicts " + f"at that depth; read the aggregate and paired deltas for those. Raising --reps narrows " + f"a per-task interval but no fixed count makes one conclusive." + ) + + +__all__ = ["UNDERPOWERED_REPS", "power_statement"] diff --git a/evals/report/schema_friction.py b/evals/report/schema_friction.py new file mode 100644 index 0000000..25ceefa --- /dev/null +++ b/evals/report/schema_friction.py @@ -0,0 +1,230 @@ +"""Success-conditioned MCP tool-error measurements for eval reports.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass + +from evals.core.error_class import NOT_FOUND, REFUSED, REJECTED, UNCLASSIFIED +from evals.core.results import CallRecord, TaskResult + +from .load import ResultRow, is_infra_error_row, is_meta_row, read_result +from .statistics import median + +SCHEMA_FRICTION_LIMITATION = ( + "limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; " + "an error that is the correct task outcome still contributes, while calling the wrong tool " + "successfully does not" +) + +FRICTION_SPLIT_LIMITATION = ( + "limitation: a first not_found is read as the answer to an existence question, since asking " + "has no cheaper form; only a repeat on the same tool and action is counted as friction. A " + "surface that misleads an agent into one wrong lookup is therefore not charged for it" +) + + +def split_errors(calls: list[CallRecord]) -> dict[str, int]: + """Count a row's errored calls by what kind of "no" they received. + + ``not_found`` is split in two. The first absent read of a given tool and action + is the answer to an existence question -- there is no cheaper way to ask -- and + lands in ``answered``. A second identical one means the first was not understood, + so it joins ``surface``. + """ + counts = dict.fromkeys(("navigation", "surface", "answered", "other", "unclassified", "unflagged"), 0) + seen_absent: set[tuple[str, str]] = set() + for call in calls: + if not call.is_error and call.error_class is None: + continue + if not call.is_error: + # A refusal the server reported as a successful result. It is counted here + # and named separately, because it is absent from `errored_calls` -- the + # protocol-flag total the rest of the report and every earlier run use. + counts["unflagged"] += 1 + kind = call.error_class or UNCLASSIFIED + if kind == REFUSED: + counts["navigation"] += 1 + elif kind == REJECTED: + counts["surface"] += 1 + elif kind == NOT_FOUND: + key = (call.tool, call.action or "") + if key in seen_absent: + counts["surface"] += 1 + else: + seen_absent.add(key) + counts["answered"] += 1 + elif kind == UNCLASSIFIED: + # Kept apart from `other`, which holds errors we did classify and chose + # not to charge to tool design. A row written before this field existed + # lands here in full, and a split reading zero surface friction because + # nothing was classified must not read as a surface with no friction. + counts["unclassified"] += 1 + else: + # denied/failed: real, but not attributable to tool design. + counts["other"] += 1 + return counts + + +@dataclass(frozen=True, slots=True) +class TaskSchemaFriction: + """Absolute and attempt-normalized errors for one task's eligible rows.""" + + task_id: str + repetitions: int + errored_calls: int + total_calls: int + median_errored_calls: float + errored_call_rate: float | None + navigation_calls: int = 0 + surface_calls: int = 0 + answered_calls: int = 0 + other_calls: int = 0 + unclassified_calls: int = 0 + unflagged_refusals: int = 0 + + @property + def address(self) -> str: + rate = f"{self.errored_call_rate:.1%}" if self.errored_call_rate is not None else "n/a; zero attempts" + return f"{self.task_id}={self.errored_calls}/{self.total_calls} ({rate})" + + +@dataclass(frozen=True, slots=True) +class SchemaFrictionMeasurement: + """Task-cluster aggregate over successful, trace-intact result rows.""" + + tasks: dict[str, TaskSchemaFriction] + task_mean_errored_calls: float | None + task_mean_errored_call_rate: float | None + rate_task_count: int + navigation_calls: int = 0 + surface_calls: int = 0 + answered_calls: int = 0 + other_calls: int = 0 + unclassified_calls: int = 0 + unflagged_refusals: int = 0 + total_calls: int = 0 + + @property + def task_count(self) -> int: + return len(self.tasks) + + @property + def errored_task_ids(self) -> tuple[str, ...]: + return tuple(task_id for task_id, task in self.tasks.items() if task.errored_calls) + + +def successful_trace_rows(rows: list[ResultRow]) -> list[TaskResult]: + """Return the exact row population used for successful call deltas.""" + eligible: list[TaskResult] = [] + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row) or is_infra_error_row(row) or row.error or row.skipped or not row.trace_integrity: + continue + if row.success: + eligible.append(row) + return eligible + + +def measure_schema_friction(rows: list[ResultRow]) -> SchemaFrictionMeasurement: + """Measure absolute errors and error rate with tasks as sampling units.""" + by_task: dict[str, list[TaskResult]] = defaultdict(list) + for row in successful_trace_rows(rows): + by_task[row.task_id].append(row) + + tasks: dict[str, TaskSchemaFriction] = {} + for task_id in sorted(by_task): + task_rows = by_task[task_id] + errored_calls = sum(row.errored_calls for row in task_rows) + total_calls = sum(row.num_calls for row in task_rows) + split = {key: 0 for key in ("navigation", "surface", "answered", "other", "unclassified", "unflagged")} + for row in task_rows: + for key, value in split_errors(row.calls).items(): + split[key] += value + tasks[task_id] = TaskSchemaFriction( + task_id=task_id, + repetitions=len(task_rows), + errored_calls=errored_calls, + total_calls=total_calls, + median_errored_calls=float(median([float(row.errored_calls) for row in task_rows]) or 0.0), + errored_call_rate=(errored_calls / total_calls if total_calls else None), + navigation_calls=split["navigation"], + surface_calls=split["surface"], + answered_calls=split["answered"], + other_calls=split["other"], + unclassified_calls=split["unclassified"], + unflagged_refusals=split["unflagged"], + ) + + absolute_values = [task.median_errored_calls for task in tasks.values()] + rate_values = [task.errored_call_rate for task in tasks.values() if task.errored_call_rate is not None] + return SchemaFrictionMeasurement( + tasks=tasks, + task_mean_errored_calls=(sum(absolute_values) / len(absolute_values) if absolute_values else None), + task_mean_errored_call_rate=(sum(rate_values) / len(rate_values) if rate_values else None), + rate_task_count=len(rate_values), + navigation_calls=sum(task.navigation_calls for task in tasks.values()), + surface_calls=sum(task.surface_calls for task in tasks.values()), + answered_calls=sum(task.answered_calls for task in tasks.values()), + other_calls=sum(task.other_calls for task in tasks.values()), + unclassified_calls=sum(task.unclassified_calls for task in tasks.values()), + unflagged_refusals=sum(task.unflagged_refusals for task in tasks.values()), + total_calls=sum(task.total_calls for task in tasks.values()), + ) + + +def _split_lines(measurement: SchemaFrictionMeasurement) -> tuple[str, ...]: + """The three numbers the single rate used to conflate.""" + total = measurement.total_calls + + def share(count: int) -> str: + return f"{count}" + (f" ({count / total:.1%})" if total else "") + + surface = measurement.surface_calls + unclassified = measurement.unclassified_calls + lines = [ + f" by kind, of {total} calls: " + f"surface friction={share(surface)}, " + f"navigation={share(measurement.navigation_calls)}, " + f"answered existence questions={share(measurement.answered_calls)}, " + f"other={share(measurement.other_calls)}, " + f"unclassified={share(unclassified)}", + ] + if measurement.unflagged_refusals: + lines.append( + f" {measurement.unflagged_refusals} refusal(s) arrived flagged as successful results, so they " + "are counted above but not in the errored-call total" + ) + if unclassified: + # Never let "no surface friction" stand in for "nothing was classified". + lines.append( + f" split incomplete: {unclassified} errored call(s) carry no class — a run recorded " + "before error classes existed, or payloads the classifier does not recognise" + ) + else: + lines.append( + " surface friction is the number to act on: a well-formed call the API refused on meaning" + + ("" if surface else " — none in this run") + ) + return tuple(lines) + + +def schema_friction_statement(measurement: SchemaFrictionMeasurement) -> str: + """Render explicit zeros, task addresses, and the measurement boundary.""" + absolute = measurement.task_mean_errored_calls + absolute_text = f"{absolute:.1f}" if absolute is not None else "n/a" + rate = measurement.task_mean_errored_call_rate + rate_text = f"{rate:.1%}" if rate is not None else "n/a" + flagged = [task.address for task in measurement.tasks.values() if task.errored_calls] + flagged_text = f" [{', '.join(flagged)}]" if flagged else " []" + return "\n".join( + ( + "schema friction (same successful, trace-intact rows as call deltas): " + f"task-mean median errored calls={absolute_text} across {measurement.task_count} tasks; " + f"task-mean errored-call rate={rate_text} across {measurement.rate_task_count} tasks with calls", + f" errored-call tasks: {len(flagged)}/{measurement.task_count}{flagged_text}", + *_split_lines(measurement), + f" {SCHEMA_FRICTION_LIMITATION}", + f" {FRICTION_SPLIT_LIMITATION}", + ) + ) diff --git a/evals/report/statistics.py b/evals/report/statistics.py new file mode 100644 index 0000000..d511c40 --- /dev/null +++ b/evals/report/statistics.py @@ -0,0 +1,157 @@ +"""Statistical calculations for evaluation reports.""" + +from __future__ import annotations + +import itertools +import math +import random + +EXACT_PERMUTATION_LIMIT = 20 +MONTE_CARLO_PERMUTATIONS = 100_000 +BOOTSTRAP_RESAMPLES = 20_000 + + +def wilson_interval(k: int, n: int, z: float = 1.96) -> tuple[float, float]: + """95% Wilson score interval for a binomial proportion.""" + if n <= 0: + return (0.0, 0.0) + p = k / n + z2 = z * z + denom = 1.0 + z2 / n + centre = p + z2 / (2.0 * n) + margin = z * math.sqrt((p * (1.0 - p) + z2 / (4.0 * n)) / n) + lo = max(0.0, (centre - margin) / denom) + hi = min(1.0, (centre + margin) / denom) + return (lo, hi) + + +def paired_permutation_pvalue( + deltas: list[float], + *, + exact_limit: int = EXACT_PERMUTATION_LIMIT, + permutations: int = MONTE_CARLO_PERMUTATIONS, + seed: int = 0, +) -> float | None: + """Two-sided paired sign-flip permutation test on the mean delta. + + The null assumes each pair's A/B labels are exchangeable and pairs are + independent. The statistic is the absolute mean of *all* paired deltas, so + zero-delta ties remain in the sample and its denominator. A zero contributes + the same value under either sign; enumerating its duplicate sign assignments + once is exactly equivalent to enumerating both. + + Tests with at most ``exact_limit`` non-zero contributions enumerate the exact + randomization distribution. Larger tests use a deterministic Monte Carlo + sample and the standard plus-one correction. ``None`` means there were no + pairs; an all-tie sample returns 1.0. + """ + if not deltas: + return None + pair_count = len(deltas) + contributions = [float(delta) for delta in deltas if delta != 0] + observed = abs(sum(deltas) / pair_count) + tolerance = 1e-12 + if not contributions: + return 1.0 + + def is_extreme(signs: tuple[int, ...] | list[int]) -> bool: + permuted = abs(sum(sign * delta for sign, delta in zip(signs, contributions, strict=True)) / pair_count) + return permuted + tolerance >= observed + + if len(contributions) <= exact_limit: + assignments = itertools.product((-1, 1), repeat=len(contributions)) + extreme = sum(1 for signs in assignments if is_extreme(signs)) + return extreme / (2 ** len(contributions)) + + if permutations <= 0: + raise ValueError("permutations must be positive") + generator = random.Random(seed) + extreme = 0 + for _ in range(permutations): + signs = [generator.choice((-1, 1)) for _ in contributions] + extreme += is_extreme(signs) + return (extreme + 1) / (permutations + 1) + + +def paired_bootstrap_mean_ci( + deltas: list[float], + *, + confidence: float = 0.95, + resamples: int = BOOTSTRAP_RESAMPLES, + seed: int = 0, +) -> tuple[float | None, float | None]: + """Percentile paired-bootstrap CI for the mean per-pair delta. + + Resampling whole paired deltas preserves the A/B pairing. The interval treats + tasks as independent sampling units drawn from a task population and assumes + the two labels measured comparable task instances. It captures task-sampling + uncertainty, not dependence between tasks or systematic run/environment drift. + + Small samples are intentionally not narrowed by row-level repetitions: for up + to five pairs the complete ``n**n`` bootstrap distribution is enumerated. + Larger samples use a deterministic Monte Carlo bootstrap. + """ + if not deltas: + return (None, None) + if not 0.0 < confidence < 1.0: + raise ValueError("confidence must be between 0 and 1") + if resamples <= 0: + raise ValueError("resamples must be positive") + + values = [float(delta) for delta in deltas] + sample_size = len(values) + bootstrap_means: list[float] = [] + if sample_size**sample_size <= resamples: + for sample in itertools.product(values, repeat=sample_size): + bootstrap_means.append(sum(sample) / sample_size) + else: + generator = random.Random(seed) + for _ in range(resamples): + bootstrap_means.append(sum(generator.choice(values) for _ in range(sample_size)) / sample_size) + + tail = (1.0 - confidence) / 2.0 + return (percentile(bootstrap_means, tail), percentile(bootstrap_means, 1.0 - tail)) + + +def cluster_bootstrap_mean_ci( + task_rates: list[float], + *, + confidence: float = 0.95, + resamples: int = BOOTSTRAP_RESAMPLES, + seed: int = 0, +) -> tuple[float | None, float | None]: + """Bootstrap the mean success rate by resampling whole task clusters.""" + return paired_bootstrap_mean_ci( + task_rates, + confidence=confidence, + resamples=resamples, + seed=seed, + ) + + +def median(values: list[float]) -> float | None: + if not values: + return None + sorted_values = sorted(values) + middle = len(sorted_values) // 2 + if len(sorted_values) % 2: + return float(sorted_values[middle]) + return (sorted_values[middle - 1] + sorted_values[middle]) / 2.0 + + +def percentile(values: list[float], proportion: float) -> float | None: + if not values: + return None + sorted_values = sorted(values) + if len(sorted_values) == 1: + return float(sorted_values[0]) + position = (len(sorted_values) - 1) * proportion + floor = math.floor(position) + ceiling = math.ceil(position) + if floor == ceiling: + return float(sorted_values[int(position)]) + return float(sorted_values[floor] + (sorted_values[ceiling] - sorted_values[floor]) * (position - floor)) + + +def iqr(values: list[float]) -> tuple[float | None, float | None, float | None]: + return (percentile(values, 0.25), median(values), percentile(values, 0.75)) diff --git a/evals/report/summary.py b/evals/report/summary.py new file mode 100644 index 0000000..ce5e96b --- /dev/null +++ b/evals/report/summary.py @@ -0,0 +1,397 @@ +"""Aggregate evaluation results and measure observed task instability.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import Literal + +from evals.core.results import TRACE_INTEGRITY_SCHEMA_VERSION, TaskResult +from evals.core.task_metadata import TaskMetadata, entry_needs, task_metadata_from_rows +from evals.skip_taxonomy import is_expected_environment_capability_skip, skip_reason_family + +from .economics import EconomicsMeasurement, measure_economics +from .failure_kinds import FailureKindMeasurement, measure_failure_kinds +from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .lookup_reuse import LookupReuseMeasurement, measure_lookup_reuse +from .off_surface import OffSurfaceMeasurement, measure_off_surface +from .schema_friction import SchemaFrictionMeasurement, measure_schema_friction +from .statistics import cluster_bootstrap_mean_ci, iqr, median, percentile, wilson_interval + +ResultTokensMode = Literal["measured", "estimated", "mixed", "unlabeled", "unavailable"] + + +@dataclass(slots=True) +class TaskSummary: + task_id: str + n: int + k: int + wilson_lo: float + wilson_hi: float + med_calls: float | None + calls_min: float | None + calls_max: float | None + calls_q1: float | None + calls_q3: float | None + tool_reps: int + failed_tool_reps: int + tool_rep_frequency: dict[str, float] + tool_call_counts: dict[str, int] + errored_calls: int + capped: int + harness_err: int + infra_err: int + med_result_tokens: float | None + p95_result_tokens: float | None + result_tokens_mode: ResultTokensMode + med_cum_input: float | None + + @property + def unstable(self) -> bool: + """True when repetitions of this task disagreed on pass/fail.""" + return self.n > 1 and 0 < self.k < self.n + + @property + def success(self) -> str: + return f"{self.k}/{self.n}" if self.n else "0/0" + + @property + def tool_distribution_available(self) -> bool: + return self.tool_reps >= 2 + + @property + def variable_tool_names(self) -> list[str]: + return [tool for tool, frequency in self.tool_rep_frequency.items() if frequency < 1.0] + + +@dataclass(slots=True) +class Summary: + tasks: dict[str, TaskSummary] + total_tasks: int + expected_rows: int + completed_rows: int + infra_errors: int + harness_errors: int + expected_skips: int + unexpected_skips: int + cleanup_errors: int + trace_invalid_rows: int + expected_skip_reasons: dict[str, int] + unexpected_skip_reasons: dict[str, int] + skipped_task_reasons: dict[str, list[str]] + aggregate_k: int + aggregate_n: int + aggregate_wilson_lo: float + aggregate_wilson_hi: float + task_mean_success: float | None + task_cluster_lo: float | None + task_cluster_hi: float | None + missing_run_keys: tuple[str, ...] + unexpected_run_keys: tuple[str, ...] + multi_rep: bool + result_tokens_mode: ResultTokensMode + off_surface: OffSurfaceMeasurement + schema_friction: SchemaFrictionMeasurement + economics: EconomicsMeasurement + failure_kinds: FailureKindMeasurement + lookup_reuse: LookupReuseMeasurement + + @property + def complete(self) -> bool: + return ( + self.completed_rows == self.expected_rows + and not self.missing_run_keys + and not self.unexpected_run_keys + and self.infra_errors == 0 + and self.harness_errors == 0 + and self.unexpected_skips == 0 + and self.cleanup_errors == 0 + and self.trace_invalid_rows == 0 + ) + + @property + def unstable_task_ids(self) -> list[str]: + return [task_id for task_id, task in self.tasks.items() if task.unstable] + + @property + def unstable_tasks(self) -> int: + return len(self.unstable_task_ids) + + @property + def variable_tool_tasks(self) -> int: + return sum(bool(task.variable_tool_names) for task in self.tasks.values()) + + @property + def tool_distribution_available(self) -> bool: + return any(task.tool_distribution_available for task in self.tasks.values()) + + +def result_tokens_mode(rows: list[ResultRow]) -> ResultTokensMode: + """Classify token counts without treating unmarked legacy data as measured.""" + labels: set[str] = set() + for raw_row in rows: + row = read_result(raw_row) + if not row.trace_integrity: + continue + for call in row.calls: + if call.result_tokens is None: + continue + estimated = ( + call.result_tokens_estimated + if call.result_tokens_estimated is not None + else row.result_tokens_estimated + ) + if estimated is True: + labels.add("estimated") + elif estimated is False: + labels.add("measured") + else: + labels.add("unlabeled") + if not labels: + return "unavailable" + if labels == {"estimated"}: + return "estimated" + if labels == {"measured"}: + return "measured" + if "unlabeled" in labels: + return "unlabeled" + return "mixed" + + +def _format_reason_counts(reasons: dict[str, int]) -> str: + return ", ".join(f"{reason}={count}" for reason, count in sorted(reasons.items())) + + +def completeness_statement(summary: Summary) -> str: + """Render completeness independently from the model success rate.""" + prefix = "RUN COMPLETE" if summary.complete else "RUN INCOMPLETE" + parts = [f"{summary.completed_rows}/{summary.expected_rows} rows completed"] + if summary.infra_errors: + parts.append(f"infra errors={summary.infra_errors}") + if summary.harness_errors: + parts.append(f"harness errors={summary.harness_errors}") + if summary.unexpected_skips: + reasons = _format_reason_counts(summary.unexpected_skip_reasons) + parts.append(f"unexpected skips={summary.unexpected_skips} [{reasons}]") + if summary.cleanup_errors: + parts.append(f"cleanup errors={summary.cleanup_errors}") + if summary.trace_invalid_rows: + parts.append(f"trace-invalid rows={summary.trace_invalid_rows}") + if summary.missing_run_keys: + parts.append(f"missing keys=[{', '.join(summary.missing_run_keys)}]") + if summary.unexpected_run_keys: + parts.append(f"unexpected keys=[{', '.join(summary.unexpected_run_keys)}]") + if summary.expected_skips: + reasons = _format_reason_counts(summary.expected_skip_reasons) + parts.append(f"expected skips={summary.expected_skips} [{reasons}]") + return f"{prefix}: " + "; ".join(parts) + + +def execution_coverage_statement(summary: Summary) -> str: + """Render rows actually evaluated, independently from run completeness.""" + if summary.expected_rows: + rate = summary.aggregate_n / summary.expected_rows + amount = f"{summary.aggregate_n}/{summary.expected_rows} rows evaluated ({rate:.1%})" + else: + amount = f"{summary.aggregate_n}/0 rows evaluated (n/a)" + parts = [amount] + if summary.skipped_task_reasons: + skips = "; ".join( + f"{','.join(task_ids)} ({reason})" for reason, task_ids in sorted(summary.skipped_task_reasons.items()) + ) + parts.append(f"skipped tasks=[{skips}]") + return "EXECUTION COVERAGE: " + "; ".join(parts) + + +def summarize( + rows: list[ResultRow], + *, + expected_rows: int | None = None, + run_keys: RunKeyValidation | None = None, + task_metadata: TaskMetadata | None = None, +) -> Summary: + """Aggregate per-task metrics. + + Rows with ``error_class`` starting ``infra_`` are excluded from success-rate + denominators and counted separately as ``infra_errors``. Other non-null + ``error`` rows remain harness errors (excluded from success, counted in + ``harness_err``). + """ + # The run's own task facts, so skip expectations and mutation intent describe what ran + # rather than what the working tree happens to say now. A caller that already filtered + # the meta header out of ``rows`` passes it explicitly. + if task_metadata is None: + task_metadata = task_metadata_from_rows(rows) + by_task: dict[str, list[TaskResult]] = defaultdict(list) + harness_errors_by_task: dict[str, int] = defaultdict(int) + infrastructure_errors_by_task: dict[str, int] = defaultdict(int) + repetitions_by_task: dict[str, set[int]] = defaultdict(set) + infrastructure_errors = 0 + harness_errors = 0 + completed_rows = 0 + expected_skips = 0 + unexpected_skips = 0 + cleanup_errors = 0 + trace_invalid_rows = 0 + expected_skip_reasons: dict[str, int] = defaultdict(int) + unexpected_skip_reasons: dict[str, int] = defaultdict(int) + skipped_task_reasons: dict[str, set[str]] = defaultdict(set) + declared_expected_rows = 0 + for raw_row in rows: + row = read_result(raw_row) + if is_meta_row(row): + continue + declared_expected_rows = max(declared_expected_rows, row.expected_rows) + task_id = row.task_id + repetitions_by_task[task_id].add(row.rep) + if row.trace_integrity is False or ( + row.trace_integrity is None and row.schema_version >= TRACE_INTEGRITY_SCHEMA_VERSION + ): + trace_invalid_rows += 1 + if row.cleanup_error: + cleanup_errors += 1 + if is_infra_error_row(row): + infrastructure_errors += 1 + infrastructure_errors_by_task[task_id] += 1 + continue # infra seed/cli — excluded from success aggregates + if row.error: + harness_errors += 1 + harness_errors_by_task[task_id] += 1 + continue # harness/API errors excluded from success/medians (F4) + if row.skipped: + family = skip_reason_family(row.skipped) + skipped_task_reasons[row.skipped].add(task_id) + if is_expected_environment_capability_skip( + row.skipped, + task_id=task_id, + task_needs=entry_needs(task_metadata.get(task_id)) if task_metadata else None, + ): + expected_skips += 1 + expected_skip_reasons[family] += 1 + completed_rows += 1 + else: + unexpected_skips += 1 + unexpected_skip_reasons[family] += 1 + continue # skipped rows are excluded from success denominators + completed_rows += 1 + by_task[task_id].append(row) + + # Include tasks that only had harness/infra errors so columns stay visible. + all_task_ids = sorted(set(by_task) | set(harness_errors_by_task) | set(infrastructure_errors_by_task)) + + output: dict[str, TaskSummary] = {} + total_passes = 0 + total_repetitions = 0 + for task_id in all_task_ids: + task_results = by_task.get(task_id, []) + repetition_count = len(task_results) + pass_count = sum(1 for row in task_results if row.success) + total_passes += pass_count + total_repetitions += repetition_count + lower, upper = wilson_interval(pass_count, repetition_count) if repetition_count else (0.0, 0.0) + successful_calls = [float(row.num_calls) for row in task_results if row.success and row.trace_integrity] + first_quartile, median_calls, third_quartile = iqr(successful_calls) + minimum_calls = min(successful_calls) if successful_calls else None + maximum_calls = max(successful_calls) if successful_calls else None + successful_results = [row for row in task_results if row.success and row.trace_integrity] + tool_reps = len(successful_results) + failed_tool_reps = ( + repetition_count + - tool_reps + + harness_errors_by_task.get(task_id, 0) + + infrastructure_errors_by_task.get(task_id, 0) + ) + tool_rep_counts: dict[str, int] = defaultdict(int) + tool_call_counts: dict[str, int] = defaultdict(int) + for row in successful_results: + tools_in_rep: set[str] = set() + for call in row.calls: + if not call.tool: + continue + tool_call_counts[call.tool] += 1 + tools_in_rep.add(call.tool) + for tool in tools_in_rep: + tool_rep_counts[tool] += 1 + tool_rep_frequency = ( + {tool: tool_rep_counts[tool] / tool_reps for tool in sorted(tool_rep_counts)} if tool_reps >= 2 else {} + ) + errored_calls = 0 + result_tokens: list[float] = [] + for row in task_results: + if not row.trace_integrity: + continue + for call in row.calls: + if call.is_error: + errored_calls += 1 + if call.result_tokens is not None: + result_tokens.append(float(call.result_tokens)) + capped = sum(1 for row in task_results if row.hit_max_iterations or row.stop_reason == "max_tokens") + cumulative_inputs = [float(row.cum_input_tokens or 0) for row in task_results] + output[task_id] = TaskSummary( + task_id=task_id, + n=repetition_count, + k=pass_count, + wilson_lo=lower, + wilson_hi=upper, + med_calls=median_calls, + calls_min=minimum_calls, + calls_max=maximum_calls, + calls_q1=first_quartile, + calls_q3=third_quartile, + tool_reps=tool_reps, + failed_tool_reps=failed_tool_reps, + tool_rep_frequency=tool_rep_frequency, + tool_call_counts={tool: tool_call_counts[tool] for tool in sorted(tool_call_counts)}, + errored_calls=errored_calls, + capped=capped, + harness_err=harness_errors_by_task.get(task_id, 0), + infra_err=infrastructure_errors_by_task.get(task_id, 0), + med_result_tokens=median(result_tokens), + p95_result_tokens=percentile(result_tokens, 0.95), + result_tokens_mode=result_tokens_mode(task_results), + med_cum_input=median(cumulative_inputs), + ) + aggregate_lower, aggregate_upper = ( + wilson_interval(total_passes, total_repetitions) if total_repetitions else (0.0, 0.0) + ) + task_rates = [task.k / task.n for task in output.values() if task.n] + task_mean_success = sum(task_rates) / len(task_rates) if task_rates else None + task_cluster_lower, task_cluster_upper = cluster_bootstrap_mean_ci(task_rates) + resolved_expected_rows = ( + run_keys.expectation.expected_rows + if run_keys is not None + else max(expected_rows, declared_expected_rows) + if expected_rows is not None + else declared_expected_rows or sum(1 for row in rows if not is_meta_row(row)) + ) + return Summary( + tasks=output, + total_tasks=len(repetitions_by_task), + expected_rows=resolved_expected_rows, + completed_rows=completed_rows, + infra_errors=infrastructure_errors, + harness_errors=harness_errors, + expected_skips=expected_skips, + unexpected_skips=unexpected_skips, + cleanup_errors=cleanup_errors, + trace_invalid_rows=trace_invalid_rows, + expected_skip_reasons=dict(expected_skip_reasons), + unexpected_skip_reasons=dict(unexpected_skip_reasons), + skipped_task_reasons={reason: sorted(task_ids) for reason, task_ids in sorted(skipped_task_reasons.items())}, + aggregate_k=total_passes, + aggregate_n=total_repetitions, + aggregate_wilson_lo=aggregate_lower, + aggregate_wilson_hi=aggregate_upper, + task_mean_success=task_mean_success, + task_cluster_lo=task_cluster_lower, + task_cluster_hi=task_cluster_upper, + missing_run_keys=run_keys.missing if run_keys is not None else (), + unexpected_run_keys=run_keys.unexpected if run_keys is not None else (), + multi_rep=any(len(repetitions) > 1 for repetitions in repetitions_by_task.values()), + result_tokens_mode=result_tokens_mode([row for task_results in by_task.values() for row in task_results]), + off_surface=measure_off_surface(rows, task_catalog=task_metadata or None), + schema_friction=measure_schema_friction(rows), + economics=measure_economics(rows), + failure_kinds=measure_failure_kinds(rows), + lookup_reuse=measure_lookup_reuse(rows), + ) diff --git a/evals/report/table.py b/evals/report/table.py new file mode 100644 index 0000000..947a5ba --- /dev/null +++ b/evals/report/table.py @@ -0,0 +1,450 @@ +"""Plain-text and Markdown tables for evaluation reports.""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path +from typing import Any + +from evals.core.results import TaskResult +from evals.core.task_metadata import TaskMetadata, entry_prompt, task_metadata_from_rows + +from .economics import economics_statement +from .failure_kinds import failure_kind_statement +from .load import ResultRow, RunKeyValidation, is_infra_error_row, is_meta_row, read_result +from .off_surface import off_surface_statement +from .power import power_statement +from .schema_friction import schema_friction_statement +from .statistics import wilson_interval +from .summary import ( + Summary, + TaskSummary, + completeness_statement, + execution_coverage_statement, + summarize, +) + + +def format_number(value: float | None, digits: int = 1) -> str: + if value is None: + return "-" + return f"{value:.{digits}f}" + + +def result_tokens_marker(mode: str) -> str: + return {"estimated": "~", "mixed": "*", "unlabeled": "?"}.get(mode, "") + + +def format_result_tokens(value: float | None, mode: str) -> str: + formatted = format_number(value, 0) + if formatted == "-": + return formatted + return f"{result_tokens_marker(mode)}{formatted}" + + +def format_tool_distribution(task: TaskSummary) -> str: + """Render success-conditioned tool frequency with its exclusions visible.""" + conditioning = f"success-only n={task.tool_reps}; failed excluded={task.failed_tool_reps}" + if not task.tool_distribution_available: + return f"{conditioning}; frequency=—" + if not task.tool_rep_frequency: + return f"{conditioning}; no tools" + core = [ + f"{tool}({task.tool_call_counts[tool]}c)" + for tool, frequency in task.tool_rep_frequency.items() + if frequency == 1.0 + ] + variable = [ + f"{tool}={frequency:.0%}({task.tool_call_counts[tool]}c)" + for tool, frequency in task.tool_rep_frequency.items() + if frequency < 1.0 + ] + groups = [] + if core: + groups.append(f"core:{','.join(core)}") + if variable: + groups.append(f"variable:{','.join(variable)}") + return f"{conditioning}; {'; '.join(groups)}" + + +def format_tool_variability(summary: Summary, total_tasks: int | None = None) -> str: + """Render the fleet count of tasks with variable tool use.""" + if not summary.tool_distribution_available: + return "—" + total = summary.total_tasks if total_tasks is None else total_tasks + return f"{summary.variable_tool_tasks}/{total} tasks" + + +def print_table(summary: Summary, title: str) -> None: + print(title) + token_mode = summary.result_tokens_mode + if token_mode == "estimated": + print("result-token columns marked ~: entirely estimated from result characters") + elif token_mode == "mixed": + print("result-token columns marked *: mixed measured and estimated values (~ marks estimated tasks)") + elif token_mode == "unlabeled": + print("result-token columns marked ?: include legacy values with unknown measurement status") + aggregate_count = summary.aggregate_n + if aggregate_count and summary.task_mean_success is not None: + task_count = sum(task.n > 0 for task in summary.tasks.values()) + print( + f"task-cluster success: {summary.task_mean_success:.1%} across {task_count} tasks " + f"cluster-bootstrap95 [{summary.task_cluster_lo:.2f},{summary.task_cluster_hi:.2f}]" + ) + pooled_rate = summary.aggregate_k / aggregate_count + print( + f"pooled repetition success: {summary.aggregate_k}/{aggregate_count} ({pooled_rate:.1%}) " + f"Wilson95 [{summary.aggregate_wilson_lo:.2f},{summary.aggregate_wilson_hi:.2f}]" + ) + else: + print("task-cluster success: n/a (no evaluated tasks)") + print("pooled repetition success: 0/0 (n/a; no evaluated rows)") + print(execution_coverage_statement(summary)) + print(off_surface_statement(summary.off_surface)) + print(schema_friction_statement(summary.schema_friction)) + power = power_statement(summary) + if power: + print(power) + print(failure_kind_statement(summary.failure_kinds)) + print(summary.lookup_reuse.statement()) + print(economics_statement(summary.economics)) + print(completeness_statement(summary)) + if summary.infra_errors: + print(f"infra errors: {summary.infra_errors}") + print(f"tool variability: {format_tool_variability(summary)}") + multiple_repetitions = summary.multi_rep + # Multi-rep files keep the repetition-aware layout even when errors leave + # only one completed result in every task's success-rate denominator. + show_variation = multiple_repetitions or any(task.n > 1 for task in summary.tasks.values()) + token_marker = result_tokens_marker(token_mode) + median_result_tokens_header = f"med_rtok{token_marker}" + percentile_result_tokens_header = f"p95_rtok{token_marker}" + if show_variation: + unstable_header = f"{'unstable':>8} " if multiple_repetitions else "" + header = ( + f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{unstable_header}" + f"{'success_calls_min':>17} {'success_calls_med':>17} " + f"{'success_calls_max':>17} {'success_calls_q1-q3':>19} {'err':>4} " + f"{'capped':>6} {'h_err':>5} {'i_err':>5} " + f"{median_result_tokens_header:>9} {percentile_result_tokens_header:>9} " + f"{'med_cum_in':>10} tool distribution" + ) + else: + header = ( + f"{'task':<6} {'n':>3} {'success':>8} {'wilson95':>16} " + f"{'success_calls_med':>17} {'success_calls_min':>17} " + f"{'success_calls_q1-q3':>19} {'err':>4} " + f"{'capped':>6} {'h_err':>5} {'i_err':>5} " + f"{median_result_tokens_header:>9} {percentile_result_tokens_header:>9} " + f"{'med_cum_in':>10} tool distribution" + ) + print(header) + print("-" * len(header)) + for task_id, values in summary.tasks.items(): + wilson = f"[{values.wilson_lo:.2f},{values.wilson_hi:.2f}]" + quartiles = f"{format_number(values.calls_q1)}-{format_number(values.calls_q3)}" + task_token_mode = values.result_tokens_mode + if show_variation: + unstable = ("YES" if values.unstable else "no") if multiple_repetitions else "" + unstable_cell = f"{unstable:>8} " if multiple_repetitions else "" + print( + f"{task_id:<6} {values.n:>3} {values.success:>8} {wilson:>16} " + f"{unstable_cell}" + f"{format_number(values.calls_min):>17} " + f"{format_number(values.med_calls):>17} " + f"{format_number(values.calls_max):>17} " + f"{quartiles:>19} {values.errored_calls:>4} {values.capped:>6} " + f"{values.harness_err:>5} {values.infra_err:>5} " + f"{format_result_tokens(values.med_result_tokens, task_token_mode):>9} " + f"{format_result_tokens(values.p95_result_tokens, task_token_mode):>9} " + f"{format_number(values.med_cum_input, 0):>10} {format_tool_distribution(values)}" + ) + else: + print( + f"{task_id:<6} {values.n:>3} {values.success:>8} {wilson:>16} " + f"{format_number(values.med_calls):>17} {format_number(values.calls_min):>17} " + f"{quartiles:>19} {values.errored_calls:>4} {values.capped:>6} " + f"{values.harness_err:>5} {values.infra_err:>5} " + f"{format_result_tokens(values.med_result_tokens, task_token_mode):>9} " + f"{format_result_tokens(values.p95_result_tokens, task_token_mode):>9} " + f"{format_number(values.med_cum_input, 0):>10} {format_tool_distribution(values)}" + ) + + +def task_sort_key(task_id: str) -> tuple[str, int]: + digits = "".join(character for character in task_id if character.isdigit()) + return (task_id[0] if task_id else "", int(digits) if digits else 0) + + +def format_surface_cell(row: ResultRow | None) -> str: + """Cell for a single-repetition surface result.""" + if row is None: + return "—" + result = read_result(row) + if result.skipped: + return "skip" + if result.error or is_infra_error_row(result): + return "ERR" + passed = "✅" if result.success else "❌" + call_count = str(result.num_calls) + return f"{passed} {call_count}c · tools —" + + +def format_multi_rep_surface_cell(rows: list[ResultRow]) -> str: + """Aggregate distinct repetitions into one task/surface cell.""" + results = [read_result(row) for row in rows] + completed = [row for row in results if not is_infra_error_row(row) and not row.error and not row.skipped] + if completed: + repetition_count = len(completed) + pass_count = sum(1 for row in completed if row.success) + lower, upper = wilson_interval(pass_count, repetition_count) + if 0 < pass_count < repetition_count: + marker = "⚠ UNSTABLE" + elif pass_count == repetition_count: + marker = "✅" + else: + marker = "❌" + calls = [row.num_calls for row in completed] + call_span = f"{min(calls)}c" if min(calls) == max(calls) else f"{min(calls)}-{max(calls)}c" + task_summary = summarize(results).tasks[results[0].task_id] + tools = format_tool_distribution(task_summary) + return f"{marker} {pass_count}/{repetition_count} [{lower:.2f},{upper:.2f}] {call_span} · tools {tools}" + if any(row.error or is_infra_error_row(row) for row in results): + return "ERR" + if any(row.skipped for row in results): + return "skip" + return "—" + + +def build_multi_surface_table( + file_rows: list[tuple[str, list[ResultRow]]], + *, + expected_rows_by_column: dict[str, int | None] | None = None, + run_keys_by_column: dict[str, RunKeyValidation | None] | None = None, +) -> dict[str, Any]: + """Build a per-task × per-surface grid from labeled row sets. + + ``file_rows`` is a list of ``(column_label, rows)``. Column labels default + to each file's dominant ``label`` field when the caller passes that label. + Rows are grouped by task and repetition. Single-rep columns retain the + historical one-cell rendering; multi-rep columns aggregate all repetitions. + """ + columns: list[str] = [] + rows_by_column: dict[str, dict[str, list[TaskResult]]] = {} + multiple_repetitions_by_column: dict[str, bool] = {} + # Prompt excerpts and mutation intent come from the runs being rendered, so collect the + # metadata every input file declared before the meta rows are filtered out below. + task_metadata: dict[str, dict[str, Any]] = {} + for _label, rows in file_rows: + task_metadata.update(task_metadata_from_rows(rows)) + for label, rows in file_rows: + columns.append(label) + column_rows: dict[str, list[TaskResult]] = defaultdict(list) + for raw_row in rows: + if is_meta_row(raw_row): + continue + row = read_result(raw_row) + column_rows[row.task_id].append(row) + rows_by_column[label] = dict(column_rows) + multiple_repetitions_by_column[label] = any( + len({row.rep for row in task_rows}) > 1 for task_rows in column_rows.values() + ) + + multiple_repetitions = any(multiple_repetitions_by_column.values()) + + all_tasks = sorted( + {task_id for column in rows_by_column.values() for task_id in column}, + key=task_sort_key, + ) + cells: dict[str, dict[str, str]] = {} + raw: dict[str, dict[str, list[TaskResult]]] = {} + for task_id in all_tasks: + cells[task_id] = {} + raw[task_id] = {} + for column in columns: + task_rows = rows_by_column[column].get(task_id, []) + raw[task_id][column] = task_rows + if multiple_repetitions: + cells[task_id][column] = format_multi_rep_surface_cell(task_rows) + else: + cells[task_id][column] = format_surface_cell(task_rows[-1] if task_rows else None) + + # Aggregate footer per column. + footer: dict[str, dict[str, Any]] = {} + for column in columns: + successes = repetitions = calls = 0 + infrastructure_errors = 0 + for task_rows in rows_by_column[column].values(): + for row in task_rows: + if is_infra_error_row(row): + infrastructure_errors += 1 + continue + if row.error: + continue + if row.skipped: + continue + repetitions += 1 + if row.success: + successes += 1 + calls += row.num_calls + column_summary = summarize( + [row for task_rows in rows_by_column[column].values() for row in task_rows], + expected_rows=(expected_rows_by_column or {}).get(column), + run_keys=(run_keys_by_column or {}).get(column), + task_metadata=task_metadata, + ) + footer[column] = { + "success": successes, + "n": repetitions, + "calls": calls, + "infra_errors": infrastructure_errors, + "multi_rep": multiple_repetitions_by_column[column], + "tool_variability": ( + column_summary.variable_tool_tasks if column_summary.tool_distribution_available else None + ), + "tasks": len(rows_by_column[column]), + "task_mean_success": column_summary.task_mean_success, + "task_cluster_lo": column_summary.task_cluster_lo, + "task_cluster_hi": column_summary.task_cluster_hi, + "complete": column_summary.complete, + "completeness": completeness_statement(column_summary), + "coverage": execution_coverage_statement(column_summary), + "off_surface": off_surface_statement(column_summary.off_surface), + "schema_friction": schema_friction_statement(column_summary.schema_friction), + } + return { + "columns": columns, + "task_ids": all_tasks, + "cells": cells, + "raw": raw, + "footer": footer, + "multi_rep": multiple_repetitions, + "multi_rep_by_col": multiple_repetitions_by_column, + "task_metadata": task_metadata, + } + + +def prompt_excerpt(task_id: str, task_metadata: TaskMetadata | None = None) -> str: + """Render a short prompt excerpt from the run's own metadata. + + Empty when the file predates the persisted header: an excerpt taken from the current + checkout can describe a prompt the run never used. + """ + prompt = entry_prompt((task_metadata or {}).get(task_id)).replace("{project}", "P") + return (prompt[:32] + "…") if len(prompt) > 32 else prompt + + +def render_multi_surface_table(table: dict[str, Any], *, markdown: bool = False) -> str: + """Render multi-surface table as plain text or GitHub markdown.""" + columns: list[str] = table["columns"] + task_ids: list[str] = table["task_ids"] + task_metadata: TaskMetadata = table.get("task_metadata") or {} + cells: dict[str, dict[str, str]] = table["cells"] + footer: dict[str, dict[str, Any]] = table["footer"] + multiple_repetitions = bool(table.get("multi_rep")) + lines: list[str] = [] + + if markdown: + header = "| task | what | " + " | ".join(columns) + " |" + separator = "| --- | --- | " + " | ".join("---" for _ in columns) + " |" + lines.append(header) + lines.append(separator) + for task_id in task_ids: + row_cells = " | ".join(cells[task_id].get(column, "—") for column in columns) + lines.append(f"| {task_id} | {prompt_excerpt(task_id, task_metadata)} | {row_cells} |") + # Footer + footer_parts = [] + for column in columns: + values = footer[column] + pooled = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + if values["task_mean_success"] is None: + rate = f"task-cluster n/a; pooled {pooled}" + else: + rate = ( + f"task-cluster {values['task_mean_success']:.1%} " + f"[{values['task_cluster_lo']:.2f},{values['task_cluster_hi']:.2f}]; pooled {pooled}" + ) + variability = ( + f"{values['tool_variability']}/{values['tasks']} variable" + if values["tool_variability"] is not None + else "tools —" + ) + footer_parts.append(f"{rate} ({values['calls']}c, {variability}, i={values['infra_errors']})") + lines.append("| **agg** | | " + " | ".join(footer_parts) + " |") + lines.append( + "| **execution coverage** | | " + " | ".join(footer[column]["coverage"] for column in columns) + " |" + ) + lines.append( + "| **off-surface indicators** | | " + + " | ".join(footer[column]["off_surface"].replace("\n", "
") for column in columns) + + " |" + ) + lines.append( + "| **schema friction** | | " + + " | ".join(footer[column]["schema_friction"].replace("\n", "
") for column in columns) + + " |" + ) + lines.append( + "| **completeness** | | " + " | ".join(footer[column]["completeness"] for column in columns) + " |" + ) + return "\n".join(lines) + "\n" + + column_width = max(14, max((len(column) for column in columns), default=14)) + if multiple_repetitions: + column_width = max( + column_width, + max( + (len(value) for task in cells.values() for value in task.values()), + default=14, + ), + ) + heading = f"{'task':5} {'what':34} " + " ".join(f"{column:{column_width}}" for column in columns) + lines.append(heading) + lines.append("-" * len(heading)) + for task_id in task_ids: + line = f"{task_id:5} {prompt_excerpt(task_id, task_metadata):34} " + for column in columns: + line += f"{cells[task_id].get(column, '—'):{column_width}} " + lines.append(line.rstrip()) + lines.append("-" * len(heading)) + for column in columns: + values = footer[column] + pooled = f"{values['success']}/{values['n']}" if values["n"] else "0/0" + if values["task_mean_success"] is None: + rate = f"task-cluster n/a; pooled {pooled}" + else: + rate = ( + f"task-cluster {values['task_mean_success']:.1%} " + f"[{values['task_cluster_lo']:.2f},{values['task_cluster_hi']:.2f}]; pooled {pooled}" + ) + variability = ( + f"{values['tool_variability']}/{values['tasks']} tasks" if values["tool_variability"] is not None else "—" + ) + lines.append( + f"{column:12} success {rate} total calls {values['calls']}" + f" tool variability {variability} infra {values['infra_errors']}" + ) + for column in columns: + lines.append(f"{column:12} {footer[column]['coverage']}") + for column in columns: + for line in footer[column]["off_surface"].splitlines(): + lines.append(f"{column:12} {line}") + for column in columns: + for line in footer[column]["schema_friction"].splitlines(): + lines.append(f"{column:12} {line}") + for column in columns: + lines.append(f"{column:12} {footer[column]['completeness']}") + return "\n".join(lines) + "\n" + + +def surface_label_for_file(path: Path, rows: list[TaskResult]) -> str: + """Pick a column label from the file's dominant label field, else stem.""" + counts: dict[str, int] = defaultdict(int) + for row in rows: + label = row.label + if label: + counts[str(label)] += 1 + if counts: + return max(counts, key=counts.get) # type: ignore[arg-type] + return path.stem diff --git a/evals/result_lifecycle.py b/evals/result_lifecycle.py new file mode 100644 index 0000000..1ff672b --- /dev/null +++ b/evals/result_lifecycle.py @@ -0,0 +1,24 @@ +"""Shared terminal/retryable classification for persisted evaluation results.""" + +from __future__ import annotations + +from typing import Any + +from evals.core.results import TaskResult +from evals.skip_taxonomy import is_expected_environment_capability_skip + + +def is_terminal_result(row: TaskResult | dict[str, Any]) -> bool: + """Return whether a result is authoritative rather than eligible for retry.""" + result = row if isinstance(row, TaskResult) else TaskResult.from_row(row) + error_class = result.error_class + if isinstance(error_class, str) and error_class.startswith("infra_"): + return False + if result.error is not None or result.cleanup_error is not None: + return False + if result.skipped is not None: + return is_expected_environment_capability_skip(result.skipped, task_id=result.task_id) + return True + + +__all__ = ["is_terminal_result"] diff --git a/evals/runner/__init__.py b/evals/runner/__init__.py new file mode 100644 index 0000000..c668940 --- /dev/null +++ b/evals/runner/__init__.py @@ -0,0 +1,28 @@ +"""Live evaluation execution, resume support, metadata, and verifier canary.""" + +from .canary import run_canary +from .live import ( + MAX_ITERATIONS, + MAX_TOKENS, + is_infra_cli_stop_reason, + run_agent_task_via_driver, + run_live, + stdio_server_env, +) +from .meta import is_meta_or_non_task_row, make_run_meta_row, maybe_write_run_meta +from .resume import load_resume_skip_keys, should_skip_resume_row + +__all__ = [ + "MAX_ITERATIONS", + "MAX_TOKENS", + "is_infra_cli_stop_reason", + "is_meta_or_non_task_row", + "load_resume_skip_keys", + "make_run_meta_row", + "maybe_write_run_meta", + "run_agent_task_via_driver", + "run_canary", + "run_live", + "should_skip_resume_row", + "stdio_server_env", +] diff --git a/evals/runner/canary.py b/evals/runner/canary.py new file mode 100644 index 0000000..93241cc --- /dev/null +++ b/evals/runner/canary.py @@ -0,0 +1,157 @@ +"""Verifier canary execution for evaluation tasks.""" + +from __future__ import annotations + +import sys +import uuid +from typing import Any + +from evals.core.errors import TaskSkipped +from evals.seed import make_plane_client, seed, teardown +from evals.tasks.catalog import battery_fingerprint + +CANARY_CANNED_OUTPUTS: dict[str, tuple[str, ...]] = { + "R1": ("state: In Progress",), + "R2": ("count: 0",), + "R3": ("item: Example work item",), + "R4": ("cycle: Sprint 13\nitem: Example work item\noverdue: none",), + "R5": ("comment: looks good",), + "R6": ("project: EVAL deadbeef B",), + "R7": ("state: Backlog | group: backlog",), + "C2": ("release: 1.2.0\nshipped: guessed change",), + "I2": ("state: Backlog",), + "L1": ("logged-minutes: 90\nsummary-work-item-id: guessed-id",), + "L2": ("count: 0",), + "L5": ("count: 0",), +} + + +def canary_probe_texts(task_id: str) -> tuple[str, ...]: + """Return empty plus plausible zero-call answers for a verifier canary.""" + values = ("", "count: 0", *CANARY_CANNED_OUTPUTS.get(task_id, ())) + return tuple(dict.fromkeys(values)) + + +async def run_canary( + tasks: list[dict[str, Any]], + *, + label: str, + required_task_ids: set[str] | frozenset[str] | None = None, +) -> int: + """Seed + verify zero-call probes + teardown per task; no driver/model. + + ``required_task_ids`` enables strict coverage for an explicit environment capability + set. Legitimate skips outside that set remain non-fatal but are always reported. + """ + label = (label or "local").strip() or "local" + battery = battery_fingerprint(tasks) + plane, _workspace_slug = make_plane_client() + print(f"canary battery={battery} label={label} tasks={[task['id'] for task in tasks]}") + + broken_ids: list[str] = [] + verified_ids: list[str] = [] + skipped_reasons: dict[str, str] = {} + errored_reasons: dict[str, list[str]] = {} + + def record_error(task_id: str, reason: str) -> None: + errored_reasons.setdefault(task_id, []).append(reason) + + for task in tasks: + task_id = str(task["id"]) + context: dict[str, Any] = {} + task_needs = set(task.get("needs") or set()) + verifier_exercised = False + try: + try: + seed( + plane, + run_id=uuid.uuid4().hex, + needs=task_needs, + ctx=context, + task_id=task_id, + ) + except TaskSkipped as skip: + skipped_reasons[task_id] = skip.reason + print(f" {task_id} SKIPPED: {skip.reason}") + continue + except Exception as exc: + reason = f"infra_seed {type(exc).__name__}: {exc}" + record_error(task_id, reason) + print(f" {task_id} canary ERROR[infra_seed]: {exc}", file=sys.stderr) + continue + if "bug_type" in task_needs and not context.get("bug_type"): + reason = context.get("bug_type_skip_reason") or "bug_type unavailable" + skipped_reasons[task_id] = str(reason) + print(f" {task_id} SKIPPED: {reason}") + continue + for probe_index, final_text in enumerate(canary_probe_texts(task_id)): + probe = { + "final_text": final_text, + "calls": [], + "call_source": "canary", + } + try: + ok, note = await task["verify"](plane, context, probe) + except TaskSkipped as skip: + skipped_reasons[task_id] = skip.reason + print(f" {task_id} SKIPPED during verifier: {skip.reason}") + break + verifier_exercised = True + if ok: + broken_ids.append(task_id) + print( + f" BROKEN VERIFIER: {task_id} accepted canary probe " + f"{probe_index} final_text={final_text!r} note={note!r}" + ) + break + print(f" {task_id} probe={probe_index} ok=False note={note!r}") + if task_id not in skipped_reasons: + verified_ids.append(task_id) + except Exception as exc: + record_error(task_id, f"{type(exc).__name__}: {exc}") + print(f" {task_id} canary ERROR: {exc}", file=sys.stderr) + finally: + try: + teardown(plane, context) + except Exception as exc: + record_error(task_id, f"teardown {type(exc).__name__}: {exc}") + print(f" {task_id} teardown ERROR: {exc}", file=sys.stderr) + + if ( + verifier_exercised + and task_id not in verified_ids + and task_id not in skipped_reasons + and task_id not in errored_reasons + ): + verified_ids.append(task_id) + + skipped_ids = list(skipped_reasons) + errored_ids = list(errored_reasons) + total = len(tasks) + print(f"canary coverage: verified={len(verified_ids)}/{total} ids={verified_ids}") + print(f"canary coverage: skipped={len(skipped_ids)} ids={skipped_ids} reasons={skipped_reasons}") + print(f"canary coverage: errored={len(errored_ids)} ids={errored_ids} reasons={errored_reasons}") + + required = set(required_task_ids or ()) + missing_required = sorted(required - set(verified_ids)) + if missing_required: + print( + f"canary strict coverage FAILED: missing required ids={missing_required}", + file=sys.stderr, + ) + + if broken_ids: + for task_id in broken_ids: + print(f"BROKEN VERIFIER: {task_id}", file=sys.stderr) + if not verified_ids: + print( + "error: canary verified 0 tasks — nothing exercised; refusing exit 0", + file=sys.stderr, + ) + if broken_ids or errored_ids or missing_required or not verified_ids: + return 1 + print(f"canary: verified zero-call probes rejected ({len(verified_ids)} verifier(s))") + return 0 + + +__all__ = ["CANARY_CANNED_OUTPUTS", "canary_probe_texts", "run_canary"] diff --git a/evals/runner/live.py b/evals/runner/live.py new file mode 100644 index 0000000..564da9a --- /dev/null +++ b/evals/runner/live.py @@ -0,0 +1,754 @@ +"""Live evaluation execution and task result assembly.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evals.core.errors import TaskSkipped, describe_exception +from evals.core.evidence import configured_evidence_labels +from evals.core.results import TaskResult, agent_run_to_task_result +from evals.core.server_env import stdio_server_env +from evals.core.task_metadata import build_task_metadata, entry_requires_mutation +from evals.drivers import KNOWN_DRIVERS, get_driver +from evals.drivers.api import MODEL_TIERS +from evals.report.load import RunExpectation, dedupe_rows_latest, load_rows, validate_run_keys +from evals.report.off_surface import off_surface_statement +from evals.report.schema_friction import schema_friction_statement +from evals.report.summary import completeness_statement, execution_coverage_statement, summarize +from evals.seed import make_plane_client, seed, teardown +from evals.seed.identities import capture_seed_artifacts +from evals.tasks.catalog import battery_fingerprint, task_author, task_fingerprint +from evals.tasks.prompts import PromptBindError, format_task_prompt + +from .meta import make_run_meta_row, maybe_write_run_meta, read_git_revision +from .resume import load_resume_skip_keys + +# Only the API driver enforces this: CLI drivers are handed max_turns and discard it (the +# antigravity driver says so outright -- "no turn-cap flag"), so their vendor loop decides. +# A cap that binds therefore biases API arms against CLI arms of the same model rather than +# limiting both, and 15 does bind: codex spent 18 calls on W6 in a CLI arm, so tasks in this +# battery legitimately need more. Kept as the default so no existing run changes, and made +# settable so a cross-driver comparison can lift it out of the way and say it did. +MAX_ITERATIONS = 15 +MAX_TOKENS = 8192 + + +def _system_preamble(workspace_slug: str, project_name: str) -> str: + """Keep under 100 words — part of measured context.""" + return ( + f"You are evaluating Plane project management tools. " + f"Workspace slug: {workspace_slug}. Project name: {project_name}. " + f"Complete the task using the available tools, then stop." + ) + + +def is_infra_cli_stop_reason(stop_reason: str | None) -> bool: + """True when a CLI AgentRun stop_reason should be classified as infra_cli. + + ``timeout`` and Claude error subtypes (``error_during_execution``, bare + ``error``, …) are infrastructure. ``error_max_turns`` is a genuine task + failure and stays in the success-rate denominator. + """ + if not stop_reason: + return False + reason = str(stop_reason) + if reason == "timeout": + return True + if reason == "error_max_turns": + return False + if reason == "error" or reason.startswith("error_"): + return True + return False + + +def _elapsed(since: float) -> str: + """Wall time as mm:ss (or h:mm:ss past an hour) for progress lines.""" + seconds = int(time.monotonic() - since) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + if hours: + return f"{hours}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + +def _timeout_error_message(agent: TaskResult) -> str: + """Prefer the driver's recorded timeout note over recomputing MAX_ITERATIONS.""" + for note in agent.driver_notes: + if isinstance(note, str) and note.startswith("timeout after"): + return note + return "timeout" + + +async def run_agent_task_via_driver( + *, + driver: Any, + model_id: str | None, + task: dict[str, Any], + ctx: dict[str, Any], + workspace_slug: str, + server_env: dict[str, str] | None = None, + artifact_dir: Path | None = None, + max_iterations: int = MAX_ITERATIONS, +) -> TaskResult: + """Run one task through the selected driver.""" + project_name = ctx["project_name"] + system = _system_preamble(workspace_slug, project_name) + prompt = format_task_prompt(task, ctx, strict=True) + + mcp_env = stdio_server_env(extra=server_env) + # Drivers are sync (CLI subprocess or API loop); keep them off this loop. + agent_run = await asyncio.to_thread( + driver.run_task, + prompt, + mcp_env, + model_id, + max_iterations, + system=system, + cwd=Path(__file__).resolve().parent.parent.parent, + evidence_sentinels=ctx.get("evidence_sentinels"), + evidence_targets=ctx.get("evidence_targets"), + evidence_aggregates=ctx.get("evidence_aggregates"), + artifact_dir=artifact_dir, + ) + return agent_run_to_task_result(agent_run) + + +def _make_task_row( + *, + run_id: str, + git_revision: str, + label: str, + driver_name: str, + provider: str | None, + model_id: str | None, + model_request: str | None, + requested_tier: str | None, + task: dict[str, Any], + repetition: int, + expected_rows: int, + battery: str, + server: str, +) -> TaskResult: + return TaskResult( + run_id=run_id, + fixture_seed_id=uuid.uuid4().hex, + ts=datetime.now(timezone.utc).isoformat(), + git_sha=git_revision, + battery=battery, + task_fingerprint=task_fingerprint(task), + label=label, + driver=driver_name, + provider=provider, + server=server, + model=model_id, + requested_model=model_request, + requested_tier=requested_tier, + resolved_model=model_id, + task_id=str(task["id"]), + author=task_author(task), + rep=repetition, + expected_rows=expected_rows, + ) + + +def _seed_fixtures( + plane: Any, + task: dict[str, Any], + context: dict[str, Any], + *, + row: TaskResult, + repetition: int, +) -> bool: + """Seed one task and record fixture skips or infrastructure failures.""" + task_needs = set(task.get("needs") or set()) + # Seed wrap: TaskSkipped → skip; other failures → infra_seed. + try: + seed( + plane, + run_id=row.fixture_seed_id, + needs=task_needs, + ctx=context, + task_id=str(task["id"]), + ) + # Aggregates count as registered evidence: the proxy matches an exact total_count for a + # targeted request exactly as it matches a sentinel value. Omitting them here made the + # gate stricter than the matcher, so a task whose answer *is* a count could not seed — + # L2 registered its activity count and was rejected as having registered nothing. + if "read" in set(task.get("tags") or set()) and not configured_evidence_labels( + context.get("evidence_sentinels"), + context.get("evidence_targets"), + context.get("evidence_aggregates"), + ): + raise RuntimeError(f"{task['id']} seed did not register any response evidence") + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print( + f" {task['id']} rep={repetition} SKIPPED: {skip.reason}" + + (f" — {skip.detail}" if getattr(skip, "detail", None) else ""), + flush=True, + ) + return False + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + flush=True, + ) + if context.get("project_name"): + print( + f" orphaned project may remain: {context['project_name']}", + file=sys.stderr, + flush=True, + ) + return False + if "bug_type" in task_needs and not context.get("bug_type"): + reason = context.get("bug_type_skip_reason") or "bug_type unavailable" + row.skipped = reason + row.verify_note = reason + print(f" {task['id']} rep={repetition} SKIPPED: {reason}", flush=True) + return False + return True + + +async def _drive_agent( + *, + driver: Any, + model_id: str | None, + task: dict[str, Any], + context: dict[str, Any], + workspace_slug: str, + server_env: dict[str, str] | None, + row: TaskResult, + repetition: int, + is_api_driver: bool, + artifact_dir: Path, + max_iterations: int = MAX_ITERATIONS, +) -> TaskResult | None: + """Run the agent and classify launch or prompt failures.""" + # Agent wrap: API failures and CLI failures are infrastructure. + # Contained CLI stops (timeout / error subtypes) return AgentRun. + try: + return await run_agent_task_via_driver( + driver=driver, + model_id=model_id, + task=task, + ctx=context, + workspace_slug=workspace_slug, + server_env=server_env, + artifact_dir=artifact_dir, + max_iterations=max_iterations, + ) + except PromptBindError as exc: + # Empty/missing seed IDs in the prompt — not an agent failure. + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "infra_seed" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_seed]: {exc}", + file=sys.stderr, + flush=True, + ) + return None + except Exception as exc: + if hasattr(exc, "trace_integrity"): + row.trace_integrity = bool(exc.trace_integrity) + row.trace_integrity_reason = getattr(exc, "trace_integrity_reason", None) + row.tool_manifest_fingerprint = getattr(exc, "tool_manifest_fingerprint", None) + if is_api_driver: + agent_error_class = "infra_api" + else: + agent_error_class = "infra_cli" + row.success = False + # Flattened, not str(exc): an ExceptionGroup's message names only its sub-exception + # count, and anyio wraps every driver call in a task group. + described = describe_exception(exc) + row.error = described + row.error_class = agent_error_class + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[{agent_error_class}]: {described}", + file=sys.stderr, + ) + return None + + +def _apply_agent_run( + row: TaskResult, + agent: TaskResult, + *, + model_alias: str, + requested_tier: str | None, + model_id: str | None, +) -> None: + """Copy agent metrics and restore the run-level model and server identity.""" + row.apply_agent_result(agent) + # Driver-level requested_model is the resolved ID. + # Restore run-level intent and retain both identities. + row.requested_model = model_alias + row.requested_tier = requested_tier + row.resolved_model = model_id + + +def _record_cli_infra_stop( + row: TaskResult, + agent: TaskResult, + *, + task: dict[str, Any], + repetition: int, + driver_name: str, +) -> bool: + """Record contained CLI infrastructure stops and block verification.""" + # CLI infra stops: timeout + error subtypes except error_max_turns. + stop_reason = agent.stop_reason + if not driver_name.endswith("-cli") or not is_infra_cli_stop_reason( + str(stop_reason) if stop_reason is not None else None + ): + return False + row.success = False + row.error_class = "infra_cli" + if stop_reason == "timeout": + row.error = _timeout_error_message(agent) + else: + notes = [note for note in agent.driver_notes if isinstance(note, str)] + detail = "; ".join(notes) if notes else str(stop_reason) + row.error = detail + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[infra_cli]: {row.error}", + file=sys.stderr, + ) + return True + + +def _record_trace_infra( + row: TaskResult, + agent: TaskResult, + *, + task: dict[str, Any], + repetition: int, +) -> bool: + """Make recorder/protocol trace loss completeness-visible infrastructure.""" + if agent.trace_integrity or agent.trace_integrity_reason == "result_pair_mismatch": + return False + error_class = "infra_protocol" if agent.trace_integrity_reason == "protocol_violation" else "infra_trace" + detail = next( + ( + note + for note in agent.driver_notes + if isinstance(note, str) and note.startswith(("proxy_sidecar_incomplete", "proxy_sidecar_empty")) + ), + f"trace_integrity={agent.trace_integrity_reason or 'recorder_loss'}", + ) + row.success = False + row.error_class = error_class + row.error = detail + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[{error_class}]: {detail}", + file=sys.stderr, + ) + return True + + +async def _verify_task( + plane: Any, + task: dict[str, Any], + context: dict[str, Any], + agent: TaskResult, + *, + row: TaskResult, + repetition: int, +) -> None: + """Run one verifier and record task outcomes or verifier failures.""" + verify = task["verify"] + + def _require_surface_for_mutation(task: dict, agent: Any, *, ok: bool, note: str) -> tuple[bool, str]: + """A write task that changed Plane without calling a tool did not demonstrate the surface. + + Write verifiers read Plane back, so they answer "did the state change", not "did the + agent change it through the surface under test". Measured: an agent that could not + work the tools out read the repo it was standing in, took the API key from its own + environment and mutated Plane over REST. The state was correct and the task scored a + pass with no tool call recorded. + + Only applied to a trustworthy trace — when integrity is false the row is already an + infrastructure error, and zero calls there means the recording failed, not the agent. + """ + if not ok or not entry_requires_mutation({"tags": task.get("tags") or ()}): + return ok, note + if agent.trace_integrity is False: + return ok, note + calls = [call for call in (agent.to_row().get("calls") or []) if not bool(call.get("is_error"))] + if calls: + return ok, note + return False, f"{note}; surface=missing (write task changed Plane with 0 successful tool calls)" + + try: + agent_row = agent.to_row() + ok, note = await verify( + plane, + context, + { + "final_text": agent.final_text, + "calls": agent_row["calls"], + "call_source": agent.call_source, + "evidence_trace_available": agent.evidence_trace_available, + "driver_notes": list(agent.driver_notes), + "result_pair_mismatch": agent.result_pair_mismatch, + "trace_integrity": agent.trace_integrity, + "trace_integrity_reason": agent.trace_integrity_reason, + }, + ) + ok, note = _require_surface_for_mutation(task, agent, ok=bool(ok), note=note) + row.success = bool(ok) + row.verify_note = note + print( + f" {task['id']} rep={repetition} success={ok} calls={agent.num_calls} note={note!r}", + flush=True, + ) + except TaskSkipped as skip: + row.skipped = skip.reason + row.verify_note = skip.reason + print( + f" {task['id']} rep={repetition} SKIPPED: {skip.reason}" + + (f" — {skip.detail}" if getattr(skip, "detail", None) else ""), + flush=True, + ) + except Exception as exc: + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" + print( + f" {task['id']} rep={repetition} ERROR[task]: {exc}", + file=sys.stderr, + ) + + +def _record_unexpected( + row: TaskResult, + exc: Exception, + *, + task: dict[str, Any], + repetition: int, + context: dict[str, Any], +) -> None: + """Record failures outside the seed, driver, and verifier boundaries.""" + row.success = False + row.error = f"{type(exc).__name__}: {exc}" + row.error_class = "task" + row.verify_note = "" + print(f" {task['id']} rep={repetition} ERROR[task]: {exc}", file=sys.stderr) + if context.get("project_name"): + print( + f" orphaned project may remain: {context['project_name']}", + file=sys.stderr, + ) + + +def _remove_fixtures(plane: Any, context: dict[str, Any], row: TaskResult) -> None: + """Remove task fixtures and retain the historical teardown diagnostics.""" + try: + teardown(plane, context) + except Exception as exc: + row.cleanup_error = f"{type(exc).__name__}: {exc}" + print(f" teardown error: {exc}", file=sys.stderr) + if context.get("project_name"): + print(f" orphaned project: {context['project_name']}", file=sys.stderr) + + +async def _run_task_repetition( + *, + plane: Any, + driver: Any, + workspace_slug: str, + task: dict[str, Any], + repetition: int, + expected_rows: int, + run_id: str, + git_revision: str, + label: str, + driver_name: str, + provider_id: str | None, + model_id: str | None, + model_alias: str, + requested_tier: str | None, + battery: str, + is_api_driver: bool, + external: bool, + server_env: dict[str, str] | None, + artifact_dir: Path, + max_iterations: int = MAX_ITERATIONS, +) -> TaskResult: + """Seed, drive, verify, assemble, and remove one task repetition.""" + context: dict[str, Any] = {} + row = _make_task_row( + run_id=run_id, + git_revision=git_revision, + label=label, + driver_name=driver_name, + provider=provider_id, + model_id=model_id, + model_request=model_alias, + requested_tier=requested_tier, + task=task, + repetition=repetition, + expected_rows=expected_rows, + battery=battery, + server="external" if external else "local", + ) + try: + if _seed_fixtures(plane, task, context, row=row, repetition=repetition): + agent = await _drive_agent( + driver=driver, + model_id=model_id, + task=task, + context=context, + workspace_slug=workspace_slug, + server_env=server_env, + row=row, + repetition=repetition, + is_api_driver=is_api_driver, + artifact_dir=artifact_dir, + max_iterations=max_iterations, + ) + if agent is not None: + _apply_agent_run( + row, + agent, + model_alias=model_alias, + requested_tier=requested_tier, + model_id=model_id, + ) + infra_stop = _record_cli_infra_stop( + row, + agent, + task=task, + repetition=repetition, + driver_name=driver_name, + ) + trace_infra = False + if not infra_stop: + trace_infra = _record_trace_infra( + row, + agent, + task=task, + repetition=repetition, + ) + if not infra_stop and not trace_infra: + await _verify_task(plane, task, context, agent, row=row, repetition=repetition) + except Exception as exc: + # Anything outside seed/driver/verify wraps. + _record_unexpected(row, exc, task=task, repetition=repetition, context=context) + finally: + try: + row.seeded_entity_kinds, row.randomized_seed_namespaces = capture_seed_artifacts(context) + except Exception as exc: + _record_unexpected(row, exc, task=task, repetition=repetition, context=context) + _remove_fixtures(plane, context, row) + return row + + +async def run_live( + tasks: list[dict[str, Any]], + *, + model_alias: str, + reps: int, + label: str, + out_path: Path, + driver_name: str = "api", + provider: str = "anthropic", + server_cmd: list[str] | None = None, + server_env: dict[str, str] | None = None, + resume: bool = False, + record_result_payloads: bool = False, + resolved_model_id: str | None = None, + max_iterations: int = MAX_ITERATIONS, +) -> int: + label = (label or "local").strip() or "local" + external = server_cmd is not None + + driver_name = (driver_name or "api").strip().lower() + if driver_name not in KNOWN_DRIVERS: + print( + f"error: unknown --driver {driver_name!r}; expected one of {sorted(KNOWN_DRIVERS)}", + file=sys.stderr, + ) + return 2 + provider = (provider or "anthropic").strip().lower() + is_api_driver = driver_name == "api" + provider_id = provider if is_api_driver else None + model_id = resolved_model_id if resolved_model_id is not None else model_alias + requested_tier = model_alias if model_alias in MODEL_TIERS else None + + run_id = uuid.uuid4().hex + git_revision = read_git_revision() + battery = battery_fingerprint(tasks) + total_runs = len(tasks) * reps + out_path.parent.mkdir(parents=True, exist_ok=True) + artifact_dir = out_path.parent / f"{out_path.stem}.artifacts" / driver_name + + resume_skip: set[tuple[str, int, str]] = set() + if resume: + try: + resume_skip, skip_count, retry_count = load_resume_skip_keys( + out_path, + label=label, + battery=battery, + model=model_id, + driver=driver_name, + provider=provider_id, + ) + except SystemExit as exc: + print(exc, file=sys.stderr) + return 2 + print(f"resume: skipping {skip_count} completed rows, retrying {retry_count}", flush=True) + + # First line of a new/empty file is a meta header (skipped by loaders). + meta = make_run_meta_row( + run_id=run_id, + label=label, + server="external" if external else "local", + battery=battery, + model=model_id, + requested_model=model_alias, + requested_tier=requested_tier, + resolved_model=model_id, + driver=driver_name, + provider=provider_id, + git_sha=git_revision, + expected_rows=total_runs, + expected_task_ids=[str(task["id"]) for task in tasks], + expected_reps=reps, + task_metadata=build_task_metadata(tasks), + ) + if maybe_write_run_meta(out_path, meta): + print(f"wrote meta header battery={battery} label={label}", flush=True) + + plane, workspace_slug = make_plane_client() + # User chose --driver explicitly: codex live is allowed (they own the quota). + driver_kwargs: dict[str, Any] = {} + if is_api_driver: + driver_kwargs.update({"provider": provider, "max_tokens": MAX_TOKENS}) + if driver_name == "codex-cli": + driver_kwargs["allow_live"] = True + if not is_api_driver: + driver_kwargs["record_result_payloads"] = record_result_payloads + # --server-cmd must reach every driver; otherwise we + # silently benchmark the wrong server. + if server_cmd is not None: + driver_kwargs["server_command"] = server_cmd + driver = get_driver(driver_name, **driver_kwargs) + + print( + f"run_id={run_id} battery={battery} driver={driver_name} provider={provider_id} " + f"requested_model={model_alias} resolved_model={model_id} " + f"label={label} tasks={[task['id'] for task in tasks]} reps={reps}" + ) + print(f"writing {out_path}", flush=True) + + # A battery is tens of minutes of silence otherwise: one line before each + # repetition says what is running now, and one after says where the run is. + started_at = time.monotonic() + finished = 0 + passed = 0 + skipped = 0 + failed = 0 + + with out_path.open("a", encoding="utf-8") as file: + for task in tasks: + for repetition in range(reps): + if (task["id"], repetition, label) in resume_skip: + finished += 1 + print(f" {task['id']} rep={repetition} RESUME_SKIP", flush=True) + continue + print( + f"[{finished + 1:>2}/{total_runs}] {task['id']} rep={repetition} " + f"running ({_elapsed(started_at)} elapsed)", + flush=True, + ) + row = await _run_task_repetition( + plane=plane, + driver=driver, + workspace_slug=workspace_slug, + task=task, + repetition=repetition, + expected_rows=total_runs, + run_id=run_id, + git_revision=git_revision, + label=label, + driver_name=driver_name, + provider_id=provider_id, + model_id=model_id, + model_alias=model_alias, + requested_tier=requested_tier, + battery=battery, + is_api_driver=is_api_driver, + external=external, + server_env=server_env, + artifact_dir=artifact_dir, + max_iterations=max_iterations, + ) + file.write(json.dumps(row.to_row(), default=str) + "\n") + file.flush() + + finished += 1 + if row.skipped: + skipped += 1 + elif row.success: + passed += 1 + else: + failed += 1 + print( + f" {finished}/{total_runs} done · {passed} pass · " + f"{failed} fail · {skipped} skip · {_elapsed(started_at)} elapsed", + flush=True, + ) + + print( + f"finished {finished}/{total_runs} in {_elapsed(started_at)}: {passed} pass, {failed} fail, {skipped} skip", + flush=True, + ) + selected_task_ids = {str(task["id"]) for task in tasks} + raw_result_rows = load_rows(out_path, dedupe="none") + run_keys = validate_run_keys( + raw_result_rows, + RunExpectation(tuple(str(task["id"]) for task in tasks), reps, label), + ) + result_rows = [ + row + for row in dedupe_rows_latest(raw_result_rows) + if row.label == label + and (not row.battery or row.battery == battery) + and row.task_id in selected_task_ids + and 0 <= row.rep < reps + ] + summary = summarize(result_rows, expected_rows=total_runs, run_keys=run_keys) + if summary.task_mean_success is not None: + task_count = sum(task.n > 0 for task in summary.tasks.values()) + print( + f"success: {summary.task_mean_success:.1%} across {task_count} tasks " + f"(cluster-bootstrap95 [{summary.task_cluster_lo:.2f},{summary.task_cluster_hi:.2f}]; " + f"pooled repetitions {summary.aggregate_k}/{summary.aggregate_n})", + flush=True, + ) + else: + print("success: 0/0 (n/a; no evaluated rows)", flush=True) + print(execution_coverage_statement(summary), flush=True) + print(off_surface_statement(summary.off_surface), flush=True) + print(schema_friction_statement(summary.schema_friction), flush=True) + print(completeness_statement(summary), flush=True) + return 0 if summary.complete else 1 diff --git a/evals/runner/meta.py b/evals/runner/meta.py new file mode 100644 index 0000000..56cab81 --- /dev/null +++ b/evals/runner/meta.py @@ -0,0 +1,103 @@ +"""Run metadata and repository provenance for evaluation results.""" + +from __future__ import annotations + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evals.core.results import RESULT_SCHEMA_VERSION +from evals.core.task_metadata import METADATA_FIELD, normalize_task_metadata + + +def read_git_revision() -> str: + try: + return ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + cwd=Path(__file__).resolve().parent.parent.parent, + ) + .decode() + .strip() + ) + except Exception: + return "unknown" + + +def is_meta_or_non_task_row(row: dict[str, Any]) -> bool: + """True for run-header meta lines or any row without a task_id.""" + if row.get("row_type") == "meta": + return True + return row.get("task_id") is None + + +def make_run_meta_row( + *, + run_id: str, + label: str, + server: str, + battery: str, + model: str | None, + driver: str, + git_sha: str, + provider: str | None = None, + requested_model: str | None = None, + requested_tier: str | None = None, + resolved_model: str | None = None, + expected_rows: int | None = None, + expected_task_ids: list[str] | tuple[str, ...] | None = None, + expected_reps: int | None = None, + task_metadata: dict[str, Any] | None = None, + ts: str | None = None, +) -> dict[str, Any]: + """Build the single first-line meta record for a new output JSONL.""" + if (expected_task_ids is None) != (expected_reps is None): + raise ValueError("expected_task_ids and expected_reps must be declared together") + if expected_task_ids is not None and expected_reps is not None: + task_ids = [str(task_id) for task_id in expected_task_ids] + if not task_ids or any(not task_id for task_id in task_ids) or len(set(task_ids)) != len(task_ids): + raise ValueError("expected_task_ids must contain unique non-empty ids") + if expected_reps < 1: + raise ValueError("expected_reps must be positive") + exact_rows = len(task_ids) * expected_reps + if expected_rows is not None and expected_rows != exact_rows: + raise ValueError(f"expected_rows={expected_rows} disagrees with exact expectation={exact_rows}") + row = { + "schema_version": RESULT_SCHEMA_VERSION, + "row_type": "meta", + "run_id": run_id, + "label": label, + "server": server, + "battery": battery, + "model": model, + "requested_model": requested_model if requested_model is not None else model, + "requested_tier": requested_tier, + "resolved_model": resolved_model if resolved_model is not None else model, + "driver": driver, + "provider": provider, + "git_sha": git_sha, + "ts": ts or datetime.now(timezone.utc).isoformat(), + } + if expected_rows is not None: + row["expected_rows"] = expected_rows + if expected_task_ids is not None: + row["expected_task_ids"] = task_ids + if expected_reps is not None: + row["expected_reps"] = expected_reps + if task_metadata: + # Persisted so a report describes the run it reads rather than today's catalog. + row[METADATA_FIELD] = normalize_task_metadata(task_metadata) + return row + + +def maybe_write_run_meta(path: Path, meta: dict[str, Any]) -> bool: + """Write meta as the first line when the file is missing or empty. Returns True if written.""" + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file() and path.stat().st_size > 0: + return False + with path.open("w", encoding="utf-8") as file: + file.write(json.dumps(meta, default=str) + "\n") + return True diff --git a/evals/runner/resume.py b/evals/runner/resume.py new file mode 100644 index 0000000..759a95b --- /dev/null +++ b/evals/runner/resume.py @@ -0,0 +1,119 @@ +"""Resume decisions for evaluation result files.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +from evals.core.results import TaskResult +from evals.result_lifecycle import is_terminal_result + +from .meta import is_meta_or_non_task_row + + +def should_skip_resume_row(row: TaskResult | dict[str, Any]) -> bool: + """Return True if a prior row is a completed result that resume should skip. + + Re-run rows with errors, cleanup failures, or unexpected skips. Known missing + environment capabilities are legitimate terminal outcomes because rerunning cannot + add them; fixture collisions and unknown skips may be repairable. + Pure function — unit-tested without the live battery. + """ + return is_terminal_result(row) + + +def _resume_field_mismatch( + row: dict[str, Any], + *, + field: str, + expected: str | None, +) -> str | None: + """Return an error message if row[field] is present and disagrees with expected.""" + if expected is None: + return None + raw = row.get(field) + if raw is None or raw == "": + return None # back-compat: older rows without the key pass + # Driver/provider compare case-insensitively; label/battery/model are exact. + if field in ("driver", "provider"): + received, wanted = str(raw).strip().lower(), expected.strip().lower() + else: + received, wanted = str(raw).strip(), expected.strip() + if received != wanted: + return f"error: --resume file {field} {raw!r} does not match current {field} {expected!r}" + return None + + +def load_resume_skip_keys( + path: Path, + *, + label: str, + battery: str | None = None, + model: str | None = None, + driver: str | None = None, + provider: str | None = None, +) -> tuple[set[tuple[str, int, str]], int, int]: + """Load existing JSONL rows and decide which (task_id, rep, label) keys to skip. + + Returns ``(skip_keys, n_skip, n_retry)`` where ``n_retry = len(seen - skip_keys)`` + (keys that still need a re-run). Raises ``SystemExit`` when a row's label / + battery / model / driver / provider disagrees with the current run (missing keys pass for + back-compat). Meta lines (``row_type=meta`` or no task_id) are mismatch-checked + but not counted as task rows. Truncated/invalid JSON lines are warned and skipped. + """ + if not path.is_file(): + return set(), 0, 0 + skip_keys: set[tuple[str, int, str]] = set() + seen: set[tuple[str, int, str]] = set() + with path.open(encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"warning: --resume {path}:{line_number}: skipping invalid JSON ({exc})", + file=sys.stderr, + ) + continue + if not isinstance(row, dict): + continue + for field, expected in ( + ("label", label), + ("battery", battery), + ("driver", driver), + ("provider", provider), + ): + message = _resume_field_mismatch(row, field=field, expected=expected) + if message: + raise SystemExit(message) + # New tier-aware rows identify the resolved model explicitly. Older + # API rows use requested_model for the resolved ID, while oldest rows + # only have model (which may be provider-reported). + model_row = dict(row) + if model_row.get("resolved_model"): + model_row["model"] = model_row["resolved_model"] + elif model_row.get("requested_model"): + model_row["model"] = model_row["requested_model"] + message = _resume_field_mismatch(model_row, field="model", expected=model) + if message: + raise SystemExit(message) + # Meta / header rows: checked above, not part of resume key set. + if is_meta_or_non_task_row(row): + continue + result = TaskResult.from_row(row) + if not result.task_id: + continue + key = (result.task_id, result.rep, result.label) + seen.add(key) + if should_skip_resume_row(result): + skip_keys.add(key) + else: + # Prior infra/error row: do not skip (will re-run). Drop any earlier skip. + skip_keys.discard(key) + retry_count = len(seen - skip_keys) + return skip_keys, len(skip_keys), retry_count diff --git a/evals/seed/__init__.py b/evals/seed/__init__.py new file mode 100644 index 0000000..99d3060 --- /dev/null +++ b/evals/seed/__init__.py @@ -0,0 +1,187 @@ +"""Evaluation fixture creation and removal.""" + +from .build import check_workspace_fixture_collisions, collision_categories, seed +from .client import make_plane_client +from .customers import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, + seed_customer, +) +from .customers import ( + EVALUATION_CUSTOMER_PROPERTY_NAME as DEBIAS_CUSTOMER_PROP_DISPLAY, +) +from .cycles import CYCLE_CURRENT, CYCLE_PAST, seed_cycles +from .gates import is_plan_gate, plan_gate_skips +from .intake import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, seed_intake +from .item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + seed_item_type, +) +from .labels import LABEL_NAMES, seed_labels +from .modules import MODULE_COMPLETED_TITLES, MODULE_NAME, seed_module +from .plan import seed_plan +from .projects import ( + MAIN_PROJECT_BUG_TITLES, + PLANE_PROJECT_IDENTIFIER_MAX_LENGTH, + SECOND_PROJECT_BUG_TITLES, + create_project_with_collision_retry, + enable_project_features, + enable_workspace_features, + is_identifier_collision, + is_name_collision, + secrets, + seed_second_project, +) +from .projects import ( + MAIN_PROJECT_BUG_TITLES as R6_MAIN_BUG_TITLES, +) +from .projects import ( + SECOND_PROJECT_BUG_TITLES as R6_SECOND_BUG_TITLES, +) +from .releases import ( + EVALUATION_RELEASE_TAG_VERSION, + RELEASE_CHANGELOG_TEXT, + RELEASE_NAME, + seed_release, +) +from .releases import ( + EVALUATION_RELEASE_TAG_VERSION as DEBIAS_RELEASE_TAG_VERSION, +) +from .remove import CleanupFailure, TeardownError, teardown +from .work_items import ( + BLOCKING_REFERENCE_ADDRESS, + BLOCKING_SOURCE_TITLE, + BLOCKING_TARGET_TITLE, + CHECKOUT_COMMENT_PHRASES, + CHECKOUT_TIMEOUT_TITLE, + DARK_MODE_TITLE, + DUE_THIS_WEEK_TITLES, + PAYMENT_WEBHOOK_TITLE, + SIDEBAR_TITLE, + UNFINISHED_CYCLE_TITLES, + WORK_ITEM_FIXTURES, + find_completed_state, + list_states, + require_activities, + seed_work_items, +) +from .work_items import ( + BLOCKING_REFERENCE_ADDRESS as W7_URL, +) +from .work_items import ( + BLOCKING_SOURCE_TITLE as W7_SOURCE_TITLE, +) +from .work_items import ( + BLOCKING_TARGET_TITLE as W7_TARGET_TITLE, +) +from .work_items import ( + CHECKOUT_COMMENT_PHRASES as R5_COMMENT_PHRASES, +) +from .work_items import ( + CHECKOUT_TIMEOUT_TITLE as R5_TITLE, +) +from .work_items import ( + DARK_MODE_TITLE as W3_TITLE, +) +from .work_items import ( + DUE_THIS_WEEK_TITLES as R3_DUE_TITLES, +) +from .work_items import ( + PAYMENT_WEBHOOK_TITLE as R1_TITLE, +) +from .work_items import ( + PAYMENT_WEBHOOK_TITLE as W8_TITLE, +) +from .work_items import ( + SIDEBAR_TITLE as W2_TITLE, +) +from .work_items import ( + UNFINISHED_CYCLE_TITLES as W6_UNFINISHED_TITLES, +) +from .work_items import ( + WORK_ITEM_FIXTURES as ITEM_FIXTURES, +) +from .work_items import require_activities as _gate_activity_worker + +__all__ = [ + "BUG_TYPE_NAME", + "CUSTOMER_NAME", + "CUSTOMER_REQUEST_NAME", + "CleanupFailure", + "CYCLE_CURRENT", + "CYCLE_PAST", + "DEBIAS_CUSTOMER_PROP_DISPLAY", + "DEBIAS_RELEASE_TAG_VERSION", + "DARK_MODE_TITLE", + "DUE_THIS_WEEK_TITLES", + "EVALUATION_CUSTOMER_PROPERTY_NAME", + "EVALUATION_RELEASE_TAG_VERSION", + "INTAKE_BILLING_TITLE", + "INTAKE_SPAM_TITLE", + "ITEM_FIXTURES", + "INCIDENT_TYPE_NAME", + "LABEL_NAMES", + "MAIN_PROJECT_BUG_TITLES", + "MODULE_COMPLETED_TITLES", + "MODULE_NAME", + "PAYMENT_WEBHOOK_TITLE", + "PLANE_PROJECT_IDENTIFIER_MAX_LENGTH", + "R1_TITLE", + "R3_DUE_TITLES", + "R5_COMMENT_PHRASES", + "R5_TITLE", + "R6_MAIN_BUG_TITLES", + "R6_SECOND_BUG_TITLES", + "RELEASE_CHANGELOG_TEXT", + "RELEASE_NAME", + "SECOND_PROJECT_BUG_TITLES", + "SEVERITY_PROPERTY_NAME", + "SIDEBAR_TITLE", + "TeardownError", + "UNFINISHED_CYCLE_TITLES", + "W2_TITLE", + "W3_TITLE", + "W6_UNFINISHED_TITLES", + "W7_SOURCE_TITLE", + "W7_TARGET_TITLE", + "W7_URL", + "W8_TITLE", + "WORK_ITEM_FIXTURES", + "BLOCKING_REFERENCE_ADDRESS", + "BLOCKING_SOURCE_TITLE", + "BLOCKING_TARGET_TITLE", + "CHECKOUT_COMMENT_PHRASES", + "CHECKOUT_TIMEOUT_TITLE", + "_gate_activity_worker", + "check_workspace_fixture_collisions", + "collision_categories", + "create_project_with_collision_retry", + "enable_project_features", + "enable_workspace_features", + "find_completed_state", + "is_identifier_collision", + "is_name_collision", + "is_evaluation_customer_name", + "is_plan_gate", + "plan_gate_skips", + "list_states", + "make_plane_client", + "require_activities", + "secrets", + "seed", + "seed_customer", + "seed_cycles", + "seed_intake", + "seed_item_type", + "seed_labels", + "seed_module", + "seed_plan", + "seed_release", + "seed_second_project", + "seed_work_items", + "teardown", +] diff --git a/evals/seed/build.py b/evals/seed/build.py new file mode 100644 index 0000000..98726c1 --- /dev/null +++ b/evals/seed/build.py @@ -0,0 +1,408 @@ +"""Fixture dispatch and workspace preclean for evaluation runs.""" + +from __future__ import annotations + +import os +from typing import Any + +from plane import PlaneClient + +from evals.core.errors import TaskSkipped +from evals.core.fixtures import eval_project_name_variants + +from .customers import ( + CUSTOMER_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, + seed_customer, +) +from .cycles import seed_cycles +from .identities import record_seeded_entity +from .intake import seed_intake +from .item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + is_severity_property, + is_work_item_type_named, + list_workspace_properties_for_type, + list_workspace_work_item_types, + seed_item_type, + workspace_owns_work_item_types, +) +from .labels import seed_labels +from .modules import seed_module +from .projects import ( + create_project_with_collision_retry, + enable_project_features, + enable_workspace_features, + seed_second_project, +) +from .releases import EVALUATION_RELEASE_TAG_VERSION, seed_release +from .states import seed_r7_state_oracle +from .work_items import ( + CHECKOUT_COMMENT_PHRASES, + CHECKOUT_TIMEOUT_TITLE, + DUE_THIS_WEEK_TITLES, + UNFINISHED_CYCLE_TITLES, + require_activities, + seed_work_items, +) +from .workspace import list_workspace_rows + +_WORKSPACE_BASELINE_CATEGORIES = ( + "customers", + "release_tags", + "customer_properties", + "work_item_types", + "work_item_properties", +) +_TASK_COLLISION_CATEGORIES = { + "C1": {"customers"}, + "L3": {"release_tags"}, + "S1": {"work_item_properties"}, + "S3": {"work_item_types"}, +} + + +def _snapshot_workspace_baseline( + plane: PlaneClient, + workspace_slug: str, + categories: set[str] | None = None, +) -> dict[str, set[str] | None]: + """Capture fixed-name workspace fixtures that existed before the agent runs.""" + baseline: dict[str, set[str] | None] = dict.fromkeys(_WORKSPACE_BASELINE_CATEGORIES) + # Preserve the established always-on snapshots. Type APIs may be plan-gated, so + # their ownership baselines are read only for tasks that can create those fixtures. + wanted = {"customers", "release_tags", "customer_properties"} | set(categories or ()) + customers = getattr(plane, "customers", None) + releases = getattr(plane, "releases", None) + specs = ( + ( + "customers", + customers, + lambda row: is_evaluation_customer_name(getattr(row, "name", None)), + ), + ( + "release_tags", + getattr(releases, "tags", None) if releases is not None else None, + lambda row: (getattr(row, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION, + ), + ( + "customer_properties", + getattr(customers, "properties", None) if customers is not None else None, + lambda row: ( + (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip().casefold() + == EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + ), + ), + ) + for category, api, matches in specs: + if category not in wanted: + continue + if api is None: + continue + try: + rows = list_workspace_rows(api, workspace_slug) + except Exception as exc: + raise RuntimeError(f"workspace baseline snapshot: list {category} failed: {exc}") from exc + baseline[category] = {str(row.id) for row in rows if getattr(row, "id", None) is not None and matches(row)} + + if "work_item_types" in wanted: + api = getattr(plane, "workspace_work_item_types", None) + if callable(getattr(api, "list", None)): + try: + rows = ( + list_workspace_work_item_types(plane, workspace_slug) + if workspace_owns_work_item_types(plane, workspace_slug) + else [] + ) + except Exception as exc: + raise RuntimeError(f"workspace baseline snapshot: list work_item_types failed: {exc}") from exc + baseline["work_item_types"] = { + str(row.id) + for row in rows + if getattr(row, "id", None) is not None + and (is_work_item_type_named(row, BUG_TYPE_NAME) or is_work_item_type_named(row, INCIDENT_TYPE_NAME)) + } + + if "work_item_properties" in wanted: + type_api = getattr(plane, "workspace_work_item_types", None) + property_api = getattr(plane, "workspace_work_item_properties", None) + links_api = getattr(type_api, "properties", None) + if ( + callable(getattr(type_api, "list", None)) + and callable(getattr(links_api, "list", None)) + and callable(getattr(property_api, "list", None)) + ): + try: + rows = ( + list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME) + if workspace_owns_work_item_types(plane, workspace_slug) + else [] + ) + except Exception as exc: + raise RuntimeError(f"workspace baseline snapshot: list work_item_properties failed: {exc}") from exc + baseline["work_item_properties"] = { + str(row.id) for row in rows if getattr(row, "id", None) is not None and is_severity_property(row) + } + return baseline + + +def _raise_fixture_collision(category: str, name: str, object_id: Any) -> None: + raise TaskSkipped( + f"env:fixture-collision:{category}:{name}; pre-existing object id={object_id}; " + "run `python -m evals.cleanup --sentinels --yes`, then retry" + ) + + +def collision_categories(needs: set[str], task_id: str | None) -> set[str]: + categories: set[str] = set() + if "customer" in needs: + categories.update({"customers", "customer_properties"}) + categories.update(_TASK_COLLISION_CATEGORIES.get(task_id or "", set())) + return categories + + +def check_workspace_fixture_collisions( + plane: PlaneClient, + workspace_slug: str, + categories: set[str], +) -> None: + """Reject fixed-name workspace artifacts that would let a no-op agent false-pass. + + Missing API surfaces are silent. List failures raise so an unread workspace is never + mistaken for a clean one. + """ + customers = getattr(plane, "customers", None) + releases = getattr(plane, "releases", None) + specs = ( + ( + "customers", + customers if callable(getattr(customers, "list", None)) else None, + CUSTOMER_NAME, + lambda row: is_evaluation_customer_name(getattr(row, "name", None)), + ), + ( + "release_tags", + getattr(releases, "tags", None) if releases is not None else None, + EVALUATION_RELEASE_TAG_VERSION, + lambda row: (getattr(row, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION, + ), + ( + "customer_properties", + getattr(customers, "properties", None) if customers is not None else None, + EVALUATION_CUSTOMER_PROPERTY_NAME, + lambda row: ( + (getattr(row, "display_name", None) or getattr(row, "name", None) or "").strip().casefold() + == EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + ), + ), + ) + for category, api, fixture_name, matches in specs: + if category not in categories: + continue + if not callable(getattr(api, "list", None)): + continue + try: + rows = list_workspace_rows(api, workspace_slug) + except Exception as exc: + raise RuntimeError(f"workspace fixture collision check: list {category} failed: {exc}") from exc + collision = next((row for row in rows if matches(row)), None) + if collision is not None: + _raise_fixture_collision(category, fixture_name, getattr(collision, "id", "unknown")) + + if "work_item_types" in categories: + api = getattr(plane, "workspace_work_item_types", None) + if callable(getattr(api, "list", None)): + try: + if not workspace_owns_work_item_types(plane, workspace_slug): + rows = [] + else: + rows = list_workspace_work_item_types(plane, workspace_slug) + except Exception as exc: + raise RuntimeError(f"workspace fixture collision check: list work_item_types failed: {exc}") from exc + collision = next((row for row in rows if is_work_item_type_named(row, INCIDENT_TYPE_NAME)), None) + if collision is not None: + _raise_fixture_collision( + "work_item_types", + INCIDENT_TYPE_NAME, + getattr(collision, "id", "unknown"), + ) + + if "work_item_properties" in categories: + type_api = getattr(plane, "workspace_work_item_types", None) + property_api = getattr(plane, "workspace_work_item_properties", None) + links_api = getattr(type_api, "properties", None) + if ( + callable(getattr(type_api, "list", None)) + and callable(getattr(links_api, "list", None)) + and callable(getattr(property_api, "list", None)) + ): + try: + if not workspace_owns_work_item_types(plane, workspace_slug): + rows = [] + else: + rows = list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME) + except Exception as exc: + raise RuntimeError( + f"workspace fixture collision check: list work_item_properties failed: {exc}" + ) from exc + collision = next((row for row in rows if is_severity_property(row)), None) + if collision is not None: + _raise_fixture_collision( + "work_item_properties", + SEVERITY_PROPERTY_NAME, + getattr(collision, "id", "unknown"), + ) + + +def seed( + plane: PlaneClient, + run_id: str, + needs: set[str], + ctx: dict[str, Any], + *, + task_id: str | None = None, +) -> dict[str, Any]: + """Create the eval project and declared fixture groups. + + Mutates the caller-provided `ctx` in place so project_id is visible to teardown + even if a later fixture step raises (F5). + """ + run_prefix = run_id[:8] + name_variants = eval_project_name_variants(run_prefix) + project_name = next(name_variants) + workspace_slug = os.environ["EVAL_PLANE_WORKSPACE_SLUG"] + + # Reset known keys while preserving object identity for the caller. + ctx.clear() + ctx.update( + { + "run_id": run_id, + "run8": run_prefix, + "task_id": task_id, + "workspace_slug": workspace_slug, + "project_id": None, + "project_name": project_name, + "project_identifier": None, # filled after create (may retry suffix) + "labels": {}, + "items": {}, + "item_identifiers": {}, # title -> PROJ-N for ID-in-hand prompts + "item_ids": [], + "fixture_item_ids": {}, # stable fixture title -> API-created id + "fixture_item_titles": {}, # stable fixture title -> per-run display title + "state_names": [], # all project state display names (for R1 negative check) + "r1_state_name": None, + "bug_type": None, + "bug_type_created": False, + "bug_type_workspace_level": False, + "bug_type_skip_reason": None, + "cycles": {}, + "module": None, + "module_completed_ids": [], + "intake": {}, + "customer": None, + "customer_request": None, + "release": None, + "second_project_id": None, + "second_project_name": None, + "r3_due_titles": list(DUE_THIS_WEEK_TITLES), + "r3_due_count": len(DUE_THIS_WEEK_TITLES), + "r5_title": CHECKOUT_TIMEOUT_TITLE, + "r5_comment_phrases": list(CHECKOUT_COMMENT_PHRASES), + "w6_unfinished_titles": list(UNFINISHED_CYCLE_TITLES), + "workspace_objects": [], # [{kind, id}, ...] surviving project delete + "randomized_truth": {}, + "randomized_truth_namespaces": set(), + "seeded_entity_kinds": set(), + "evidence_sentinels": {}, + "evidence_targets": {}, + "evidence_aggregates": {}, + # None means the category was unavailable and name-based teardown must fail closed. + "workspace_baseline": dict.fromkeys(_WORKSPACE_BASELINE_CATEGORIES), + } + ) + + # Reject only artifacts that could false-pass this task. + task_collision_categories = collision_categories(needs, task_id) + check_workspace_fixture_collisions(plane, workspace_slug, task_collision_categories) + + # EV + 8 hex chars; a new suffix on soft-delete identifier collisions, the next name + # variant if the name itself is taken by residue from a crashed run. + project = create_project_with_collision_retry( + plane, + workspace_slug, + name=project_name, + identifier_prefix="EV", + initial_suffix=run_prefix.upper(), + name_variants=name_variants, + ) + ctx["project_id"] = project.id + record_seeded_entity(ctx, "project", project.id) + ctx["project_identifier"] = getattr(project, "identifier", None) + # The created name, not the requested one: the agent is told this in its preamble, so a + # name-collision retry that was not written back would name a project that does not exist. + ctx["project_name"] = getattr(project, "name", None) or project_name + + # Workspace first, then project. Seeding is per task-rep, so S5 turning workspace + # customers off must be undone in teardown or a later C1 rep 403s. + feature_exclude: set[str] = set() + workspace_feature_exclude: set[str] = set() + if "leave_cycles_worklogs_off" in needs: + feature_exclude = {"cycles", "worklogs"} + workspace_feature_exclude = {"customers"} + ctx["s5_left_customers_off"] = True + if "leave_worklogs_off" in needs: + # W11: only time tracking is off, so the agent meets one obstacle rather than a + # project with several unrelated features disabled. + feature_exclude |= {"worklogs"} + ctx["feature_exclude"] = sorted(feature_exclude) + ctx["ws_feature_exclude"] = sorted(workspace_feature_exclude) + # Prior values are captured before the write so teardown restores the workspace + # rather than forcing it to whatever this run happened to need. + ownership_categories = set(task_collision_categories) + if "bug_type" in needs: + ownership_categories.update({"work_item_types", "work_item_properties"}) + ctx["workspace_baseline"] = _snapshot_workspace_baseline( + plane, + workspace_slug, + ownership_categories, + ) + ctx["workspace_features_prior"] = enable_workspace_features( + plane, workspace_slug, exclude=workspace_feature_exclude + ) + enable_project_features(plane, workspace_slug, project.id, exclude=feature_exclude) + + if task_id == "R7": + seed_r7_state_oracle(plane, workspace_slug, ctx) + + # Labels before items so items can attach labels later if needed. + if "labels" in needs: + seed_labels(plane, workspace_slug, ctx) + if "items" in needs: + seed_work_items(plane, workspace_slug, ctx) + # L2: comments must materialize as activities (activity worker must be running). + if "activity_feed" in needs: + if "items" not in needs and not ctx.get("item_ids"): + seed_work_items(plane, workspace_slug, ctx) + require_activities(plane, workspace_slug, ctx) + if "bug_type" in needs: + seed_item_type(plane, workspace_slug, ctx) + if "cycles" in needs: + # Cycles need items to attach unfinished work; seed items if not already. + if "items" not in needs and not ctx["item_ids"]: + seed_work_items(plane, workspace_slug, ctx) + seed_cycles(plane, workspace_slug, ctx, leave_past_open="cycles_open_past" in needs) + if "module" in needs: + seed_module(plane, workspace_slug, ctx) + if "intake" in needs: + seed_intake(plane, workspace_slug, ctx) + if "customer" in needs: + seed_customer(plane, workspace_slug, ctx) + if "release" in needs: + seed_release(plane, workspace_slug, ctx) + if "second_project" in needs: + seed_second_project(plane, workspace_slug, ctx) + + return ctx diff --git a/evals/seed/client.py b/evals/seed/client.py new file mode 100644 index 0000000..9d14b3e --- /dev/null +++ b/evals/seed/client.py @@ -0,0 +1,18 @@ +"""Plane client construction for evaluation fixture runs.""" + +from __future__ import annotations + +import os + +from plane import PlaneClient + + +def make_plane_client() -> tuple[PlaneClient, str]: + """Build a PlaneClient from EVAL_* env vars (mirrors stdio client construction).""" + api_key = os.environ.get("EVAL_PLANE_API_KEY", "") + workspace_slug = os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") + base_url = os.environ.get("EVAL_PLANE_BASE_URL", "https://api.plane.so") + if not api_key or not workspace_slug: + raise RuntimeError("EVAL_PLANE_API_KEY and EVAL_PLANE_WORKSPACE_SLUG are required for live runs") + client = PlaneClient(base_url=base_url, api_key=api_key) + return client, workspace_slug diff --git a/evals/seed/customers.py b/evals/seed/customers.py new file mode 100644 index 0000000..18f6111 --- /dev/null +++ b/evals/seed/customers.py @@ -0,0 +1,49 @@ +"""Customer fixtures for evaluation workspaces.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.customers import CreateCustomer, CreateCustomerRequest + +from evals.core.fixtures import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + is_evaluation_customer_name, +) + +from .gates import plan_gate_skips +from .identities import record_seeded_entity + +__all__ = [ + "CUSTOMER_NAME", + "CUSTOMER_REQUEST_NAME", + "EVALUATION_CUSTOMER_PROPERTY_NAME", + "is_evaluation_customer_name", + "seed_customer", +] + + +def seed_customer(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Seed the L4 customer fixture, skipping the task when the plan excludes customers.""" + with plan_gate_skips("customers"): + customer = plane.customers.create( + workspace_slug=workspace_slug, + data=CreateCustomer(name=CUSTOMER_NAME), + ) + context["customer"] = {"id": customer.id, "name": CUSTOMER_NAME} + record_seeded_entity(context, "customer", customer.id) + context["workspace_objects"].append({"kind": "customer", "id": customer.id}) + request = plane.customers.requests.create( + workspace_slug=workspace_slug, + customer_id=customer.id, + data=CreateCustomerRequest(name=CUSTOMER_REQUEST_NAME), + ) + context["customer_request"] = { + "id": request.id, + "name": CUSTOMER_REQUEST_NAME, + "customer_id": customer.id, + } + record_seeded_entity(context, "customer_request", request.id) diff --git a/evals/seed/cycles.py b/evals/seed/cycles.py new file mode 100644 index 0000000..d29313b --- /dev/null +++ b/evals/seed/cycles.py @@ -0,0 +1,202 @@ +"""Cycle fixtures for evaluation projects.""" + +from __future__ import annotations + +from datetime import date, timedelta +from typing import Any + +from plane import PlaneClient +from plane.models.cycles import CreateCycle, UpdateCycle +from plane.models.work_items import UpdateWorkItem + +from evals.core.evidence import set_target_evidence +from evals.core.fixtures import CYCLE_CURRENT, CYCLE_PAST, PAYMENT_WEBHOOK_TITLE, UNFINISHED_CYCLE_TITLES + +from .identities import record_seeded_entity +from .randomize import random_truth_rng, record_randomized_truth + + +def seed_cycles( + plane: PlaneClient, + workspace_slug: str, + context: dict[str, Any], + leave_past_open: bool = False, +) -> None: + """Seed Sprint 12 (past) + Sprint 13 (active) with work items. + + Plane refuses to add issues to an ended cycle, so Sprint 12 is created with an active + window, filled, then backdated. ``leave_past_open`` skips the backdate: Plane also + rejects every edit to an ended cycle, so pre-closing it makes W6's "close it" + unachievable and leaves progress_snapshot (a transfer side effect) as the only signal. + """ + project_id = context["project_id"] + task_id = str(context.get("task_id") or "") + past_name = CYCLE_PAST + current_name = CYCLE_CURRENT + active_fixture_titles = [PAYMENT_WEBHOOK_TITLE, "Session cookie not rotated after login"] + overdue_fixture_title = "Session cookie not rotated after login" + if task_id == "R4": + rng = random_truth_rng(context, "R4:cycles") + current_number = rng.randrange(20, 100) + past_name = f"Sprint {current_number - 1}" + current_name = f"Sprint {current_number}" + active_candidates = [ + title for title in context.get("fixture_item_ids") or {} if title not in set(UNFINISHED_CYCLE_TITLES) + ] + active_fixture_titles = rng.sample(active_candidates, rng.randint(1, min(4, len(active_candidates)))) + overdue_fixture_title = rng.choice(active_fixture_titles) + record_randomized_truth( + context, + "R4.cycle_inventory", + { + "intended_cycle": current_name, + "intended_active_templates": list(active_fixture_titles), + "intended_overdue_template": overdue_fixture_title, + }, + ) + me_id = context.get("me_id") or str(plane.users.get_me().id) + today = date.today() + # Final past window for Sprint 12 after backdate (completedCycles / W6 transfer source). + past_start = (today - timedelta(days=28)).isoformat() + past_end_final = (today - timedelta(days=14)).isoformat() + # Temporary active end so create + add succeed (end must be ≥ now). When the + # cycle stays open this is its final window, so keep it short — Sprint 12 ends + # tomorrow, which is what makes "close it and roll the rest over" natural. + past_end_active = (today + timedelta(days=1 if leave_past_open else 7)).isoformat() + # Sprint 13: genuinely active at seed time (start ≤ today ≤ end). + current_start = (today - timedelta(days=3)).isoformat() + current_end = (today + timedelta(days=10)).isoformat() + + # 1) Create Sprint 12 still active (items can be added). + past = plane.cycles.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateCycle( + name=past_name, + start_date=past_start, + end_date=past_end_active, + owned_by=me_id, + project_id=str(project_id), + ), + ) + # Sprint 13: active window for R4 / W6 transfer target. + current = plane.cycles.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateCycle( + name=current_name, + start_date=current_start, + end_date=current_end, + owned_by=me_id, + project_id=str(project_id), + ), + ) + context["cycles"] = { + past_name: past.id, + current_name: current.id, + } + record_seeded_entity(context, "cycle", past.id) + record_seeded_entity(context, "cycle", current.id) + context["cycle_past_name"] = past_name + context["cycle_current_name"] = current_name + context["cycle_past_id"] = past.id + context["cycle_current_id"] = current.id + + # 2) Add unfinished items to Sprint 12 *before* backdating. + fixture_item_ids = context.get("fixture_item_ids") or context.get("items") or {} + unfinished_ids = [fixture_item_ids[title] for title in UNFINISHED_CYCLE_TITLES if title in fixture_item_ids] + if unfinished_ids: + plane.cycles.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=past.id, + issue_ids=unfinished_ids, + ) + # R4: items on the active cycle (window still open). + active_ids: list[str] = [] + for title in active_fixture_titles: + item_id = fixture_item_ids.get(title) + if item_id: + active_ids.append(item_id) + if active_ids: + plane.cycles.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + issue_ids=active_ids, + ) + overdue_id = fixture_item_ids.get(overdue_fixture_title) + if overdue_id: + plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=overdue_id, + data=UpdateWorkItem(target_date=(today - timedelta(days=3)).isoformat()), + ) + context["r4_overdue_title"] = (context.get("fixture_item_titles") or {}).get( + overdue_fixture_title, overdue_fixture_title + ) + context["r4_overdue_id"] = overdue_id + context["r4_active_item_ids"] = active_ids + + # 3) Backdate Sprint 12 so it is a completed cycle for R4 semantics — unless the + # task needs to close it itself, in which case it must still be open. + # UpdateCycle.end_date is writable; API allows past end_dates (no "can't backdate" gate + # on the update path — only add_work_items checks end_date < now). + if not leave_past_open: + plane.cycles.update( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=past.id, + data=UpdateCycle(end_date=past_end_final), + ) + # Final seeded end_date for W6 close assertion (complete_cycle sets end_date=today). + context["cycle_past_seed_end_date"] = past_end_active if leave_past_open else past_end_final + context["cycle_past_open"] = leave_past_open + context["cycle_past_end_date_before_backdate"] = past_end_active + + if task_id == "R4": + confirmed_cycle = plane.cycles.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + ) + confirmed_rows_page = plane.cycles.list_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=current.id, + ) + confirmed_rows = confirmed_rows_page.results if hasattr(confirmed_rows_page, "results") else confirmed_rows_page + confirmed_active_titles: list[str] = [] + confirmed_overdue_titles: list[str] = [] + for row in confirmed_rows or []: + item_id = getattr(row, "work_item_id", None) or getattr(row, "issue", None) or getattr(row, "id", None) + if hasattr(item_id, "id"): + item_id = item_id.id + if not item_id: + continue + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=str(item_id), + ) + name = str(getattr(detail, "name", None) or "").strip() + if not name: + raise RuntimeError(f"seed R4: active item {item_id} readback has no name") + confirmed_active_titles.append(name) + target_date = str(getattr(detail, "target_date", None) or "")[:10] + if target_date and target_date < today.isoformat(): + confirmed_overdue_titles.append(name) + if not confirmed_active_titles: + raise RuntimeError("seed R4: API readback found no active-cycle items") + context["r4_cycle_name"] = str(getattr(confirmed_cycle, "name", None) or "") + if not context["r4_cycle_name"]: + raise RuntimeError("seed R4: API readback returned an active cycle without a name") + context["r4_active_titles"] = confirmed_active_titles + context["r4_overdue_titles"] = confirmed_overdue_titles + context["randomized_truth"]["R4.cycle_inventory"]["confirmed"] = { + "cycle": context["r4_cycle_name"], + "active_titles": list(confirmed_active_titles), + "overdue_titles": list(confirmed_overdue_titles), + } + set_target_evidence(context, [context["r4_cycle_name"], *confirmed_active_titles]) diff --git a/evals/seed/gates.py b/evals/seed/gates.py new file mode 100644 index 0000000..872ca9d --- /dev/null +++ b/evals/seed/gates.py @@ -0,0 +1,61 @@ +"""Plan-gate classification, shared by every seeder that can meet a paid feature. + +This is policy, not a resource. It lived in ``projects`` because the first gate encountered +was a project one, and every other seeder then imported the project module to reach it — +which made ``projects`` a hub and produced the package's only import cycle, since +``item_types`` needs the classifier while ``projects`` needs the item-type seeder. +""" + +from __future__ import annotations + +import contextlib +from collections.abc import Iterator + +from plane.errors.errors import HttpError + +from evals.core.errors import TaskSkipped + +# Wording a refusal uses when the workspace's plan is what stands in the way. A feature +# switched off for a project says "not enabled for this project" instead, which is a +# configuration state the harness can change and so is not a gate. +PLAN_GATE_PROSE = ("upgrade your plan", "payment required", "subscription", "not available on your") + + +def is_plan_gate(exc: BaseException) -> bool: + """True only for genuine plan gates — not generic API failures. + + 402 is unambiguous. 403 and 400 are not: Plane uses 403 for ordinary permission denial + and for the initiative/teamspace plan gates in the same shape, so a bare 403 counted as + a gate turned real permission bugs into environment skips. Those two now need the + refusal to name a plan limit. + """ + if not isinstance(exc, HttpError): + return False + if exc.status_code == 402: + return True + if exc.status_code not in (400, 403): + return False + blob = f"{exc} {exc.response!s}".lower() + return any(phrase in blob for phrase in PLAN_GATE_PROSE) + + +@contextlib.contextmanager +def plan_gate_skips(feature: str) -> Iterator[None]: + """Turn a plan refusal raised inside the block into a task skip. + + An uncaught seed exception becomes infra_seed and kills the task-rep; a capability the + plan excludes is an environment fact, recorded like L2's missing activity worker. + ``TaskSkipped`` lives in a neutral module, so seed and task packages can import in + either order without a cycle. + """ + try: + yield + except Exception as exc: + if is_plan_gate(exc): + status = getattr(exc, "status_code", None) + detail = f"HTTP {status}: {exc}" if status else str(exc) + raise TaskSkipped(f"env:plan-gated:{feature}", detail=detail[:300]) from exc + raise + + +__all__ = ["PLAN_GATE_PROSE", "is_plan_gate", "plan_gate_skips"] diff --git a/evals/seed/identities.py b/evals/seed/identities.py new file mode 100644 index 0000000..49aee30 --- /dev/null +++ b/evals/seed/identities.py @@ -0,0 +1,30 @@ +"""Non-secret seed-shape metadata safe to persist beside evaluation results.""" + +from __future__ import annotations + +from typing import Any + + +def record_seeded_entity(context: dict[str, Any], kind: str, object_id: Any) -> None: + """Register that a fixture kind was seeded without retaining its target id.""" + value = str(object_id or "").strip() + if not value: + return + context.setdefault("seeded_entity_kinds", set()).add(str(kind)) + + +def capture_seed_artifacts(context: dict[str, Any]) -> tuple[list[str], list[str]]: + """Copy only fixture kinds and randomization namespaces, never ids or truth values.""" + entity_kinds = context.get("seeded_entity_kinds") + randomized_namespaces = context.get("randomized_truth_namespaces") + return ( + sorted({str(kind) for kind in entity_kinds}) if isinstance(entity_kinds, (list, set, tuple)) else [], + ( + sorted({str(namespace) for namespace in randomized_namespaces}) + if isinstance(randomized_namespaces, (list, set, tuple)) + else [] + ), + ) + + +__all__ = ["capture_seed_artifacts", "record_seeded_entity"] diff --git a/evals/seed/intake.py b/evals/seed/intake.py new file mode 100644 index 0000000..1c35eff --- /dev/null +++ b/evals/seed/intake.py @@ -0,0 +1,46 @@ +"""Intake fixtures for evaluation projects.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.intake import CreateIntakeWorkItem, WorkItemForIntakeRequest + +from evals.core.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE + +from .identities import record_seeded_entity + + +def seed_intake(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + project_id = context["project_id"] + billing = plane.intake.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateIntakeWorkItem( + issue=WorkItemForIntakeRequest(name=INTAKE_BILLING_TITLE, priority="high"), + ), + ) + spam = plane.intake.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateIntakeWorkItem( + issue=WorkItemForIntakeRequest(name=INTAKE_SPAM_TITLE, priority="none"), + ), + ) + # IntakeWorkItem.issue is the work-item id used by triage tools. + context["intake"] = { + "billing": { + "intake_id": billing.id, + "issue_id": getattr(billing, "issue", None) or billing.id, + "title": INTAKE_BILLING_TITLE, + }, + "spam": { + "intake_id": spam.id, + "issue_id": getattr(spam, "issue", None) or spam.id, + "title": INTAKE_SPAM_TITLE, + }, + } + for row in (billing, spam): + record_seeded_entity(context, "intake", row.id) + record_seeded_entity(context, "work_item", getattr(row, "issue", None) or row.id) diff --git a/evals/seed/item_types.py b/evals/seed/item_types.py new file mode 100644 index 0000000..98e928e --- /dev/null +++ b/evals/seed/item_types.py @@ -0,0 +1,138 @@ +"""Work item type fixtures for evaluation projects.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.work_item_types import CreateWorkItemType + +from .gates import is_plan_gate +from .identities import record_seeded_entity + +BUG_TYPE_NAME = "Bug" +INCIDENT_TYPE_NAME = "Incident" +# Every workspace-level work item type name this harness creates. Cleanup deletes these and +# reports anything else it finds, because the harness runs against an instance it does not own. +FIXTURE_WORK_ITEM_TYPE_NAMES = (BUG_TYPE_NAME, INCIDENT_TYPE_NAME) +SEVERITY_PROPERTY_NAME = "Severity" + + +def is_work_item_type_named(row: Any, name: str) -> bool: + """Return whether a type row has the exact fixture name, ignoring case/space.""" + return (getattr(row, "name", None) or "").strip().casefold() == name.casefold() + + +def is_severity_property(row: Any) -> bool: + """Return whether a property row is the S1 Severity fixture.""" + display = getattr(row, "display_name", None) or getattr(row, "name", None) or "" + return display.strip().casefold() == SEVERITY_PROPERTY_NAME.casefold() + + +def list_workspace_work_item_types(plane: PlaneClient, workspace_slug: str) -> list[Any]: + """List workspace-owned work-item types using the SDK's non-paginated surface.""" + result = plane.workspace_work_item_types.list(workspace_slug=workspace_slug) + return list((result.results if hasattr(result, "results") else result) or []) + + +def workspace_owns_work_item_types(plane: PlaneClient, workspace_slug: str) -> bool: + """Return the authoritative workspace-vs-project ownership mode for types.""" + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + return bool(dump.get("is_work_item_types_enabled")) + + +def list_workspace_properties_for_type( + plane: PlaneClient, + workspace_slug: str, + type_name: str, +) -> list[Any]: + """Resolve full workspace property rows linked to every type named ``type_name``.""" + item_types = list_workspace_work_item_types(plane, workspace_slug) + target_types = [row for row in item_types if is_work_item_type_named(row, type_name)] + if not target_types: + return [] + + linked_ids: set[str] = set() + for item_type in target_types: + linked = plane.workspace_work_item_types.properties.list( + workspace_slug=workspace_slug, + type_id=item_type.id, + ) + for value in linked or []: + object_id = getattr(value, "id", None) or value + linked_ids.add(str(object_id)) + + properties = plane.workspace_work_item_properties.list(workspace_slug=workspace_slug) + rows = list((properties.results if hasattr(properties, "results") else properties) or []) + return [row for row in rows if getattr(row, "id", None) is not None and str(row.id) in linked_ids] + + +def seed_item_type(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Create or resolve a 'Bug' work item type. + + Genuine plan-gate responses set bug_type=None + skip reason; all other failures raise. + Workspace feature probe uses the real key `is_work_item_types_enabled` (F10). + """ + project_id = context["project_id"] + target = BUG_TYPE_NAME + try: + # Real API key (extra='allow' on WorkspaceFeature); never trust the fictional + # work_item_types key alone. + workspace_owns = workspace_owns_work_item_types(plane, workspace_slug) + + if workspace_owns: + existing = next( + ( + item_type + for item_type in plane.workspace_work_item_types.list(workspace_slug=workspace_slug) + if (item_type.name or "").strip() == target + ), + None, + ) + created = False + if existing is None: + existing = plane.workspace_work_item_types.create( + workspace_slug=workspace_slug, data=CreateWorkItemType(name=target) + ) + created = True + plane.work_item_types.import_to_project( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_type_ids=[existing.id], + ) + context["bug_type"] = {"id": existing.id, "name": target} + record_seeded_entity(context, "work_item_type", existing.id) + context["bug_type_created"] = created + context["bug_type_workspace_level"] = True + if created: + context["workspace_objects"].append({"kind": "work_item_type", "id": existing.id}) + return + + # Per-project types. Project features expose no work-item-type toggle — do not PATCH. + existing = next( + ( + item_type + for item_type in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) + if (item_type.name or "").strip() == target + ), + None, + ) + created = False + if existing is None: + existing = plane.work_item_types.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItemType(name=target), + ) + created = True + context["bug_type"] = {"id": existing.id, "name": target} + record_seeded_entity(context, "work_item_type", existing.id) + context["bug_type_created"] = created + context["bug_type_workspace_level"] = False + except Exception as exc: + if is_plan_gate(exc): + context["bug_type"] = None + context["bug_type_skip_reason"] = "env:plan-gated:work-item-types" + return + raise diff --git a/evals/seed/labels.py b/evals/seed/labels.py new file mode 100644 index 0000000..8ee9b01 --- /dev/null +++ b/evals/seed/labels.py @@ -0,0 +1,23 @@ +"""Label fixtures for evaluation projects.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.labels import CreateLabel + +from .identities import record_seeded_entity + +LABEL_NAMES = ("auth", "triage", "perf") + + +def seed_labels(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + for name in LABEL_NAMES: + label = plane.labels.create( + workspace_slug=workspace_slug, + project_id=context["project_id"], + data=CreateLabel(name=name), + ) + context["labels"][name] = label.id + record_seeded_entity(context, "label", label.id) diff --git a/evals/seed/modules.py b/evals/seed/modules.py new file mode 100644 index 0000000..1d1d311 --- /dev/null +++ b/evals/seed/modules.py @@ -0,0 +1,59 @@ +"""Fixtures for the Plane Module object, not for Python modules.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.modules import CreateModule +from plane.models.work_items import CreateWorkItem, UpdateWorkItem + +from evals.core.fixtures import MODULE_COMPLETED_TITLES, MODULE_NAME + +from .identities import record_seeded_entity +from .work_items import find_completed_state, list_states + + +def seed_module(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + project_id = context["project_id"] + states = list_states(plane, workspace_slug, project_id) + done = find_completed_state(states) + if done is None: + raise RuntimeError("seed module: no completed-group state to place module items") + + module = plane.modules.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateModule(name=MODULE_NAME, status="in-progress"), + ) + context["module"] = {"id": module.id, "name": MODULE_NAME} + record_seeded_entity(context, "module", module.id) + completed_ids: list[str] = [] + for title in MODULE_COMPLETED_TITLES: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItem(name=title, priority="medium", state=str(done.id)), # type: ignore[arg-type] + ) + # Force completed state if create ignored it. + current = getattr(item, "state", None) + current_id = current if isinstance(current, str) else getattr(current, "id", None) + if str(current_id) != str(done.id): + item = plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + data=UpdateWorkItem(state=str(done.id)), + ) + completed_ids.append(item.id) + context["items"][title] = item.id + context["item_ids"].append(item.id) + record_seeded_entity(context, "work_item", item.id) + plane.modules.add_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + module_id=module.id, + issue_ids=completed_ids, + ) + context["module_completed_ids"] = completed_ids + context["module_completed_state_id"] = done.id diff --git a/evals/seed/plan.py b/evals/seed/plan.py new file mode 100644 index 0000000..f42edbf --- /dev/null +++ b/evals/seed/plan.py @@ -0,0 +1,71 @@ +"""Human-readable plans for evaluation fixture creation.""" + +from __future__ import annotations + +from .customers import CUSTOMER_NAME, CUSTOMER_REQUEST_NAME +from .cycles import CYCLE_CURRENT, CYCLE_PAST +from .intake import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE +from .labels import LABEL_NAMES +from .modules import MODULE_COMPLETED_TITLES, MODULE_NAME +from .releases import RELEASE_NAME +from .work_items import ( + CHECKOUT_TIMEOUT_TITLE, + PAYMENT_WEBHOOK_TITLE, + WORK_ITEM_FIXTURES, +) + + +def seed_plan(needs: set[str]) -> list[str]: + """Human-readable seed plan for --dry-run (no network).""" + lines = [ + "project: EVAL Delivery Planning {Word} (identifier EV{XXXX})", + ] + if "items" in needs: + lines.append(f"items: {len(WORK_ITEM_FIXTURES)} work items (read truth randomised per row; default 4 urgent)") + lines.append(f" - {PAYMENT_WEBHOOK_TITLE!r} (random started-group state for R1)") + lines.append(" - random assigned-to-me due-this-week selection # R3") + lines.append(f" - random comments on {CHECKOUT_TIMEOUT_TITLE!r} # R5/L2") + lines.append(f" - random attachment count on {PAYMENT_WEBHOOK_TITLE!r} # L5") + if "activity_feed" in needs: + lines.append( + f"activity_feed: gate that activities exist for {CHECKOUT_TIMEOUT_TITLE!r} " + "(TaskSkipped env:no-activity-worker if empty) # L2" + ) + if "labels" in needs: + lines.append(f"labels: {', '.join(LABEL_NAMES)}") + if "bug_type" in needs: + lines.append( + "bug_type: work item type 'Bug' (genuine plan-gate only → skip dependents; other seed errors raise)" + ) + if "cycles" in needs: + past_state = "ends tomorrow, still OPEN so it can be closed" if "cycles_open_past" in needs else "past-dated" + lines.append( + f"cycles: default {CYCLE_PAST!r} ({past_state}) + {CYCLE_CURRENT!r} (current); " + "R4 names/inventory randomised" + ) + if "module" in needs: + lines.append(f"module: {MODULE_NAME!r} with {len(MODULE_COMPLETED_TITLES)} completed items") + if "intake" in needs: + lines.append(f"intake: billing {INTAKE_BILLING_TITLE!r} + spam {INTAKE_SPAM_TITLE!r}") + if "customer" in needs: + lines.append(f"customer: {CUSTOMER_NAME!r} + request {CUSTOMER_REQUEST_NAME!r}") + if "release" in needs: + lines.append(f"release: {RELEASE_NAME!r} with changelog body (2 entries as plain text)") + if "second_project" in needs: + lines.append( + "second_project: EVAL Platform Migration {Word} " + "with random unequal open Bug counts across both projects (R6)" + ) + if "leave_cycles_worklogs_off" in needs: + lines.append( + "feature_exclusions (S5): project cycles+worklogs OFF; workspace customers OFF " + "(agent enables; teardown re-enables customers=True for later C1)" + ) + elif "leave_worklogs_off" in needs: + lines.append("feature_exclusions (W11): project worklogs OFF (agent enables); workspace customers=True") + else: + lines.append( + "workspace_features: customers=True " + "(is_customer_enabled; NOT work_item_types — leaves S1/S3 type mode alone)" + ) + return lines diff --git a/evals/seed/projects.py b/evals/seed/projects.py new file mode 100644 index 0000000..e51d61a --- /dev/null +++ b/evals/seed/projects.py @@ -0,0 +1,413 @@ +"""Project creation and feature setup for evaluation fixtures.""" + +from __future__ import annotations + +import secrets +from collections.abc import Iterator +from typing import Any + +from plane import PlaneClient +from plane.errors.errors import HttpError +from plane.models.projects import CreateProject, ProjectFeature, UpdateProject +from plane.models.query_params import PaginatedQueryParams +from plane.models.work_items import CreateWorkItem +from plane.models.workspaces import WorkspaceFeature + +from evals.core.evidence import set_target_count_evidence, set_target_evidence, set_target_grouped_count_evidence +from evals.core.fixtures import eval_project_name_variants + +from .gates import is_plan_gate +from .identities import record_seeded_entity +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + +# Plane's project identifier field is capped at 12 characters. Keep two characters +# for the eval prefix and use eight hex characters (32 bits), leaving two spare. +PLANE_PROJECT_IDENTIFIER_MAX_LENGTH = 12 +PROJECT_IDENTIFIER_SUFFIX_LENGTH = 8 +# Soft-deleted projects reserve identifiers; a long-lived workspace needs more than +# three chances even with the larger suffix space. +PROJECT_CREATE_ATTEMPT_LIMIT = 8 + +MAIN_PROJECT_BUG_TITLES = ("Main bug alpha", "Main bug beta") +SECOND_PROJECT_BUG_TITLES = ( + "Second bug one", + "Second bug two", + "Second bug three", + "Second bug four", +) + + +def _is_collision(exc: BaseException) -> bool: + """True for an HTTP 400/409 whose body reads as a uniqueness conflict.""" + if not isinstance(exc, HttpError): + return False + if exc.status_code not in (400, 409): + return False + blob = f"{exc} {exc.response!s}".lower() + return any(keyword in blob for keyword in ("already", "exists", "taken")) + + +def is_name_collision(exc: BaseException) -> bool: + """True when project create failed because the project *name* is already taken. + + Plane answers both conflicts with the same 409 and the same collision wording -- + ``name: The project name is already taken`` against + ``identifier: ...`` -- so the named field is the only thing separating them. A body + mentioning the identifier is treated as an identifier collision, because retrying a + fresh suffix is cheap and was the behaviour before names could be retried at all. + """ + if not _is_collision(exc): + return False + blob = f"{exc} {exc.response!s}".lower() + return "name" in blob and "identifier" not in blob + + +def is_identifier_collision(exc: BaseException) -> bool: + """True when project create failed because the identifier is already taken. + + Requires HTTP 400/409 *and* collision language (already/exists/taken). A bare + ``identifier`` mention (validation shape errors) must not trigger retry. + """ + return _is_collision(exc) and not is_name_collision(exc) + + +def find_project_by_identifier(plane: PlaneClient, workspace_slug: str, identifier: str) -> Any | None: + """Return the project holding this exact identifier, or None. + + Identifiers are unique per workspace, so this settles the one question an ambiguous + create leaves open: did the server create it before the client stopped waiting? + """ + cursor = None + while True: + page = plane.projects.list( + workspace_slug=workspace_slug, + params=PaginatedQueryParams(per_page=100, cursor=cursor), + ) + results = page.results if hasattr(page, "results") else page + for proj in results or []: + if (getattr(proj, "identifier", None) or "").strip().upper() == identifier.strip().upper(): + return proj + # The SDK always populates next_cursor, so paging must stop on next_page_results. + if not getattr(page, "next_page_results", False): + return None + cursor = page.next_cursor + + +def create_project_with_collision_retry( + plane: PlaneClient, + workspace_slug: str, + *, + name: str, + identifier_prefix: str, + initial_suffix: str, + name_variants: Iterator[str] | None = None, +) -> Any: + """Create a project, retrying past both kinds of uniqueness conflict. + + Plane soft-deletes reserve identifiers, so an identifier collision draws a new random + 8-char hex suffix. A *name* collision means a project of that name already exists -- + residue from a crashed run, since teardown deletes by recorded id -- and the suffix was + never the problem, so it advances to the next name from ``name_variants`` instead. + Without ``name_variants`` a name collision raises immediately rather than burning the + budget regenerating an identifier that was already fine. + + Both kinds share the ``PROJECT_CREATE_ATTEMPT_LIMIT`` budget, so this survives up to + seven collisions of either kind in one create; past that the last error is re-raised. + """ + if len(identifier_prefix) + PROJECT_IDENTIFIER_SUFFIX_LENGTH > PLANE_PROJECT_IDENTIFIER_MAX_LENGTH: + raise ValueError( + f"identifier prefix {identifier_prefix!r} leaves fewer than " + f"{PROJECT_IDENTIFIER_SUFFIX_LENGTH} suffix characters under Plane's " + f"{PLANE_PROJECT_IDENTIFIER_MAX_LENGTH}-character limit" + ) + suffix = (initial_suffix or "")[:PROJECT_IDENTIFIER_SUFFIX_LENGTH].upper() + if len(suffix) < PROJECT_IDENTIFIER_SUFFIX_LENGTH: + suffix = (suffix + secrets.token_hex(4).upper())[:PROJECT_IDENTIFIER_SUFFIX_LENGTH] + last_exc: BaseException | None = None + for _attempt in range(PROJECT_CREATE_ATTEMPT_LIMIT): + identifier = f"{identifier_prefix}{suffix}" + try: + return plane.projects.create( + workspace_slug=workspace_slug, + data=CreateProject(name=name, identifier=identifier), + ) + except Exception as exc: + if is_name_collision(exc): + next_name = next(name_variants, None) if name_variants is not None else None + if next_name is None: + raise + name = next_name + last_exc = exc + continue + if is_identifier_collision(exc): + suffix = secrets.token_hex(4).upper() # 8 hex chars / 32 bits + last_exc = exc + continue + # An exception carrying no HTTP status means the client never learned the outcome: + # the server may well have created the project before the read timed out. That is + # how the orphans got there. The id was never returned, so the caller never put it + # in the teardown context, so teardown was never asked to delete it -- and reported + # cleanup_error 0 truthfully while a project sat in the workspace, skewing every + # later workspace-wide task. Adopting the project both removes the orphan and turns + # a lost row into a normal one. + # + # Limited to no-response errors on purpose. A 5xx is also ambiguous in principle, + # but the measured failure is a client-side read timeout, and treating every HTTP + # error as maybe-created would adopt projects after refusals that created nothing. + if not isinstance(exc, HttpError): + try: + adopted = find_project_by_identifier(plane, workspace_slug, identifier) + except Exception: + adopted = None # Lookup failed too; report the original failure. + if adopted is not None: + return adopted + raise + if last_exc is None: + raise RuntimeError( + f"project create failed after {PROJECT_CREATE_ATTEMPT_LIMIT} attempts " + f"(prefix={identifier_prefix!r}) with no captured exception" + ) + raise last_exc + + +def workspace_feature_state(plane: PlaneClient, workspace_slug: str) -> dict[str, bool | None]: + """Read the workspace feature toggles this module writes, so teardown can put them back. + + The API exposes ``customers``; older payloads spell it ``is_customer_enabled``. Returns + ``None`` for a value the API did not report rather than guessing a default. + """ + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + except Exception as exc: + raise RuntimeError(f"workspace feature snapshot failed before mutation: {exc}") from exc + dump = features.model_dump() if hasattr(features, "model_dump") else {} + value = dump.get("customers") + if value is None: + value = dump.get("is_customer_enabled") + if value is None: + value = getattr(features, "customers", None) + return {"customers": None if value is None else bool(value)} + + +def enable_workspace_features( + plane: PlaneClient, + workspace_slug: str, + *, + exclude: set[str] | frozenset[str] | None = None, +) -> dict[str, bool | None]: + """Set workspace-level feature toggles, returning the prior values for teardown. + + Excluded features are written ``False``, not skipped: the workspace outlives every run, + so omitting the write silently satisfied S5's customers precondition after run one. + Never sets ``work_item_types`` — that flips type ownership and changes S1/S3 seed mode. + """ + skip = set(exclude or ()) + prior = workspace_feature_state(plane, workspace_slug) + plane.workspaces.update_features( + workspace_slug=workspace_slug, + data=WorkspaceFeature(customers="customers" not in skip), + ) + return prior + + +def enable_project_features( + plane: PlaneClient, + workspace_slug: str, + project_id: str, + *, + exclude: set[str] | frozenset[str] | None = None, +) -> None: + """Set per-project feature gates; ``exclude`` names the ones to leave off. + + Excluded features are written ``False``, not omitted: ``page_view`` defaults to True, + so omission would silently leave it on. + """ + skip = set(exclude or ()) + + update_values: dict[str, bool] = { + "cycle_view": "cycles" not in skip, + "module_view": "modules" not in skip, + "intake_view": "intakes" not in skip, + "page_view": "pages" not in skip, + "is_time_tracking_enabled": "worklogs" not in skip, + } + if update_values: + plane.projects.update( + workspace_slug=workspace_slug, + project_id=project_id, + data=UpdateProject(**update_values), + ) + + feature_values: dict[str, bool] = { + "cycles": "cycles" not in skip, + "modules": "modules" not in skip, + "intakes": "intakes" not in skip, + "pages": "pages" not in skip, + } + if feature_values: + plane.projects.update_features( + workspace_slug=workspace_slug, + project_id=project_id, + data=ProjectFeature(**feature_values), + ) + + +def _project_bug_type_id(plane: PlaneClient, workspace_slug: str, project_id: str) -> str: + """Resolve or create a project-owned Bug type, so each project answers for its own.""" + from plane.models.work_item_types import CreateWorkItemType + + from .item_types import BUG_TYPE_NAME, is_work_item_type_named + + existing = next( + ( + row + for row in plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) + if is_work_item_type_named(row, BUG_TYPE_NAME) + ), + None, + ) + if existing is None: + existing = plane.work_item_types.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItemType(name=BUG_TYPE_NAME), + ) + return str(existing.id) + + +def seed_second_project(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Seed two API-confirmed Bug counts, randomising R6's winner per row.""" + from .item_types import seed_item_type + + run_prefix = context["run8"] + variants = eval_project_name_variants(run_prefix, second=True) + name = next(variants) + project = create_project_with_collision_retry( + plane, + workspace_slug, + name=name, + identifier_prefix="EB", + initial_suffix=run_prefix.upper(), + name_variants=variants, + ) + context["second_project_id"] = project.id + record_seeded_entity(context, "project", project.id) + # The created name, not the requested one: a name collision advances to a variant. + context["second_project_name"] = getattr(project, "name", None) or name + context["second_project_identifier"] = getattr(project, "identifier", None) + enable_project_features(plane, workspace_slug, project.id) + + # Ensure Bug type exists on both projects. + if not context.get("bug_type"): + seed_item_type(plane, workspace_slug, context) + bug = context.get("bug_type") or {} + bug_id = bug.get("id") if isinstance(bug, dict) else bug + if not bug_id: + raise RuntimeError("seed second_project: bug_type required for R6 bug counts") + + # Give the second project a Bug type of its own. Workspace-owned types are shared, so + # importing is enough; project-owned types are not, and creating B's items with the main + # project's type id left them invisible to an agent that resolves 'Bug' inside B. It + # counted zero there and named the main project, always in that direction, while the + # oracle — which reads those ids back directly — saw the seeded count and disagreed. + second_bug_id = bug_id + if context.get("bug_type_workspace_level"): + try: + plane.work_item_types.import_to_project( + workspace_slug=workspace_slug, + project_id=project.id, + work_item_type_ids=[bug_id], + ) + except Exception as exc: + if not is_plan_gate(exc): + # May already be imported. + if not (isinstance(exc, HttpError) and exc.status_code in (400, 409)): + raise + else: + second_bug_id = _project_bug_type_id(plane, workspace_slug, str(project.id)) + + main_id = context["project_id"] + task_id = str(context.get("task_id") or "") + if task_id == "R6": + rng = random_truth_rng(context, "R6:project-bugs") + hidden_token = random_truth_token(context, "R6:project-bugs") + main_count, second_count = rng.sample(range(1, 6), 2) + main_titles = tuple(f"Main bug case {hidden_token}-{index + 1}" for index in range(main_count)) + second_titles = tuple(f"Second bug case {hidden_token}-{index + 1}" for index in range(second_count)) + record_randomized_truth( + context, + "R6.open_bug_counts", + {"intended_main": main_count, "intended_second": second_count}, + ) + else: + main_titles = MAIN_PROJECT_BUG_TITLES + second_titles = SECOND_PROJECT_BUG_TITLES + + main_bug_ids: list[str] = [] + for title in main_titles: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=main_id, + data=CreateWorkItem(name=title, priority="high", type_id=str(bug_id)), # type: ignore[arg-type] + ) + main_bug_ids.append(item.id) + context["items"][title] = item.id + context["item_ids"].append(item.id) + record_seeded_entity(context, "work_item", item.id) + second_bug_ids: list[str] = [] + for title in second_titles: + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project.id, + data=CreateWorkItem(name=title, priority="high", type_id=str(second_bug_id)), # type: ignore[arg-type] + ) + second_bug_ids.append(item.id) + record_seeded_entity(context, "work_item", item.id) + if task_id != "R6": + context["r6_main_bug_count"] = len(main_bug_ids) + context["r6_second_bug_count"] = len(second_bug_ids) + context["r6_more_bugs_project"] = name + return + + def confirmed_open_bug_count(project_id: str, work_item_ids: list[str], type_id: str) -> int: + count = 0 + for work_item_id in work_item_ids: + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) + if str(getattr(detail, "type_id", None) or "") != str(type_id): + continue + if getattr(detail, "completed_at", None) or getattr(detail, "archived_at", None): + continue + count += 1 + return count + + confirmed_main = confirmed_open_bug_count(main_id, main_bug_ids, str(bug_id)) + confirmed_second = confirmed_open_bug_count(str(project.id), second_bug_ids, str(second_bug_id)) + if confirmed_main == confirmed_second: + raise RuntimeError(f"seed R6: API-confirmed open Bug counts tie ({confirmed_main} each)") + main_project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=main_id) + second_project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project.id) + main_name = str(getattr(main_project, "name", None) or "") + second_name = str(getattr(second_project, "name", None) or "") + if not main_name or not second_name: + raise RuntimeError("seed R6: API project readback returned a project without a name") + context["r6_main_bug_count"] = confirmed_main + context["r6_second_bug_count"] = confirmed_second + context["r6_more_bugs_project"] = main_name if confirmed_main > confirmed_second else second_name + context["randomized_truth"]["R6.open_bug_counts"]["confirmed"] = { + "main": confirmed_main, + "second": confirmed_second, + "winner": context["r6_more_bugs_project"], + } + set_target_evidence(context, [*main_titles, *second_titles]) + # Two honest call shapes reach this answer: one count grouped by project_id, or one + # count per project. Only the grouped shape was provable, so every agent that took + # the per-project route answered correctly and scored as unproven. + set_target_grouped_count_evidence( + context, + {main_id: confirmed_main, str(project.id): confirmed_second}, + ) + set_target_count_evidence(context, confirmed_main, confirmed_second, target_ids=[main_id, project.id]) diff --git a/evals/seed/randomize.py b/evals/seed/randomize.py new file mode 100644 index 0000000..4c34fab --- /dev/null +++ b/evals/seed/randomize.py @@ -0,0 +1,46 @@ +"""Per-run hidden-truth randomisation for evaluation fixtures.""" + +from __future__ import annotations + +import hashlib +import random +from typing import Any + + +def random_truth_rng(context: dict[str, Any], namespace: str) -> random.Random: + """Return a reproducible RNG keyed by the per-repetition fixture seed and a namespace. + + The caller supplies the repetition's private fixture seed as ``context["run_id"]``. + Its full value is persisted explicitly as ``TaskResult.fixture_seed_id``, making a + failed fixture reproducible without making later repetitions' independent choices + derivable from this one. + """ + run_id = str(context.get("run_id") or "") + if not run_id: + raise RuntimeError(f"random truth {namespace}: run_id missing from seed context") + digest = hashlib.sha256(f"{run_id}:{namespace}".encode()).digest() + return random.Random(int.from_bytes(digest, "big")) + + +def random_truth_token(context: dict[str, Any], namespace: str, *, length: int = 10) -> str: + """Return a reproducible hidden token derived from the per-repetition fixture seed. + + Unlike the visible eight-character project prefix, this token depends on the full + fixture seed id and a task namespace. It gives response evidence a realistically unique + value without making failed fixture reproduction nondeterministic. + """ + run_id = str(context.get("run_id") or "") + if not run_id: + raise RuntimeError(f"random truth {namespace}: run_id missing from seed context") + if length < 8: + raise ValueError("random truth tokens must contain at least 8 hex characters") + return hashlib.sha256(f"{run_id}:{namespace}:sentinel".encode()).hexdigest()[:length] + + +def record_randomized_truth(context: dict[str, Any], key: str, value: Any) -> None: + """Retain hidden truth in memory and separately register its persistable namespace.""" + context.setdefault("randomized_truth", {})[key] = value + context.setdefault("randomized_truth_namespaces", set()).add(str(key)) + + +__all__ = ["random_truth_rng", "random_truth_token", "record_randomized_truth"] diff --git a/evals/seed/releases.py b/evals/seed/releases.py new file mode 100644 index 0000000..f82a910 --- /dev/null +++ b/evals/seed/releases.py @@ -0,0 +1,86 @@ +"""Release fixtures for evaluation workspaces.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.releases import CreateRelease, UpdateReleaseChangelog + +from evals.core.changelog import changelog_items, normalize_changelog_text +from evals.core.evidence import set_target_evidence +from evals.core.fixtures import ( + EVALUATION_RELEASE_TAG_VERSION, + RELEASE_CHANGELOG_TEXT, + RELEASE_NAME, +) + +from .gates import plan_gate_skips +from .identities import record_seeded_entity +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + +__all__ = [ + "EVALUATION_RELEASE_TAG_VERSION", + "RELEASE_CHANGELOG_TEXT", + "RELEASE_NAME", + "seed_release", +] + + +def seed_release(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Seed the C2 release fixture, skipping the task when the plan excludes releases.""" + task_id = str(context.get("task_id") or "") + release_name = RELEASE_NAME + changelog_text = RELEASE_CHANGELOG_TEXT + if task_id == "C2": + rng = random_truth_rng(context, "C2:release") + hidden_token = random_truth_token(context, "C2:release") + release_name = f"1.{rng.randint(2, 9)}.{rng.randint(0, 20)}-eval.{hidden_token[:8]}" + changelog_text = ( + f"Changelog entry one: OAuth login hardening ticket EVAL-{hidden_token}. " + f"Changelog entry two: webhook retry backoff window {rng.randint(3, 12)}-{hidden_token}." + ) + record_randomized_truth( + context, + "C2.release", + {"intended_name": release_name, "intended_changelog": changelog_text}, + ) + with plan_gate_skips("releases"): + release = plane.releases.create( + workspace_slug=workspace_slug, + data=CreateRelease(name=release_name), + ) + confirmed_release_name = str(getattr(release, "name", None) or "").strip() + if task_id == "C2" and not confirmed_release_name: + raise RuntimeError("release create response did not confirm the randomized release name") + confirmed_release_name = confirmed_release_name or release_name + context["release"] = {"id": release.id, "name": confirmed_release_name} + record_seeded_entity(context, "release", release.id) + context["release_name"] = confirmed_release_name + context["workspace_objects"].append({"kind": "release", "id": release.id}) + # Single changelog body; DESIGN's "2 entries" are encoded as plain text. + plane.releases.changelog.update( + workspace_slug=workspace_slug, + release_id=release.id, + data=UpdateReleaseChangelog( + description_html=f"

{changelog_text}

", + ), + ) + confirmed = plane.releases.changelog.retrieve( + workspace_slug=workspace_slug, + release_id=release.id, + ) + confirmed_text = normalize_changelog_text(confirmed) + if not confirmed_text: + raise RuntimeError("release changelog readback was empty after seeding") + context["release_changelog_text"] = confirmed_text + if task_id == "C2": + items = changelog_items(confirmed_text) + if not items: + raise RuntimeError("release changelog readback had no parseable entries after seeding") + context["randomized_truth"]["C2.release"]["confirmed"] = { + "name": confirmed_release_name, + "changelog": confirmed_text, + "items": list(items), + } + set_target_evidence(context, items) diff --git a/evals/seed/remove.py b/evals/seed/remove.py new file mode 100644 index 0000000..00c06bd --- /dev/null +++ b/evals/seed/remove.py @@ -0,0 +1,430 @@ +"""Fixture removal for evaluation runs.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any + +from plane import PlaneClient +from plane.errors.errors import HttpError +from plane.models.workspaces import WorkspaceFeature + +from .customers import CUSTOMER_NAME, EVALUATION_CUSTOMER_PROPERTY_NAME, is_evaluation_customer_name +from .item_types import ( + BUG_TYPE_NAME, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + is_severity_property, + is_work_item_type_named, + list_workspace_properties_for_type, + list_workspace_work_item_types, + workspace_owns_work_item_types, +) +from .releases import EVALUATION_RELEASE_TAG_VERSION +from .workspace import list_workspace_rows + + +@dataclass(frozen=True, slots=True) +class CleanupFailure: + """One cleanup operation that failed after teardown attempted it.""" + + operation: str + target: str + error_type: str + message: str + + def __str__(self) -> str: + return f"{self.operation} {self.target}: {self.error_type}: {self.message}" + + +class TeardownError(RuntimeError): + """All cleanup failures from one teardown, raised after every target was attempted.""" + + def __init__(self, failures: list[CleanupFailure]): + self.failures = tuple(failures) + details = "; ".join(str(failure) for failure in self.failures) + super().__init__(f"{len(self.failures)} cleanup operation(s) failed: {details}") + + +def _record_failure( + failures: list[CleanupFailure], + *, + operation: str, + target: Any, + exc: BaseException, +) -> None: + failures.append( + CleanupFailure( + operation=operation, + target=str(target), + error_type=type(exc).__name__, + message=str(exc), + ) + ) + + +def _baseline_ids(ctx: dict[str, Any], category: str) -> set[str] | None: + baseline = ctx.get("workspace_baseline") + if not isinstance(baseline, dict) or baseline.get(category) is None: + return None + return {str(object_id) for object_id in baseline[category]} + + +def _warn_unavailable_baseline( + category: str, + fixture_name: str, + object_ids: list[str], + failures: list[CleanupFailure], +) -> None: + if object_ids: + joined_ids = ", ".join(sorted(object_ids)) + print( + f"teardown warning: {category} baseline unavailable; leaving name-matched {fixture_name!r} ids={joined_ids}" + ) + _record_failure( + failures, + operation="preserve name-matched fixture without baseline", + target=f"{category} {fixture_name!r} ids={joined_ids}", + exc=RuntimeError("workspace baseline unavailable; ownership cannot be determined safely"), + ) + + +def _remove_severity_property( + plane: PlaneClient, + context: dict[str, Any], + failures: list[CleanupFailure], +) -> None: + """Delete only run-owned Severity properties attached to the seeded Bug type.""" + bug = context.get("bug_type") + if not bug: + return + bug_type_id = bug.get("id") if isinstance(bug, dict) else bug + if not bug_type_id: + return + workspace_slug = context.get("workspace_slug") or "" + project_id = context.get("project_id") + + # Workspace properties can appear through the project/type endpoint. Resolve both + # scopes and let the workspace scope win for deletion when an ID appears in both. + properties: dict[str, tuple[Any, str]] = {} + try: + if project_id: + project_properties = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + ) + or [] + ) + for row in project_properties: + if getattr(row, "id", None) is not None: + properties[str(row.id)] = (row, "project") + except HttpError as exc: + if exc.status_code not in (404, 405): + _record_failure(failures, operation="list", target="Severity properties", exc=exc) + except Exception as exc: + _record_failure(failures, operation="list", target="Severity properties", exc=exc) + + if context.get("bug_type_workspace_level"): + try: + for row in list_workspace_properties_for_type(plane, workspace_slug, BUG_TYPE_NAME): + if getattr(row, "id", None) is not None: + properties[str(row.id)] = (row, "workspace") + except Exception as exc: + _record_failure(failures, operation="list", target="workspace Severity properties", exc=exc) + + baseline = _baseline_ids(context, "work_item_properties") + tracked = { + str(obj.get("id")) + for obj in (context.get("workspace_objects") or []) + if obj.get("kind") == "work_item_property" and obj.get("id") is not None + } + skipped_ids: list[str] = [] + for property_id, (work_item_property, scope) in properties.items(): + if not is_severity_property(work_item_property): + continue + if property_id not in tracked: + if baseline is None: + skipped_ids.append(property_id) + continue + if property_id in baseline: + continue + try: + if scope == "workspace": + plane.workspace_work_item_properties.delete( + workspace_slug=workspace_slug, + property_id=property_id, + ) + elif project_id: + plane.work_item_properties.delete( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + work_item_property_id=property_id, + ) + except Exception as exc: + _record_failure( + failures, + operation="delete Severity property", + target=property_id, + exc=exc, + ) + _warn_unavailable_baseline( + "work item properties", + SEVERITY_PROPERTY_NAME, + skipped_ids, + failures, + ) + + +def _remove_incident_type( + plane: PlaneClient, + context: dict[str, Any], + failures: list[CleanupFailure], +) -> None: + """Clean up S3 Incident types while preserving seed-time workspace ownership.""" + if context.get("task_id") != "S3": + return + workspace_slug = context.get("workspace_slug") or "" + project_id = context.get("project_id") + try: + workspace_owns = workspace_owns_work_item_types(plane, workspace_slug) + except Exception as exc: + _record_failure(failures, operation="detect ownership", target="Incident work item types", exc=exc) + return + + try: + if workspace_owns: + item_types = list_workspace_work_item_types(plane, workspace_slug) + scope = "workspace" + elif project_id: + item_types = list(plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []) + scope = "project" + else: + return + except Exception as exc: + _record_failure(failures, operation="list", target="Incident work item types", exc=exc) + return + + for item_type in item_types: + if not is_work_item_type_named(item_type, INCIDENT_TYPE_NAME): + continue + item_type_id = str(item_type.id) + if scope == "workspace": + baseline = _baseline_ids(context, "work_item_types") + tracked = { + str(obj.get("id")) + for obj in (context.get("workspace_objects") or []) + if obj.get("kind") == "work_item_type" and obj.get("id") is not None + } + if item_type_id not in tracked: + if baseline is None: + _warn_unavailable_baseline( + "work item types", + INCIDENT_TYPE_NAME, + [item_type_id], + failures, + ) + continue + if item_type_id in baseline: + continue + try: + if scope == "workspace": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=item_type_id) + else: + assert project_id is not None + plane.work_item_types.delete( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_type_id=item_type_id, + ) + except Exception as exc: + _record_failure( + failures, + operation="delete Incident work item type", + target=item_type_id, + exc=exc, + ) + + +def teardown(plane: PlaneClient, ctx: dict[str, Any]) -> None: + """Attempt every fixture deletion, then raise all failures as one structured error.""" + if not ctx: + return + failures: list[CleanupFailure] = [] + workspace_slug = ctx.get("workspace_slug") or os.environ.get("EVAL_PLANE_WORKSPACE_SLUG", "") + project_id = ctx.get("project_id") + + # Put workspace toggles back where the run found them. Seeding writes them explicitly + # (a task may need one off), and the workspace outlives the run, so leaving this run's + # requirements behind is drift on an instance the harness does not own. + prior = ctx.get("workspace_features_prior") or {} + prior_customers = prior.get("customers") + if prior_customers is not None: + try: + plane.workspaces.update_features( + workspace_slug=workspace_slug, + data=WorkspaceFeature(customers=bool(prior_customers)), + ) + except Exception as exc: + _record_failure( + failures, + operation="restore workspace customers feature", + target=prior_customers, + exc=exc, + ) + + # Drop agent-created Severity on Bug before project/type teardown (F8 multi-rep pollution). + try: + _remove_severity_property(plane, ctx, failures) + except Exception as exc: + _record_failure(failures, operation="clean up", target="Severity properties", exc=exc) + try: + _remove_incident_type(plane, ctx, failures) + except Exception as exc: + _record_failure(failures, operation="clean up", target="Incident work item types", exc=exc) + + # Best-effort: agent-created Acme Corp customers (C1) that never hit workspace_objects. + try: + rows = list_workspace_rows(plane.customers, workspace_slug) + tracked = { + str(obj.get("id")) + for obj in (ctx.get("workspace_objects") or []) + if obj.get("kind") == "customer" and obj.get("id") is not None + } + baseline = _baseline_ids(ctx, "customers") + skipped_ids: list[str] = [] + for customer in rows or []: + if not is_evaluation_customer_name(getattr(customer, "name", None)): + continue + customer_id = getattr(customer, "id", None) + if customer_id is None or str(customer_id) in tracked: + continue + if baseline is None: + skipped_ids.append(str(customer_id)) + elif str(customer_id) not in baseline: + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": customer_id}) + _warn_unavailable_baseline("customers", CUSTOMER_NAME, skipped_ids, failures) + except Exception as exc: + _record_failure(failures, operation="scan", target="evaluation customers", exc=exc) + + # Workspace-scoped cleanup first (survive project deletion). + seen_workspace_objects: set[str] = set() + for obj in ctx.get("workspace_objects") or []: + try: + kind = obj.get("kind") + object_id = obj.get("id") + except Exception as exc: + _record_failure(failures, operation="read tracked workspace object", target=repr(obj), exc=exc) + continue + if not object_id: + continue + key = f"{kind}:{object_id}" + if key in seen_workspace_objects: + continue + seen_workspace_objects.add(key) + try: + if kind == "work_item_type": + plane.workspace_work_item_types.delete(workspace_slug=workspace_slug, type_id=object_id) + elif kind == "work_item_property": + plane.workspace_work_item_properties.delete(workspace_slug=workspace_slug, property_id=object_id) + elif kind == "customer": + plane.customers.delete(workspace_slug=workspace_slug, customer_id=object_id) + elif kind == "release": + plane.releases.delete(workspace_slug=workspace_slug, release_id=object_id) + elif kind == "release_tag": + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=object_id) + elif kind == "customer_property": + plane.customers.properties.delete(workspace_slug=workspace_slug, property_id=object_id) + else: + raise ValueError(f"unsupported workspace object kind {kind!r}") + except Exception as exc: + _record_failure( + failures, + operation=f"delete workspace {kind}", + target=object_id, + exc=exc, + ) + + # Sweep by well-known WS3 names in case tracking missed an agent-created row. + try: + rows = list_workspace_rows(plane.releases.tags, workspace_slug) + baseline = _baseline_ids(ctx, "release_tags") + skipped_ids: list[str] = [] + for tag in rows or []: + if (getattr(tag, "version", None) or "").strip() == EVALUATION_RELEASE_TAG_VERSION: + tag_id = getattr(tag, "id", None) + if tag_id and f"release_tag:{tag_id}" not in seen_workspace_objects: + if baseline is None: + skipped_ids.append(str(tag_id)) + elif str(tag_id) not in baseline: + try: + plane.releases.tags.delete(workspace_slug=workspace_slug, tag_id=tag_id) + except Exception as exc: + _record_failure( + failures, + operation="sweep release tag", + target=tag_id, + exc=exc, + ) + _warn_unavailable_baseline("release tags", EVALUATION_RELEASE_TAG_VERSION, skipped_ids, failures) + except Exception as exc: + _record_failure(failures, operation="scan", target="evaluation release tags", exc=exc) + try: + rows = list_workspace_rows(plane.customers.properties, workspace_slug) + baseline = _baseline_ids(ctx, "customer_properties") + skipped_ids: list[str] = [] + target = EVALUATION_CUSTOMER_PROPERTY_NAME.casefold() + for customer_property in rows or []: + display = ( + getattr(customer_property, "display_name", None) or getattr(customer_property, "name", None) or "" + ).strip() + if display.casefold() == target: + property_id = getattr(customer_property, "id", None) + if property_id and f"customer_property:{property_id}" not in seen_workspace_objects: + if baseline is None: + skipped_ids.append(str(property_id)) + elif str(property_id) not in baseline: + try: + plane.customers.properties.delete( + workspace_slug=workspace_slug, + property_id=property_id, + ) + except Exception as exc: + _record_failure( + failures, + operation="sweep customer property", + target=property_id, + exc=exc, + ) + _warn_unavailable_baseline( + "customer properties", + EVALUATION_CUSTOMER_PROPERTY_NAME, + skipped_ids, + failures, + ) + except Exception as exc: + _record_failure(failures, operation="scan", target="evaluation customer properties", exc=exc) + + # Second project before main (no dependency either way, but be thorough). + second_project_ids = [ctx.get("second_project_id"), *(ctx.get("second_project_ids") or [])] + for second_project_id in dict.fromkeys(second_project_ids): + if not second_project_id or second_project_id == project_id: + continue + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=second_project_id) + except Exception as exc: + _record_failure(failures, operation="delete second project", target=second_project_id, exc=exc) + + if project_id: + try: + plane.projects.delete(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + name = ctx.get("project_name", project_id) + print(f"orphaned project: {name}") + _record_failure(failures, operation="delete project", target=name, exc=exc) + + if failures: + raise TeardownError(failures) diff --git a/evals/seed/states.py b/evals/seed/states.py new file mode 100644 index 0000000..0f1d78f --- /dev/null +++ b/evals/seed/states.py @@ -0,0 +1,59 @@ +"""Project-state fixtures and immutable read oracles.""" + +from __future__ import annotations + +from typing import Any + +from plane import PlaneClient +from plane.models.states import CreateState + +from evals.core.evidence import set_target_evidence +from evals.core.state_oracle import state_name_group_pairs + +from .identities import record_seeded_entity +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + + +def seed_r7_state_oracle(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Add one hidden state and capture the complete API-confirmed state baseline.""" + project_id = str(context.get("project_id") or "") + if not project_id: + raise RuntimeError("seed R7: project id missing") + rng = random_truth_rng(context, "R7:states") + hidden_token = random_truth_token(context, "R7:states") + state_name = f"Review {hidden_token}" + state_group = rng.choice(("unstarted", "started", "completed")) + created = plane.states.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateState(name=state_name, color="#5E6AD2", group=state_group), + ) + created_id = str(getattr(created, "id", None) or "") + if not created_id: + raise RuntimeError("seed R7: randomized state create returned no id") + + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + rows = list(page.results or []) + if not rows: + raise RuntimeError("seed R7: API readback returned no project states") + pairs = state_name_group_pairs(rows) + expected_pair = f"{state_name} | group: {state_group}" + if expected_pair not in pairs: + raise RuntimeError( + f"seed R7: randomized state missing from API readback; want {expected_pair!r}; have={pairs!r}" + ) + context["r7_state_pairs"] = pairs + context["r7_random_state_id"] = created_id + record_seeded_entity(context, "state", created_id) + record_randomized_truth( + context, + "R7.states", + { + "intended": expected_pair, + "confirmed": list(pairs), + }, + ) + set_target_evidence(context, [state_name]) + + +__all__ = ["seed_r7_state_oracle"] diff --git a/evals/seed/work_items.py b/evals/seed/work_items.py new file mode 100644 index 0000000..051be83 --- /dev/null +++ b/evals/seed/work_items.py @@ -0,0 +1,490 @@ +"""Work item fixtures for evaluation projects.""" + +from __future__ import annotations + +import json +from datetime import date, timedelta +from typing import Any + +from plane import PlaneClient +from plane.models.query_params import WorkItemQueryParams +from plane.models.states import CreateState +from plane.models.work_items import CreateWorkItem, CreateWorkItemComment, UpdateWorkItem + +from evals.core.changelog import normalize_changelog_text +from evals.core.errors import TaskSkipped +from evals.core.evidence import set_target_count_evidence, set_target_evidence +from evals.core.fixtures import ( + BLOCKING_REFERENCE_ADDRESS, + BLOCKING_SOURCE_TITLE, + BLOCKING_TARGET_TITLE, + CHECKOUT_COMMENT_PHRASES, + CHECKOUT_TIMEOUT_TITLE, + DARK_MODE_TITLE, + DUE_THIS_WEEK_TITLES, + PAYMENT_WEBHOOK_TITLE, + SIDEBAR_TITLE, + UNFINISHED_CYCLE_TITLES, + WORK_ITEM_FIXTURES, +) +from evals.core.state_oracle import worklog_summary_item_ids + +from .gates import plan_gate_skips +from .identities import record_seeded_entity +from .randomize import random_truth_rng, random_truth_token, record_randomized_truth + +__all__ = [ + "BLOCKING_REFERENCE_ADDRESS", + "BLOCKING_SOURCE_TITLE", + "BLOCKING_TARGET_TITLE", + "CHECKOUT_COMMENT_PHRASES", + "CHECKOUT_TIMEOUT_TITLE", + "DARK_MODE_TITLE", + "DUE_THIS_WEEK_TITLES", + "PAYMENT_WEBHOOK_TITLE", + "SIDEBAR_TITLE", + "UNFINISHED_CYCLE_TITLES", + "WORK_ITEM_FIXTURES", + "find_completed_state", + "list_states", + "require_activities", + "seed_work_items", +] + + +def list_states(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + return list(page.results or []) + + +def find_completed_state(states: list[Any]) -> Any | None: + completed = [state for state in states if getattr(state, "group", None) == "completed"] + if not completed: + return None + # Prefer a non-default completed state named Done if present. + for state in completed: + if (state.name or "").strip().casefold() == "done": + return state + return completed[0] + + +def _enum_value(value: Any) -> str: + return str(getattr(value, "value", value) or "") + + +def _as_id(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, dict): + return str(value.get("id") or "") + return str(getattr(value, "id", None) or "") + + +def _list_all_work_items(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[Any]: + rows: list[Any] = [] + cursor: str | None = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + rows.extend(page.results or []) + if not page.next_page_results: + return rows + cursor = page.next_cursor + + +def _resolve_state_name( + plane: PlaneClient, + workspace_slug: str, + project_id: str, + state_ref: Any, +) -> str: + direct = getattr(state_ref, "name", None) or (state_ref.get("name") if isinstance(state_ref, dict) else None) + if direct: + return str(direct) + state_id = _as_id(state_ref) + if not state_id: + return "" + state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) + return str(state.name or "") + + +def _confirm_open_urgent_items(plane: PlaneClient, workspace_slug: str, project_id: str) -> list[str]: + states = list_states(plane, workspace_slug, project_id) + closed = { + str(state.id) + for state in states + if _enum_value(getattr(state, "group", None)).casefold() in {"completed", "cancelled"} + } + titles: list[str] = [] + for item in _list_all_work_items(plane, workspace_slug, project_id): + if _enum_value(getattr(item, "priority", None)).casefold() != "urgent": + continue + if _as_id(getattr(item, "state", None)) in closed: + continue + title = str(getattr(item, "name", None) or "").strip() + if not title: + raise RuntimeError(f"seed R2: API readback returned urgent item without a name: {item!r}") + titles.append(title) + return titles + + +def _serialized_rows(rows: list[Any]) -> str: + payload = [row.model_dump(mode="json") if hasattr(row, "model_dump") else row for row in rows] + return json.dumps(payload, default=str, ensure_ascii=False) + + +def _comment_text(comment: Any) -> str: + stripped = str(getattr(comment, "comment_stripped", None) or "").strip() + if stripped: + return " ".join(stripped.split()) + return normalize_changelog_text(str(getattr(comment, "comment_html", None) or "")) + + +def seed_work_items(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + project_id = context["project_id"] + task_id = str(context.get("task_id") or "") + rng = random_truth_rng(context, f"{task_id or 'shared'}:work-items") + hidden_token = random_truth_token(context, f"{task_id or 'shared'}:work-items") + states = list_states(plane, workspace_slug, project_id) + context["state_names"] = sorted({(state.name or "").strip() for state in states if (state.name or "").strip()}) + + # Prefer a non-default started-group state so R1 cannot be passed by guessing the default. + started = [ + state for state in states if getattr(state, "group", None) == "started" and not getattr(state, "default", False) + ] + if not started: + started = [state for state in states if getattr(state, "group", None) == "started"] + if not started: + raise RuntimeError( + "seed items: no started-group state available to place the R1 target; " + f"states={[(state.name, state.group, state.default) for state in states]}" + ) + base_started_state = started[0] + state_targets: dict[str, Any] = {PAYMENT_WEBHOOK_TITLE: base_started_state} + if task_id in {"R1", "I2"}: + random_state_name = f"Investigating {hidden_token}" + random_state = plane.states.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateState( + name=random_state_name, + color="#5E6AD2", + group="started", + ), + ) + if not getattr(random_state, "id", None): + raise RuntimeError(f"seed {task_id}: random state create returned no id") + state_targets[PAYMENT_WEBHOOK_TITLE if task_id == "R1" else SIDEBAR_TITLE] = random_state + record_seeded_entity(context, "state", random_state.id) + record_randomized_truth( + context, + f"{task_id}.state", + {"intended": random_state_name, "created_id": str(random_state.id)}, + ) + + r1_state = state_targets[PAYMENT_WEBHOOK_TITLE] + context["r1_state_name"] = r1_state.name + context["r1_state_id"] = r1_state.id + + me = plane.users.get_me() + me_id = str(me.id) + context["me_id"] = me_id + # Due dates must stay inside the current ISO week (Mon–Sun). + # today+2d alone escapes the week on Sat/Sun — clamp to this week's Sunday. + today = date.today() + days_to_week_end = 6 - today.weekday() # Mon=0 … Sun=6 + due_this_week = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)).isoformat() + context["r3_due_date"] = due_this_week + + urgent_target = rng.randint(2, 7) if task_id == "R2" else 4 + if task_id == "R2": + record_randomized_truth(context, "R2.urgent_open_count", {"intended": urgent_target}) + + r3_templates: set[str] = set(DUE_THIS_WEEK_TITLES) + if task_id == "R3": + r3_count = rng.randint(1, 4) + candidates = [title for title, _priority in WORK_ITEM_FIXTURES] + r3_templates = set(rng.sample(candidates, r3_count)) + record_randomized_truth(context, "R3.due_templates", sorted(r3_templates)) + + randomize_titles = task_id in {"R2", "R3", "R4"} + urgent_count = 0 + for index, (fixture_title, fixture_priority) in enumerate(WORK_ITEM_FIXTURES): + title = ( + f"{fixture_title} · case {hidden_token}-{index + 1}" + if randomize_titles and (task_id in {"R2", "R4"} or fixture_title in r3_templates) + else fixture_title + ) + priority = "urgent" if index < urgent_target else ("high" if fixture_priority == "urgent" else fixture_priority) + data_kwargs: dict[str, Any] = {"name": title, "priority": priority} + target_state = state_targets.get(fixture_title) + if target_state is not None: + data_kwargs["state"] = str(target_state.id) + if fixture_title in r3_templates: + data_kwargs["assignees"] = [me_id] + data_kwargs["target_date"] = due_this_week + item = plane.work_items.create( + workspace_slug=workspace_slug, + project_id=project_id, + data=CreateWorkItem(**data_kwargs), # type: ignore[arg-type] + ) + # Some APIs ignore state on create; force via update if needed. + if target_state is not None: + current = getattr(item, "state", None) + current_id = current if isinstance(current, str) else getattr(current, "id", None) + if str(current_id) != str(target_state.id): + item = plane.work_items.update( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + data=UpdateWorkItem(state=str(target_state.id)), + ) + context["items"][title] = item.id + context["fixture_item_ids"][fixture_title] = item.id + context["fixture_item_titles"][fixture_title] = title + context["item_ids"].append(item.id) + record_seeded_entity(context, "work_item", item.id) + sequence = getattr(item, "sequence_id", None) + if sequence is not None and context.get("project_identifier"): + context["item_identifiers"][fixture_title] = f"{context['project_identifier']}-{sequence}" + if priority == "urgent": + urgent_count += 1 + assert urgent_count == urgent_target, ( + f"fixture invariant: expected {urgent_target} urgent items, got {urgent_count}" + ) + + # R5: seed discussion comments on the known item. + target_id = context["fixture_item_ids"].get(CHECKOUT_TIMEOUT_TITLE) + comment_phrases = list(CHECKOUT_COMMENT_PHRASES) + if task_id in {"R5", "L2"}: + comment_count = rng.randint(1, 4) + comment_phrases = [ + f"{CHECKOUT_COMMENT_PHRASES[index % len(CHECKOUT_COMMENT_PHRASES)]} ref-{hidden_token}-{index + 1}" + for index in range(comment_count) + ] + truth_key = "R5.comments" if task_id == "R5" else "L2.activity_count" + truth_value = ( + {"intended": list(comment_phrases)} if task_id == "R5" else {"intended_comment_count": comment_count} + ) + record_randomized_truth(context, truth_key, truth_value) + if task_id == "L2": + context["l2_comment_phrases"] = list(comment_phrases) + comment_ids: set[str] = set() + if target_id: + for phrase in comment_phrases: + created_comment = plane.work_items.comments.create( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=target_id, + data=CreateWorkItemComment(comment_html=f"

{phrase}

"), + ) + if getattr(created_comment, "id", None) is not None: + comment_ids.add(str(created_comment.id)) + record_seeded_entity(context, "comment", created_comment.id) + + # Capture every affected read oracle from API-confirmed state, never from the random choice. + if task_id in {"R1", "I2"}: + target_fixture = PAYMENT_WEBHOOK_TITLE if task_id == "R1" else SIDEBAR_TITLE + target_id = context["fixture_item_ids"][target_fixture] + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=target_id, + ) + confirmed_name = _resolve_state_name(plane, workspace_slug, project_id, detail.state) + if not confirmed_name: + raise RuntimeError(f"seed {task_id}: API readback could not resolve target state") + oracle_key = "r1_state_name" if task_id == "R1" else "i2_state_name" + context[oracle_key] = confirmed_name + context["randomized_truth"][f"{task_id}.state"]["confirmed"] = confirmed_name + set_target_evidence(context, [confirmed_name]) + + if task_id == "R2": + confirmed_titles = _confirm_open_urgent_items(plane, workspace_slug, project_id) + if not confirmed_titles: + raise RuntimeError("seed R2: API readback found no urgent open work items") + context["r2_urgent_open_count"] = len(confirmed_titles) + context["randomized_truth"]["R2.urgent_open_count"]["confirmed"] = len(confirmed_titles) + set_target_evidence(context, confirmed_titles) + set_target_count_evidence(context, len(confirmed_titles), target_ids=[project_id]) + + if task_id == "R3": + confirmed_due_titles: list[str] = [] + week_start = today - timedelta(days=today.weekday()) + week_end = week_start + timedelta(days=6) + for item_id in context["item_ids"]: + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item_id, + ) + target_date = str(getattr(detail, "target_date", None) or "")[:10] + assignee_ids = {_as_id(value) for value in (getattr(detail, "assignees", None) or [])} + if target_date and week_start.isoformat() <= target_date <= week_end.isoformat() and me_id in assignee_ids: + confirmed_due_titles.append(str(detail.name)) + if not confirmed_due_titles: + raise RuntimeError("seed R3: API readback found no assigned due-this-week items") + context["r3_due_titles"] = confirmed_due_titles + context["r3_due_count"] = len(confirmed_due_titles) + context["randomized_truth"]["R3.due_templates"] = { + "intended": sorted(r3_templates), + "confirmed": { + "titles": list(confirmed_due_titles), + "count": len(confirmed_due_titles), + }, + } + set_target_evidence(context, confirmed_due_titles) + + if task_id == "R5": + page = plane.work_items.comments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=target_id, + ) + confirmed_comments = [ + _comment_text(comment) + for comment in (page.results or []) + if not comment_ids or str(getattr(comment, "id", "")) in comment_ids + ] + confirmed_comments = [text for text in confirmed_comments if text] + if len(confirmed_comments) != len(comment_phrases): + raise RuntimeError( + f"seed R5: API readback returned {len(confirmed_comments)} eval comments; want {len(comment_phrases)}" + ) + context["r5_comment_phrases"] = confirmed_comments + context["randomized_truth"]["R5.comments"]["confirmed"] = list(confirmed_comments) + set_target_evidence(context, confirmed_comments) + + if task_id == "L1": + work_item_id = str(context["fixture_item_ids"].get(PAYMENT_WEBHOOK_TITLE) or "") + if not work_item_id: + raise RuntimeError("seed L1: target work item id missing") + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) + confirmed_id = str(getattr(detail, "id", None) or "") + if confirmed_id != work_item_id: + raise RuntimeError(f"seed L1: target work item readback id={confirmed_id!r}; want {work_item_id!r}") + # Seed a worklog the agent did not create, on an item it is not told about. Without + # one, the summary contains only the row the agent just wrote, so reporting the id it + # already holds was both the correct answer and its own provenance — L1 could be + # passed without ever reading the summary it exists to exercise. + other_id = _second_worklog_item_id(context, exclude=confirmed_id) + seeded_minutes = rng.randrange(15, 240, 15) + with plan_gate_skips("worklogs"): + plane.work_items.work_logs.create( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=other_id, + # This resource takes a plain mapping, not a request model like its siblings. + data={"duration": seeded_minutes, "description": f"seeded {hidden_token}"}, + ) + confirmed_summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) + confirmed_summary_ids = worklog_summary_item_ids(confirmed_summary) + if other_id not in confirmed_summary_ids: + raise RuntimeError( + f"seed L1: seeded worklog on {other_id} is absent from the project summary {confirmed_summary_ids!r}" + ) + record_randomized_truth( + context, + "L1.seeded_worklog", + {"intended_item": other_id, "intended_minutes": seeded_minutes}, + ) + context["randomized_truth"]["L1.seeded_worklog"]["confirmed"] = { + "summary_ids": list(confirmed_summary_ids), + } + # The agent's own 90-minute log adds the target row during the run. + context["l1_expected_summary_ids"] = sorted({*confirmed_summary_ids, confirmed_id}) + set_target_evidence(context, [other_id]) + + if task_id == "L5": + attachment_target_id = context["fixture_item_ids"][PAYMENT_WEBHOOK_TITLE] + attachment_count = rng.randint(1, 3) + record_randomized_truth(context, "L5.attachment_count", {"intended": attachment_count}) + intended_names: list[str] = [] + for index in range(attachment_count): + name = f"diagnostic-{hidden_token}-{index + 1}.txt" + intended_names.append(name) + plane.work_items.attachments.upload_from_bytes( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=attachment_target_id, + file_bytes=f"eval attachment {hidden_token}-{index + 1}\n".encode(), + name=name, + content_type="text/plain", + ) + attachments = plane.work_items.attachments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=attachment_target_id, + ) + # attachments.list returns a bare list[WorkItemAttachment], not the paged envelope the + # other list endpoints return. Assuming `.results` raised AttributeError on every L5 + # repetition, which surfaced as infra_seed and cost the task its whole row budget. + confirmed_rows = list(attachments if isinstance(attachments, list) else (attachments.results or [])) + for attachment in confirmed_rows: + record_seeded_entity(context, "attachment", getattr(attachment, "id", None)) + confirmed_attachment_count = len(confirmed_rows) + context["l5_attachment_count"] = confirmed_attachment_count + context["randomized_truth"]["L5.attachment_count"]["confirmed"] = confirmed_attachment_count + response_blob = _serialized_rows(confirmed_rows) + confirmed_names = [name for name in intended_names if name in response_blob] + if len(confirmed_names) != attachment_count: + raise RuntimeError( + f"seed L5: attachment readback exposed {len(confirmed_names)} randomized names; want {attachment_count}" + ) + set_target_evidence(context, confirmed_names) + + +def _second_worklog_item_id(context: dict[str, Any], *, exclude: str) -> str: + """Pick the seeded item that carries L1's pre-existing worklog. + + Deterministic per run so the oracle is reproducible from the fixture seed, and never the + item the prompt names — the point is a summary row the agent can only learn by reading + the summary. + """ + candidates = sorted( + str(item_id) + for item_id in (context.get("fixture_item_ids") or {}).values() + if str(item_id) and str(item_id) != str(exclude) + ) + if not candidates: + raise RuntimeError("seed L1: no second work item available to carry a seeded worklog") + rng = random_truth_rng(context, "L1:second-worklog") + return rng.choice(candidates) + + +def require_activities(plane: PlaneClient, workspace_slug: str, context: dict[str, Any]) -> None: + """Require L2's seeded comments to materialize as activities. + + Only a successful, empty read evidences a missing activity worker and becomes the + expected ``env:no-activity-worker`` capability skip. Missing fixture identifiers and + inconsistent readback are fixture errors; API read failures propagate as infrastructure. + """ + project_id = context.get("project_id") + work_item_id = (context.get("items") or {}).get(CHECKOUT_TIMEOUT_TITLE) + if not project_id or not work_item_id: + missing = [name for name, value in (("project_id", project_id), ("work_item_id", work_item_id)) if not value] + raise RuntimeError(f"seed L2 fixture error: missing {', '.join(missing)}") + + page = plane.work_items.activities.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=work_item_id, + ) + rows = page.results if hasattr(page, "results") else page + activity_rows = list(rows or []) + activity_count = len(activity_rows) + if activity_count < 1: + raise TaskSkipped("env:no-activity-worker") + context["l2_activity_count"] = activity_count + if str(context.get("task_id") or "") == "L2": + randomised = context.setdefault("randomized_truth", {}).setdefault("L2.activity_count", {}) + randomised["confirmed"] = activity_count + # Evidence is the activity count, which is what L2 actually asks for and what its + # verifier checks. It used to require the seeded comment phrase to appear in the + # activity readback — evidence Plane's activity API never emits: the endpoint returns + # the creation row and no comment text, so every L2 repetition died in seeding. + set_target_count_evidence(context, activity_count, target_ids=[work_item_id]) diff --git a/evals/seed/workspace.py b/evals/seed/workspace.py new file mode 100644 index 0000000..4e59e45 --- /dev/null +++ b/evals/seed/workspace.py @@ -0,0 +1,22 @@ +"""Shared helpers for workspace-scoped evaluation fixtures.""" + +from __future__ import annotations + +from typing import Any + +from plane.models.query_params import PaginatedQueryParams + + +def list_workspace_rows(api: Any, workspace_slug: str) -> list[Any]: + """List every row from a paginated workspace-scoped API.""" + rows: list[Any] = [] + cursor = None + while True: + page = api.list( + workspace_slug=workspace_slug, + params=PaginatedQueryParams(per_page=100, cursor=cursor), + ) + rows.extend((page.results if hasattr(page, "results") else page) or []) + if not getattr(page, "next_page_results", False): + return rows + cursor = page.next_cursor diff --git a/evals/skip_taxonomy.py b/evals/skip_taxonomy.py new file mode 100644 index 0000000..728a326 --- /dev/null +++ b/evals/skip_taxonomy.py @@ -0,0 +1,118 @@ +"""Explicit run-completeness taxonomy for task skip reasons. + +Known missing environment capabilities are expected skips: they reduce execution +coverage but do not make an otherwise clean run incomplete. A dirty environment that +requires operator cleanup, such as a fixture collision, is unexpected. Unknown reasons +are also unexpected by default; there is deliberately no ``env:*`` catch-all. Plan-gate +reasons must name one of the explicitly supported capabilities below, and the activity +worker reason must match exactly. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Literal + +SkipDisposition = Literal["expected-capability", "dirty-environment", "unexpected"] + +PLAN_GATED_PREFIX = "env:plan-gated:" +NO_ACTIVITY_WORKER_REASON = "env:no-activity-worker" +FIXTURE_COLLISION_PREFIX = "env:fixture-collision:" + +# Derived from the plan-gated seed surfaces: customer and release fixture seeders, the +# work-item-type seeder, and the initiative/teamspace plan refusals characterized by +# seed.projects.is_plan_gate. Keep this closed: a new capability is unexpected until its +# actual gate site is reviewed and added deliberately. +PLAN_GATED_CAPABILITIES = frozenset( + { + "customers", + "initiatives", + "releases", + "teamspaces", + "work-item-types", + } +) + +# Task eligibility is derived from the fixture ``needs`` in the task catalog. This +# maps fixture groups to the one capability refusal that their seeder can emit; it +# deliberately does not duplicate a task-id table. +_PLAN_GATED_CAPABILITY_BY_NEED = { + "bug_type": "work-item-types", + "customer": "customers", + "release": "releases", +} +_ACTIVITY_WORKER_NEED = "activity_feed" + + +def _plan_gated_capability(reason: str) -> str | None: + if not reason.startswith(PLAN_GATED_PREFIX): + return None + capability = reason.removeprefix(PLAN_GATED_PREFIX) + return capability if capability in PLAN_GATED_CAPABILITIES else None + + +def _task_expected_capability_reasons(task_id: str, task_needs: Iterable[str] | None = None) -> frozenset[str]: + if task_needs is None: + # No run metadata: fall back to the checkout. Only reached for files written before + # the meta header carried each task's needs. + from evals.tasks import TASKS_BY_ID + + task = TASKS_BY_ID.get(task_id) + task_needs = task.get("needs") or () if task is not None else () + needs = set(str(need) for need in task_needs) + reasons = { + f"{PLAN_GATED_PREFIX}{capability}" + for need, capability in _PLAN_GATED_CAPABILITY_BY_NEED.items() + if need in needs + } + if _ACTIVITY_WORKER_NEED in needs: + reasons.add(NO_ACTIVITY_WORKER_REASON) + return frozenset(reasons) + + +def classify_skip_reason( + reason: str, + *, + task_id: str | None = None, + task_needs: Iterable[str] | None = None, +) -> SkipDisposition: + """Classify a known capability skip, dirty environment, or unknown reason.""" + is_known_capability = _plan_gated_capability(reason) is not None or reason == NO_ACTIVITY_WORKER_REASON + if is_known_capability and (task_id is None or reason in _task_expected_capability_reasons(task_id, task_needs)): + return "expected-capability" + if reason.startswith(FIXTURE_COLLISION_PREFIX) and reason.removeprefix(FIXTURE_COLLISION_PREFIX): + return "dirty-environment" + return "unexpected" + + +def is_expected_environment_capability_skip( + reason: str, + *, + task_id: str | None = None, + task_needs: Iterable[str] | None = None, +) -> bool: + """Return whether a known absent environment capability caused the skip.""" + return classify_skip_reason(reason, task_id=task_id, task_needs=task_needs) == "expected-capability" + + +def skip_reason_family(reason: str) -> str: + """Return the stable reporting family for a skip reason.""" + if _plan_gated_capability(reason) is not None: + return "plan-gated" + if reason == NO_ACTIVITY_WORKER_REASON: + return "no-activity-worker" + if reason.startswith(FIXTURE_COLLISION_PREFIX) and reason.removeprefix(FIXTURE_COLLISION_PREFIX): + return "fixture-collision" + return reason or "" + + +__all__ = [ + "FIXTURE_COLLISION_PREFIX", + "NO_ACTIVITY_WORKER_REASON", + "PLAN_GATED_CAPABILITIES", + "PLAN_GATED_PREFIX", + "SkipDisposition", + "classify_skip_reason", + "is_expected_environment_capability_skip", + "skip_reason_family", +] diff --git a/evals/tasks/__init__.py b/evals/tasks/__init__.py new file mode 100644 index 0000000..f7d93c0 --- /dev/null +++ b/evals/tasks/__init__.py @@ -0,0 +1,147 @@ +"""Public task catalog and verifier API.""" + +from evals.core.errors import TaskSkipped +from evals.tasks.answers import ( + contract_values, + get_final_text, + reports_contract_int, + reports_contract_value, + reports_contract_values, + reports_exact_int, + whole_answer_int, + word_boundary, +) +from evals.tasks.catalog import ( + CATALOG_REVISION, + EXPECTED_TASK_IDS, + TASKS, + TASKS_BY_ID, + battery_fingerprint, + get_tasks, + task_author, + task_fingerprint, + task_fingerprint_payload, +) +from evals.tasks.cross import verify_c1, verify_c2 +from evals.tasks.debias import ( + I1_TITLE, + I2_TITLE, + I3_TITLE, + I4_TITLE, + L1_TITLE, + L2_TITLE, + L3_TAG_VERSION, + L4_PROP_DISPLAY, + L4_PROP_VALUE, + L5_TITLE, + verify_i1, + verify_i2, + verify_i3, + verify_i4, + verify_i5, + verify_l1, + verify_l2, + verify_l3, + verify_l4, + verify_l5, +) +from evals.tasks.lookups import ( + as_id, + count_open_urgent, + find_item_by_name, + find_items_by_name, + ids, + is_not_found, + state_group, + state_name, +) +from evals.tasks.prompts import PromptBindError, format_task_prompt +from evals.tasks.read import verify_r1, verify_r2, verify_r3, verify_r4, verify_r5, verify_r6, verify_r7 +from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.write import ( + verify_w1, + verify_w2, + verify_w3, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w8, + verify_w9, + verify_w10, +) + +__all__ = [ + "EXPECTED_TASK_IDS", + "PromptBindError", + "TASKS", + "CATALOG_REVISION", + "TASKS_BY_ID", + "TaskSkipped", + "as_id", + "battery_fingerprint", + "contract_values", + "count_open_urgent", + "find_item_by_name", + "find_items_by_name", + "format_task_prompt", + "get_final_text", + "get_tasks", + "ids", + "is_not_found", + "reports_contract_int", + "reports_contract_value", + "reports_contract_values", + "reports_exact_int", + "state_group", + "state_name", + "task_author", + "task_fingerprint", + "task_fingerprint_payload", + "whole_answer_int", + "word_boundary", + "I1_TITLE", + "I2_TITLE", + "I3_TITLE", + "I4_TITLE", + "L1_TITLE", + "L2_TITLE", + "L3_TAG_VERSION", + "L4_PROP_DISPLAY", + "L4_PROP_VALUE", + "L5_TITLE", + "verify_r1", + "verify_r2", + "verify_r3", + "verify_r4", + "verify_r5", + "verify_r6", + "verify_r7", + "verify_w1", + "verify_w2", + "verify_w3", + "verify_w4", + "verify_w5", + "verify_w6", + "verify_w7", + "verify_w8", + "verify_w9", + "verify_w10", + "verify_s1", + "verify_s2", + "verify_s3", + "verify_s4", + "verify_s5", + "verify_c1", + "verify_c2", + "verify_i1", + "verify_i2", + "verify_i3", + "verify_i4", + "verify_i5", + "verify_l1", + "verify_l2", + "verify_l3", + "verify_l4", + "verify_l5", +] diff --git a/evals/tasks/answers.py b/evals/tasks/answers.py new file mode 100644 index 0000000..cc5454b --- /dev/null +++ b/evals/tasks/answers.py @@ -0,0 +1,191 @@ +"""Answer-contract matching for task verifiers.""" + +from __future__ import annotations + +import re +from collections import Counter +from html import unescape +from typing import Any + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE + + +def word_boundary(value: str) -> re.Pattern[str]: + """Compile a case-insensitive word-boundary match for an exact seeded value.""" + return re.compile(rf"\b{re.escape(value)}\b", re.IGNORECASE) + + +def reports_exact_int(text: str, n: int) -> bool: + """True when ``text`` contains integer ``n`` as a whole word (not a substring of 10).""" + return bool(word_boundary(str(int(n))).search(text or "")) + + +def whole_answer_int(text: str) -> int | None: + """If the answer (or its last non-empty line) is exactly an integer, return it. + + Letters must not appear — only surrounding whitespace/punctuation is ignored — + so prose like ``There are 3 comments…`` is not a whole-answer int. A **leading + minus** attached to the number is preserved (``-3`` → -3, not 3). + """ + + def _as_int(s: str) -> int | None: + # Collapse whitespace; then the whole string must be optional sign + digits + # with only non-word punctuation wrappers (prefix must not eat the sign). + compact = re.sub(r"\s+", "", s or "") + m = re.fullmatch(r"[^\w+-]*([+-]?\d+)[^\w+-]*", compact, flags=re.UNICODE) + if m: + return int(m.group(1)) + return None + + blob = text or "" + v = _as_int(blob) + if v is not None: + return v + lines = [ln for ln in blob.splitlines() if ln.strip()] + if lines: + return _as_int(lines[-1]) + return None + + +def reports_contract_int(text: str, truth: int) -> bool: + """True when final text reports ``truth`` via the explicit ``count: N`` contract. + + 1. Scan lines matching ``^count:\\s*(-?\\d+)\\s*$`` (case-insensitive, surrounding + whitespace allowed). Use the **last** match; require signed equality with + ``truth``. + 2. Fallback: whole-answer / last-line bare integer (:func:`whole_answer_int`). + 3. No match at all → False (ignoring an explicit format instruction is a fail). + """ + last: int | None = None + for line in (text or "").splitlines(): + m = re.fullmatch(r"\s*count:\s*(-?\d+)\s*", line, flags=re.IGNORECASE) + if m: + last = int(m.group(1)) + if last is not None: + return last == int(truth) + whole = whole_answer_int(text) + if whole is not None: + return whole == int(truth) + return False + + +def contract_values(text: str, field: str) -> list[str]: + """Return non-empty values from exact ``field: value`` contract lines. + + The field name is case-insensitive, as with :func:`reports_contract_int`, + while the value is preserved for exact comparison. Prose, bullets, inline + mentions, and malformed/empty contract lines are ignored. + """ + values: list[str] = [] + pattern = re.compile(rf"\s*{re.escape(field)}:\s*(.*?)\s*", flags=re.IGNORECASE) + for line in (text or "").splitlines(): + match = pattern.fullmatch(line) + if match and match.group(1): + values.append(match.group(1)) + return values + + +def reports_contract_value(text: str, field: str, truth: str) -> bool: + """True when exactly one ``field: value`` line equals ``truth`` exactly.""" + return contract_values(text, field) == [str(truth)] + + +def reports_contract_values(text: str, field: str, truths: list[str] | tuple[str, ...]) -> bool: + """True when contract lines equal the expected value multiset. + + Ordering is deliberately ignored: the output contract defines one exact + fact per line, not a presentation order. Missing, duplicate, or extra field + lines fail. + """ + return Counter(contract_values(text, field)) == Counter(str(value) for value in truths) + + +def get_final_text(run: dict[str, Any]) -> str: + return run.get("final_text") or "" + + +def normalize_rich_text(value: Any) -> str: + """Return exact comparable text from a rich-text API model, mapping, or string. + + Prefer authoritative stripped fields when the API exposes them, then normalize HTML + entities, tags, and whitespace. Case and punctuation remain significant. + """ + + def field(name: str) -> Any: + return value.get(name) if isinstance(value, dict) else getattr(value, name, None) + + candidates = ( + value if isinstance(value, str) else None, + field("comment_stripped"), + field("description_stripped"), + field("comment_html"), + field("description_html"), + ) + for candidate in candidates: + if isinstance(candidate, str) and candidate.strip(): + without_tags = re.sub(r"<[^>]*>", " ", candidate) + return " ".join(unescape(without_tags).split()) + return "" + + +def has_response_evidence(run: dict[str, Any], label: str = TARGET_ENTITY_EVIDENCE) -> bool: + """Return whether a successful Plane response exposed the target's hidden fact. + + Tool identity is deliberately irrelevant. The transport records only non-sensitive + sentinel labels after matching the in-memory response; no response body is required. + """ + calls = run.get("calls") + if not isinstance(calls, list): + return False + return any( + isinstance(call, dict) and not bool(call.get("is_error")) and label in (call.get("observed_sentinels") or []) + for call in calls + ) + + +def answer_with_provenance( + answer_correct: bool, + answer_note: str, + run: dict[str, Any], +) -> tuple[bool, str]: + """Combine answer correctness with route-agnostic response evidence. + + The two facts stay separate in the note. Provenance needs a seeded value to have + appeared in a response the agent received; a run of successful calls that never + surfaced one does not satisfy it. + """ + calls = run.get("calls") + source = str(run.get("call_source") or "unknown") + trace_incomplete = run.get("trace_integrity") is False + available = bool(run.get("evidence_trace_available")) + provenance = not trace_incomplete and has_response_evidence(run) + if trace_incomplete: + provenance_note = f"trace incomplete (source={source}; proxy sidecar was not authoritative)" + elif provenance: + provenance_note = f"observed seeded-value response evidence (source={source})" + elif not available: + provenance_note = f"unavailable (source={source}; response-evidence matching was not active)" + elif isinstance(calls, list) and calls: + successful = sum(1 for call in calls if isinstance(call, dict) and not bool(call.get("is_error"))) + provenance_note = ( + f"missing (0 evidence-bearing of {successful} successful Plane calls; {len(calls)} total; source={source})" + ) + else: + provenance_note = f"missing (0 Plane calls observed; source={source})" + note = f"answer_correct={str(bool(answer_correct)).lower()} ({answer_note}); provenance={provenance_note}" + return bool(answer_correct) and provenance, note + + +__all__ = [ + "contract_values", + "answer_with_provenance", + "get_final_text", + "has_response_evidence", + "normalize_rich_text", + "reports_contract_int", + "reports_contract_value", + "reports_contract_values", + "reports_exact_int", + "whole_answer_int", + "word_boundary", +] diff --git a/evals/tasks/catalog.py b/evals/tasks/catalog.py new file mode 100644 index 0000000..2cb2817 --- /dev/null +++ b/evals/tasks/catalog.py @@ -0,0 +1,178 @@ +"""Task catalog assembly, lookup, authorship, and fingerprinting.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from evals.tasks.cross import CROSS_TASKS +from evals.tasks.debias import DEBIAS_TASKS +from evals.tasks.read import READ_TASKS +from evals.tasks.schema import SCHEMA_TASKS +from evals.tasks.write import WRITE_TASKS + +EXPECTED_TASK_IDS = ( + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "W11", + "S1", + "S2", + "S3", + "S4", + "S5", + "C1", + "C2", + "R7", + "I1", + "I2", + "I3", + "I4", + "I5", + "L1", + "L2", + "L3", + "L4", + "L5", +) + +# Preserve the historical catalog order exactly: R7 was added after C1/C2. +TASKS: list[dict[str, Any]] = [ + *READ_TASKS[:6], + *WRITE_TASKS, + *SCHEMA_TASKS, + *CROSS_TASKS, + READ_TASKS[6], + *DEBIAS_TASKS, +] +if tuple(task["id"] for task in TASKS) != EXPECTED_TASK_IDS: + raise RuntimeError("assembled task order changed; battery/result compatibility would break") + +TASKS_BY_ID: dict[str, dict[str, Any]] = {task["id"]: task for task in TASKS} + + +def get_tasks(ids: list[str] | None = None) -> list[dict[str, Any]]: + """Return tasks filtered by id list (None = all).""" + if ids is None: + return list(TASKS) + missing = [i for i in ids if i not in TASKS_BY_ID] + if missing: + raise SystemExit(f"Unknown task id(s): {', '.join(missing)}. Known: {', '.join(TASKS_BY_ID)}") + return [TASKS_BY_ID[i] for i in ids] + + +def task_author(task: dict[str, Any]) -> str: + """Return the task author; default ``claude`` when the key is absent.""" + return str(task.get("author") or "claude") + + +CATALOG_REVISION = 12 +"""Bumped when a deliberate change to a fixture or verifier redefines what a task asks. + +Revision 12 also disambiguates L1's answer contract. It asked for "exactly one +'logged-minutes: 90' line and one 'summary-work-item-id' line for every row", which was +unambiguous only while the summary held a single row. With a seeded second row every agent read +"for every row" as governing both clauses and emitted one logged-minutes line per row — wrong for +a row whose seeded duration is not 90. The prompt now separates the two clauses and says other +items may already carry worklogs. + +Revision 12 gives R6's second project a Bug type of its own. Work item types are project-owned +unless the workspace owns them, so creating that project's bugs with the main project's type id +left them invisible to an agent resolving 'Bug' inside it: the agent counted zero there and named +the main project, always in that direction, while the oracle read those ids back directly and +disagreed. The seeded counts are unchanged; what changes is that the answer is now findable. + +Revision 11 gives L1 a worklog it did not create, on an item it is not told about. Its answer +was the id of the item it had just logged time on, and that id is echoed by the write itself, +so both the answer and its provenance were satisfied without ever reading the project worklog +summary the task exists to exercise. L1 results are not comparable across this transition. + +Revision 10 replaces the per-task list of accepted provenance shapes with one rule per kind of +evidence. A sentinel is a per-run random string that exists only inside Plane, so its presence +in a response the agent received proves surface use on its own; the request no longer has to +name a particular entity. A count is guessable, so it still counts only from a request naming a +seeded entity, and R6 accepts one count per project as well as one count grouped by project. +The old rule enumerated routes through a 183-action surface and could never be complete: it +rejected reading a state by listing a project's states, finding a cycle by listing a project's +cycles, and counting two projects separately — all correct answers scored as unproven. Every +read task's results are not comparable across this transition. +Revision 9 also binds R1/I2 provenance to the seeded state, not the work item alone. A work +item's `state` is an id, so resolving its name takes a second call, and the old rule needed the +target id and the answer in one response — unsatisfiable against a surface that does not expand +state. Both tasks failed every repetition while answering correctly. +Revision 8 binds L2's provenance to the activity count its verifier already checks, instead of +requiring the seeded comment phrase to appear in the activity readback. Plane's activity API +never emits comment text — it returns the creation row only — so the old requirement was +unsatisfiable and L2 failed in seeding on every repetition. L2 results are not comparable +across this transition, because before it there were none. +Revision 7 makes read tasks require response evidence bound to the target entity and gives +C2/R7 randomised, immutable seed-time oracles; pre-revision read results are not comparable. +Revision 6 stops W8 and W9 asking for unverifiable logged-date and batching properties; +it also tightens W3/W10 end-state contracts and paginates affected verifier reads. Revision +5 makes R1-R6, I2, L2, and L5 require observed successful Plane tool-call provenance and +randomises their hidden truth. Read results across either transition are not comparable. +Revision 4 rewrote R7 into an exact live state-and-group listing. Revision 3 removed +declared per-task tool sets and call floors and added fixture names. Fixture names being +covered means swapping a task's fixtures no longer needs a manual bump; changing a seeder's +behaviour under the same name still does. +""" + + +def task_fingerprint_payload(task: dict[str, Any]) -> dict[str, Any]: + """Return the canonical question payload shared by task and battery hashes.""" + return { + "id": task.get("id"), + "prompt": task.get("prompt"), + "needs": sorted(task.get("needs") or []), + } + + +def _short_fingerprint(document: Any) -> str: + blob = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + + +def task_fingerprint(task: dict[str, Any]) -> str: + """Return a stable short hash of one task's own question payload.""" + return _short_fingerprint(task_fingerprint_payload(task)) + + +def battery_fingerprint(tasks: list[dict[str, Any]] | None = None) -> str: + """Stable short hash of the revision and each task's ID, prompt and fixture names. + + Everything hashed is a fact about what the agent was asked — never an expectation + about how it should answer. Fixture *names* are covered, so swapping a task's + fixtures is caught mechanically; seeder and verifier *bodies* are not, which is the + hole CATALOG_REVISION exists to close by hand. A --tasks subset hashes differently + from the full catalog. + """ + src = list(TASKS if tasks is None else tasks) + payload = [task_fingerprint_payload(task) for task in sorted(src, key=lambda item: str(item.get("id") or ""))] + document = {"revision": CATALOG_REVISION, "tasks": payload} + return _short_fingerprint(document) + + +__all__ = [ + "EXPECTED_TASK_IDS", + "TASKS", + "TASKS_BY_ID", + "battery_fingerprint", + "get_tasks", + "task_author", + "task_fingerprint", + "task_fingerprint_payload", +] diff --git a/evals/tasks/cross.py b/evals/tasks/cross.py new file mode 100644 index 0000000..8484a1a --- /dev/null +++ b/evals/tasks/cross.py @@ -0,0 +1,216 @@ +"""Cross-entity task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from evals.core.changelog import changelog_items, normalize_changelog_text +from evals.core.fixtures import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + R1_TITLE, +) +from evals.tasks.answers import ( + answer_with_provenance, + contract_values, + get_final_text, + reports_contract_value, + reports_contract_values, +) +from evals.tasks.lookups import as_id, collect_paginated, find_item_by_name, ids +from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error + + +async def verify_c1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """C1: customer 'Acme Corp' has request 'SSO support' linked to the R1 work item. + + Anchors: exact customer name, exact request name, and the R1_TITLE work item id + resolved from the eval project at verify time (must be among linked ids). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + # Resolve the required link target first (seeded Payment webhook item). + r1 = find_item_by_name(plane, workspace_slug, project_id, R1_TITLE) + if r1 is None: + return False, f"R1 item {R1_TITLE!r} not found in project" + + rows = collect_paginated( + lambda cursor: plane.customers.list( + workspace_slug=workspace_slug, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) + # Exact name only — do not match arbitrary acme* customers. + acme = next((c for c in (rows or []) if (c.name or "").strip() == CUSTOMER_NAME), None) + if acme is None: + return False, f"customer {CUSTOMER_NAME!r} not found" + + # Track for teardown if agent-created + if not ctx.get("customer"): + ctx.setdefault("workspace_objects", []).append({"kind": "customer", "id": acme.id}) + + rrows = collect_paginated( + lambda cursor: plane.customers.requests.list( + workspace_slug=workspace_slug, + customer_id=acme.id, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) + sso = next( + (r for r in (rrows or []) if (r.name or "").strip() == CUSTOMER_REQUEST_NAME), + None, + ) + if sso is None: + ok = False + notes.append(f"request {CUSTOMER_REQUEST_NAME!r} missing") + else: + notes.append("SSO request present") + + # Require the R1 work item among customer-linked work items. + try: + wi_rows = collect_paginated( + lambda cursor: plane.customers.work_items.list( + workspace_slug=workspace_slug, + customer_id=acme.id, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) + linked_ids = ids(wi_rows) + # Plain string ids also count. + for row in wi_rows: + if isinstance(row, str): + linked_ids.add(row) + elif isinstance(row, dict) and row.get("id"): + linked_ids.add(str(row["id"])) + else: + # Customer work item wrappers may expose work_item / issue field. + for attr in ("work_item", "work_item_id", "issue", "issue_id"): + ref = getattr(row, attr, None) if not isinstance(row, dict) else row.get(attr) + rid = as_id(ref) + if rid: + linked_ids.add(str(rid)) + if str(r1.id) not in linked_ids: + ok = False + notes.append(f"R1 item {r1.id} not linked; linked={sorted(linked_ids)}") + else: + notes.append(f"R1 item {r1.id} linked") + except Exception as exc: + raise_verifier_read_error("C1", f"listing work items linked to customer {acme.id}", exc) + + return ok, "; ".join(notes) + + +C1_TASK: dict[str, Any] = { + "id": "C1", + "tags": {"write"}, + "prompt": ( + f"Create customer '{CUSTOMER_NAME}' (if it does not already exist), add a " + f"request named '{CUSTOMER_REQUEST_NAME}', and link that request to the work " + f"item '{R1_TITLE}' in project {{project}}." + ), + # No pre-seeded customer — agent creates; items needed for link target. + "needs": {"items"}, + "verify": verify_c1, +} + + +async def verify_c2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """C2: report the immutable release baseline with target-response evidence.""" + final_text = get_final_text(run) + notes: list[str] = [] + ok = True + release = ctx.get("release") or {} + expected_release = str( + ctx.get("release_name") + or (release.get("name") if isinstance(release, dict) else getattr(release, "name", None)) + or "" + ) + if not expected_release: + return answer_with_provenance(False, "fixture missing: seeded release name is unavailable", run) + if not reports_contract_value(final_text, "release", expected_release): + ok = False + notes.append(f"release values={contract_values(final_text, 'release')!r}; want [{expected_release!r}]") + else: + notes.append(f"release={expected_release!r}") + + release_id = release.get("id") if isinstance(release, dict) else getattr(release, "id", release) + if not release_id: + return answer_with_provenance(False, "fixture missing: seeded release id is unavailable", run) + baseline = ctx.get("release_changelog_text") + if not isinstance(baseline, str) or not baseline.strip(): + return answer_with_provenance(False, "fixture missing: seeded changelog baseline is empty", run) + try: + live_release = plane.releases.retrieve( + workspace_slug=ctx["workspace_slug"], + release_id=release_id, + ) + response = plane.releases.changelog.retrieve( + workspace_slug=ctx["workspace_slug"], + release_id=release_id, + ) + except Exception as exc: + if is_verifier_not_found(exc): + return answer_with_provenance( + False, + f"seeded release/changelog no longer exists at verification ({exc})", + run, + ) + raise_verifier_read_error("C2", f"reading release {release_id} and its changelog", exc) + live_release_name = str(getattr(live_release, "name", None) or "").strip() + if live_release_name != expected_release: + return answer_with_provenance( + False, + f"release name was mutated after seeding: live={live_release_name!r}; baseline={expected_release!r}", + run, + ) + live = normalize_changelog_text(response) + if live != baseline: + if not live: + mutation_note = "changelog was mutated after seeding: live changelog is empty" + else: + mutation_note = f"changelog was mutated after seeding: live={live!r}; baseline={baseline!r}" + return answer_with_provenance(False, mutation_note, run) + shipped = changelog_items(baseline) + if not shipped: + return answer_with_provenance( + False, + f"fixture missing: seeded changelog baseline has no parseable entries: {baseline!r}", + run, + ) + if not reports_contract_values(final_text, "shipped", shipped): + ok = False + notes.append(f"shipped values={contract_values(final_text, 'shipped')!r}; want {shipped!r}") + else: + notes.append(f"{len(shipped)} exact shipped items") + return answer_with_provenance(ok, "; ".join(notes), run) + + +def _bind_c2(ctx: dict[str, Any]) -> dict[str, str]: + release = ctx.get("release") or {} + name = ctx.get("release_name") or (release.get("name") if isinstance(release, dict) else None) + return {"release_name": str(name or "")} + + +C2_TASK: dict[str, Any] = { + "id": "C2", + "tags": {"read"}, + "prompt": ( + "What shipped in release {release_name}? Summarize the changelog in any prose " + "you like, then provide these exact contract lines: 'release: {release_name}' " + "and one 'shipped: ' line per changelog item. " + "For each 'shipped:' value, copy only the text after the changelog entry label, " + "without its sentence-ending punctuation." + ), + "prompt_bind": _bind_c2, + "needs": {"release"}, + "verify": verify_c2, +} + + +CROSS_TASKS: list[dict[str, Any]] = [C1_TASK, C2_TASK] + + +__all__ = ["CROSS_TASKS", "verify_c1", "verify_c2"] diff --git a/evals/tasks/debias.py b/evals/tasks/debias.py new file mode 100644 index 0000000..f8e7d5a --- /dev/null +++ b/evals/tasks/debias.py @@ -0,0 +1,528 @@ +"""ID-in-hand and long-tail de-biasing tasks with their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from plane.models.query_params import RetrieveQueryParams + +from evals.core.fixtures import ( + CUSTOMER_NAME, + CYCLE_CURRENT, + DEBIAS_CUSTOMER_PROP_DISPLAY, + DEBIAS_RELEASE_TAG_VERSION, + R1_TITLE, + R5_TITLE, + W2_TITLE, + W3_TITLE, + W8_TITLE, +) +from evals.core.state_oracle import worklog_summary_item_ids +from evals.tasks.answers import ( + answer_with_provenance, + contract_values, + get_final_text, + reports_contract_int, + reports_contract_value, + reports_contract_values, +) +from evals.tasks.lookups import collect_paginated, ids +from evals.tasks.verification import raise_verifier_read_error + +I1_TITLE = R1_TITLE + + +I2_TITLE = W2_TITLE + + +I3_TITLE = "Footer year still says 2024" + + +I4_TITLE = W3_TITLE + + +L1_TITLE = W8_TITLE + + +L2_TITLE = R5_TITLE + + +L5_TITLE = R1_TITLE + + +L3_TAG_VERSION = DEBIAS_RELEASE_TAG_VERSION + + +L4_PROP_DISPLAY = DEBIAS_CUSTOMER_PROP_DISPLAY + + +L4_PROP_VALUE = "Enterprise" + + +def _bind_item_uuid(title: str): + def _bind(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(title) or "") + return {"work_item_id": wid} + + return _bind + + +def _bind_item_identifier(title: str): + def _bind(ctx: dict[str, Any]) -> dict[str, str]: + ident = str((ctx.get("item_identifiers") or {}).get(title) or "") + return {"work_item_identifier": ident} + + return _bind + + +def _bind_i3(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") + cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") + return {"work_item_id": wid, "cycle_id": cycle_id} + + +def _bind_i4(ctx: dict[str, Any]) -> dict[str, str]: + wid = str((ctx.get("items") or {}).get(I4_TITLE) or "") + label_id = str((ctx.get("labels") or {}).get("perf") or "") + return {"work_item_id": wid, "label_id": label_id} + + +async def verify_i1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I1: seeded R1 item priority is high (updated by UUID, not name).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I1_TITLE) + if not wid: + return False, f"seed item {I1_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + pr = (detail.priority or "").lower() if detail.priority else "" + if pr == "high": + return True, f"work_item {wid} priority=high" + return False, f"work_item {wid} priority={pr!r} (want high)" + + +I1_TASK: dict[str, Any] = { + "id": "I1", + "author": "post-hoc-debias", + "tags": {"write", "id_in_hand", "debias"}, + "prompt": ("In project {project}, update work item {work_item_id}: set its priority to high."), + "prompt_bind": _bind_item_uuid(I1_TITLE), + "needs": {"items"}, + "verify": verify_i1, +} + + +async def verify_i2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I2: final text reports the API-confirmed seed state with call provenance.""" + name = str(ctx.get("i2_state_name") or "") + if not name: + return answer_with_provenance(False, "API-confirmed target state missing from seed ctx", run) + final_text = get_final_text(run) + answer_correct = reports_contract_value(final_text, "state", name) + answer_note = ( + f"final text reports state {name!r} via contract" + if answer_correct + else f"state values={contract_values(final_text, 'state')!r}; want [{name!r}]" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +I2_TASK: dict[str, Any] = { + "id": "I2", + "author": "post-hoc-debias", + "tags": {"read", "id_in_hand", "debias"}, + "prompt": ( + "In project {project}, what is the current state of work item " + "{work_item_identifier}? Return exactly one line: 'state: '." + ), + "prompt_bind": _bind_item_identifier(I2_TITLE), + "needs": {"items"}, + "verify": verify_i2, +} + + +async def verify_i3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I3: work item UUID is on the target cycle UUID.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = str((ctx.get("items") or {}).get(I3_TITLE) or "") + cycle_id = str(ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) or "") + if not wid or not cycle_id: + return False, "seed work_item_id/cycle_id missing" + page = plane.cycles.list_work_items(workspace_slug=workspace_slug, project_id=project_id, cycle_id=cycle_id) + rows = page.results if hasattr(page, "results") else page + ids = {str(getattr(r, "id", None) or r) for r in (rows or [])} + # list may return issue wrappers with issue/id fields + for r in rows or []: + for attr in ("id", "issue", "work_item_id"): + v = getattr(r, attr, None) + if v is not None: + ids.add(str(v if not hasattr(v, "id") else v.id)) + if wid in ids: + return True, f"item {wid} on cycle {cycle_id}" + return False, f"item {wid} not on cycle {cycle_id}; have {sorted(ids)[:12]}" + + +I3_TASK: dict[str, Any] = { + "id": "I3", + "author": "post-hoc-debias", + "tags": {"write", "id_in_hand", "debias"}, + "prompt": ("In project {project}, add work item {work_item_id} to cycle {cycle_id}."), + "prompt_bind": _bind_i3, + "needs": {"items", "cycles"}, + "verify": verify_i3, +} + + +async def verify_i4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I4: work item has the seeded perf label id attached.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I4_TITLE) + label_id = (ctx.get("labels") or {}).get("perf") + if not wid or not label_id: + return False, "seed work_item_id/label_id missing" + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=wid, + params=RetrieveQueryParams(expand="labels"), + ) + label_ids = ids(detail.labels) + if str(label_id) in label_ids: + return True, f"label {label_id} on {wid}" + return False, f"labels={sorted(label_ids)} missing perf={label_id}" + + +I4_TASK: dict[str, Any] = { + "id": "I4", + "author": "post-hoc-debias", + "tags": {"write", "id_in_hand", "debias"}, + "prompt": ("In project {project}, attach label {label_id} to work item {work_item_id}."), + "prompt_bind": _bind_i4, + "needs": {"items", "labels"}, + "verify": verify_i4, +} + + +async def verify_i5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """I5: target item priority is low (updated by UUID).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(I3_TITLE) + if not wid: + return False, f"seed item {I3_TITLE!r} missing" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + pr = (detail.priority or "").lower() if detail.priority else "" + if pr == "low": + return True, f"work_item {wid} priority=low" + return False, f"work_item {wid} priority={pr!r} (want low)" + + +I5_TASK: dict[str, Any] = { + "id": "I5", + "author": "post-hoc-debias", + "tags": {"write", "id_in_hand", "debias"}, + "prompt": ("In project {project}, set the priority of work item {work_item_id} to low."), + "prompt_bind": _bind_item_uuid(I3_TITLE), # footer item; not high-traffic elsewhere + "needs": {"items"}, + "verify": verify_i5, +} + + +async def verify_l1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L1: 90-minute log exists and reporting uses an immutable target-id oracle.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + wid = (ctx.get("items") or {}).get(L1_TITLE) + if not wid: + return answer_with_provenance(False, f"seed item {L1_TITLE!r} missing", run) + expected_summary_ids = [str(value) for value in (ctx.get("l1_expected_summary_ids") or [])] + if str(wid) not in expected_summary_ids: + return answer_with_provenance( + False, + f"L1 fixture oracle mismatch: summary ids={expected_summary_ids!r} omit target={wid!r}", + run, + ) + if len(expected_summary_ids) < 2: + return answer_with_provenance( + False, + f"L1 fixture error: the summary oracle {expected_summary_ids!r} holds only the agent's own " + "row, so the answer does not require reading the summary", + run, + ) + # SDK: 90m log must be on THIS work item (list is already scoped to work_item_id). + logs = plane.work_items.work_logs.list(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 90 not in durations: + return answer_with_provenance(False, f"no 90-minute work log on target item {wid}; durations={durations}", run) + + try: + summary = plane.projects.get_worklog_summary(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + raise_verifier_read_error("L1", "reading the project worklog summary", exc) + + summary_ids = worklog_summary_item_ids(summary) + if str(wid) not in summary_ids: + return answer_with_provenance( + False, + f"target item {wid} missing from project worklog summary ids={summary_ids!r}", + run, + ) + if sorted(summary_ids) != sorted(expected_summary_ids): + return answer_with_provenance( + False, + f"worklog summary was mutated beyond the seeded oracle: live={summary_ids!r}; " + f"expected={expected_summary_ids!r}", + run, + ) + + final_text = get_final_text(run) + if not reports_contract_value(final_text, "logged-minutes", "90"): + return answer_with_provenance( + False, + f"logged-minutes values={contract_values(final_text, 'logged-minutes')!r}; want ['90']", + run, + ) + if not reports_contract_values(final_text, "summary-work-item-id", expected_summary_ids): + return answer_with_provenance( + False, + f"summary-work-item-id values={contract_values(final_text, 'summary-work-item-id')!r}; " + f"want {expected_summary_ids!r}", + run, + ) + return answer_with_provenance( + True, + f"90m log on {wid} + exact contract for {len(expected_summary_ids)} immutable summary row(s)", + run, + ) + + +L1_TASK: dict[str, Any] = { + "id": "L1", + "author": "post-hoc-debias", + "tags": {"write", "read", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, log 1.5 hours (90 minutes) of work on the item titled " + f"'{L1_TITLE}', then report the project's worklog summary. End with exactly one " + "'logged-minutes: 90' line, and then, for each row the project worklog summary " + "returns, one 'summary-work-item-id: ' line. The project " + "may already have worklogs on other items. Include no other lines with those prefixes." + ), + "needs": {"items"}, + "verify": verify_l1, +} + + +async def verify_l2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L2: final text reports the API-confirmed seed activity count with provenance.""" + n = ctx.get("l2_activity_count") + if not isinstance(n, int) or n < 1: + return answer_with_provenance(False, "API-confirmed activity count missing from seed ctx", run) + final_text = get_final_text(run) + answer_correct = reports_contract_int(final_text, n) + answer_note = ( + f"final text reports activity count {n} via contract" + if answer_correct + else f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +L2_TASK: dict[str, Any] = { + "id": "L2", + "author": "post-hoc-debias", + "tags": {"read", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, list the activity history for the work item titled " + f"'{L2_TITLE}'. Summarize how many activities there are and mention any " + "notable comment phrases you see. End your answer with a line of the form " + "'count: N' where N is the number of activities." + ), + "needs": {"items", "activity_feed"}, + "verify": verify_l2, +} + + +async def verify_l3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L3: workspace has a release tag with version eval-rc1.""" + workspace_slug = ctx["workspace_slug"] + try: + rows = collect_paginated( + lambda cursor: plane.releases.tags.list( + workspace_slug=workspace_slug, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) + except Exception as exc: + raise_verifier_read_error("L3", "listing workspace release tags", exc) + versions = {(getattr(t, "version", None) or "").strip() for t in (rows or [])} + if L3_TAG_VERSION in versions: + # Track for teardown if id available. + for t in rows or []: + if (getattr(t, "version", None) or "").strip() == L3_TAG_VERSION: + tid = getattr(t, "id", None) + if tid: + objs = ctx.setdefault("workspace_objects", []) + if not any(o.get("kind") == "release_tag" and str(o.get("id")) == str(tid) for o in objs): + objs.append({"kind": "release_tag", "id": tid}) + break + return True, f"release tag {L3_TAG_VERSION!r} present" + return False, f"tag {L3_TAG_VERSION!r} missing; have {sorted(versions)}" + + +L3_TASK: dict[str, Any] = { + "id": "L3", + "author": "post-hoc-debias", + "tags": {"write", "long_tail", "debias"}, + "prompt": (f"Create a release tag with version '{L3_TAG_VERSION}' (a version marker for the eval run)."), + "needs": set(), # workspace-level tag; no project fixture required + "verify": verify_l3, +} + + +def _property_type_is_text(prop: Any) -> bool: + raw = getattr(prop, "property_type", None) + if raw is None: + raw = getattr(prop, "type", None) + if raw is None: + return False + if hasattr(raw, "value"): + raw = raw.value + if hasattr(raw, "name"): + # Enum member: PropertyType.TEXT + name = str(raw.name) + if name.upper() == "TEXT": + return True + s = str(raw).upper() + return s == "TEXT" or s.endswith(".TEXT") or s == "PROPERTYTYPE.TEXT" + + +async def verify_l4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L4: right customer has TEXT property 'Eval Industry' = Enterprise (exact name).""" + workspace_slug = ctx["workspace_slug"] + cust = ctx.get("customer") or {} + customer_id = cust.get("id") if isinstance(cust, dict) else cust + if not customer_id: + return False, "customer missing from seed" + try: + prop_rows = collect_paginated( + lambda cursor: plane.customers.properties.list( + workspace_slug=workspace_slug, + params={"per_page": 100, **({"cursor": cursor} if cursor else {})}, + ) + ) + except Exception as exc: + raise_verifier_read_error("L4", "listing workspace customer properties", exc) + target_prop: Any | None = None + for p in prop_rows or []: + # Exact display_name match only (case-insensitive full match) — not substring "Industry". + display = (getattr(p, "display_name", None) or "").strip() + if display.casefold() != L4_PROP_DISPLAY.casefold(): + continue + if not _property_type_is_text(p): + return False, ( + f"property {display!r} exists but property_type is not TEXT (got {getattr(p, 'property_type', None)!r})" + ) + target_prop = p + break + if target_prop is None: + return False, f"no TEXT customer property named exactly {L4_PROP_DISPLAY!r}" + pid = str(target_prop.id) + # Track for teardown. + objs = ctx.setdefault("workspace_objects", []) + if not any(o.get("kind") == "customer_property" and str(o.get("id")) == pid for o in objs): + objs.append({"kind": "customer_property", "id": pid}) + try: + values = plane.customers.property_values.list(workspace_slug=workspace_slug, customer_id=customer_id) + except Exception as exc: + raise_verifier_read_error("L4", f"reading property values for customer {customer_id}", exc) + if not isinstance(values, dict): + return False, f"unexpected property_values shape: {type(values)}" + vals = values.get(pid) or values.get(str(pid)) or [] + flat = [str(v) for v in (vals if isinstance(vals, list) else [vals])] + if any(L4_PROP_VALUE.casefold() == v.casefold() for v in flat): + return True, f"customer {customer_id} property {pid} ({L4_PROP_DISPLAY})={L4_PROP_VALUE!r}" + return False, f"customer {customer_id} property {pid} values {flat} lack {L4_PROP_VALUE!r}" + + +L4_TASK: dict[str, Any] = { + "id": "L4", + "author": "post-hoc-debias", + "tags": {"write", "long_tail", "debias"}, + "prompt": ( + f"For customer '{CUSTOMER_NAME}', ensure there is a text customer property " + f"named '{L4_PROP_DISPLAY}' and set its value to '{L4_PROP_VALUE}'." + ), + "needs": {"customer"}, + "verify": verify_l4, +} + + +async def verify_l5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """L5: final text reports the API-confirmed seed attachment count with provenance.""" + n = ctx.get("l5_attachment_count") + if not isinstance(n, int): + return answer_with_provenance(False, "API-confirmed attachment count missing from seed ctx", run) + final_text = get_final_text(run) + answer_correct = reports_contract_int(final_text, n) + answer_note = ( + f"final text reports attachment count {n} via contract" + if answer_correct + else f"final text missing contract count: {n} (need 'count: {n}' or bare integer)" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +L5_TASK: dict[str, Any] = { + "id": "L5", + "author": "post-hoc-debias", + "tags": {"read", "long_tail", "debias"}, + "prompt": ( + f"In project {{project}}, how many file attachments does the work item titled " + f"'{L5_TITLE}' have? End your answer with a line of the form 'count: N' " + "where N is the number of file attachments." + ), + "needs": {"items"}, + "verify": verify_l5, +} + + +DEBIAS_TASKS: list[dict[str, Any]] = [ + I1_TASK, + I2_TASK, + I3_TASK, + I4_TASK, + I5_TASK, + L1_TASK, + L2_TASK, + L3_TASK, + L4_TASK, + L5_TASK, +] + + +__all__ = [ + "DEBIAS_TASKS", + "I1_TITLE", + "I2_TITLE", + "I3_TITLE", + "I4_TITLE", + "L1_TITLE", + "L2_TITLE", + "L3_TAG_VERSION", + "L4_PROP_DISPLAY", + "L4_PROP_VALUE", + "L5_TITLE", + "verify_i1", + "verify_i2", + "verify_i3", + "verify_i4", + "verify_i5", + "verify_l1", + "verify_l2", + "verify_l3", + "verify_l4", + "verify_l5", +] diff --git a/evals/tasks/lookups.py b/evals/tasks/lookups.py new file mode 100644 index 0000000..c5139f1 --- /dev/null +++ b/evals/tasks/lookups.py @@ -0,0 +1,162 @@ +"""Plane reads used by task verifiers to establish truth.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.query_params import WorkItemQueryParams + + +def as_id(obj: Any) -> str | None: + if obj is None: + return None + if isinstance(obj, str): + return obj + return getattr(obj, "id", None) or (obj.get("id") if isinstance(obj, dict) else None) + + +def ids(items: Any) -> set[str]: + out: set[str] = set() + for item in items or []: + i = as_id(item) + if i: + out.add(str(i)) + return out + + +def collect_paginated(fetch_page: Callable[[str | None], Any]) -> list[Any]: + """Collect every result from a cursor-paginated SDK endpoint. + + ``fetch_page`` receives ``None`` for the first request and the prior response's + ``next_cursor`` thereafter. Endpoints documented as unpaginated may return a bare + list; in that case the list is already complete. + """ + rows: list[Any] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + while True: + page = fetch_page(cursor) + if isinstance(page, list): + rows.extend(page) + return rows + results = page.results if hasattr(page, "results") else page + rows.extend(list(results or [])) + if not bool(getattr(page, "next_page_results", False)): + return rows + next_cursor = str(getattr(page, "next_cursor", None) or "") + if not next_cursor or next_cursor in seen_cursors: + raise RuntimeError("paginated API reported another page without a new next_cursor") + seen_cursors.add(next_cursor) + cursor = next_cursor + + +def find_items_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> list[Any]: + """Return all work items with exact name, newest first (by created_at).""" + matches: list[Any] = [] + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + page = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in page.results or []: + if (item.name or "").strip() == name: + matches.append(item) + if not page.next_page_results: + break + cursor = page.next_cursor + + def _created_key(item: Any) -> str: + return str(getattr(item, "created_at", None) or "") + + matches.sort(key=_created_key, reverse=True) + return matches + + +def find_item_by_name(plane: Any, workspace_slug: str, project_id: str, name: str) -> Any | None: + """Locate a work item by exact name; when duplicates exist, prefer the newest.""" + matches = find_items_by_name(plane, workspace_slug, project_id, name) + return matches[0] if matches else None + + +def state_name(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + """Resolve a state UUID or expanded object to its display name.""" + if state_ref is None: + return None + if hasattr(state_ref, "name") and state_ref.name: + return str(state_ref.name) + if isinstance(state_ref, dict) and state_ref.get("name"): + return str(state_ref["name"]) + state_id = as_id(state_ref) + if not state_id: + return None + try: + state = plane.states.retrieve(workspace_slug=workspace_slug, project_id=project_id, state_id=state_id) + return state.name + except HttpError as exc: + if exc.status_code not in (404, 405): + raise + # Fall back to listing states and matching by id. + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + results = page.results if hasattr(page, "results") else page + for s in results or []: + if str(s.id) == str(state_id): + return s.name + return None + + +def state_group(plane: Any, workspace_slug: str, project_id: str, state_ref: Any) -> str | None: + if state_ref is None: + return None + if hasattr(state_ref, "group") and state_ref.group: + return str(state_ref.group) + if isinstance(state_ref, dict) and state_ref.get("group"): + return str(state_ref["group"]) + state_id = as_id(state_ref) + if not state_id: + return None + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + results = page.results if hasattr(page, "results") else page + for s in results or []: + if str(s.id) == str(state_id): + return getattr(s, "group", None) + return None + + +def is_not_found(exc: BaseException) -> bool: + return isinstance(exc, HttpError) and exc.status_code in (404, 405) + + +def count_open_urgent(plane: Any, workspace_slug: str, project_id: str) -> int: + """Count urgent items whose state group is not completed/cancelled (resolve at verify).""" + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + closed_ids = {str(s.id) for s in (page.results or []) if getattr(s, "group", None) in ("completed", "cancelled")} + n = 0 + cursor = None + while True: + params = WorkItemQueryParams(cursor=cursor, per_page=100) if cursor else WorkItemQueryParams(per_page=100) + resp = plane.work_items.list(workspace_slug=workspace_slug, project_id=project_id, params=params) + for item in resp.results or []: + if (getattr(item, "priority", None) or "").lower() != "urgent": + continue + sid = as_id(item.state) + if sid and str(sid) in closed_ids: + continue + n += 1 + if not resp.next_page_results: + break + cursor = resp.next_cursor + return n + + +__all__ = [ + "as_id", + "collect_paginated", + "count_open_urgent", + "find_item_by_name", + "find_items_by_name", + "ids", + "is_not_found", + "state_group", + "state_name", +] diff --git a/evals/tasks/prompts.py b/evals/tasks/prompts.py new file mode 100644 index 0000000..8d2448a --- /dev/null +++ b/evals/tasks/prompts.py @@ -0,0 +1,63 @@ +"""Task prompt binding.""" + +from __future__ import annotations + +import string +from typing import Any + + +class PromptBindError(RuntimeError): + """Live prompt could not bind required seed IDs (classified as infra_seed).""" + + +def format_task_prompt( + task: dict[str, Any], + ctx: dict[str, Any] | None = None, + *, + strict: bool = False, +) -> str: + """Render a task prompt with seed-bound placeholders. + + Always supplies ``project``; tasks needing concrete UUIDs add keys via an optional + ``prompt_bind(ctx)``. Live runs use strict=True so an empty value or a binder error + raises PromptBindError and is recorded infra_seed, rather than sending the agent a + blank ID; dry runs fill missing keys with ```` markers instead. + """ + tpl = str(task.get("prompt") or "") + fields: dict[str, Any] = { + "project": (ctx or {}).get("project_name") or "EVAL deadbeef", + } + binder = task.get("prompt_bind") + if callable(binder) and ctx is not None: + try: + extra = binder(ctx) or {} + except Exception as exc: + if strict: + raise PromptBindError( + f"prompt_bind failed for task {task.get('id')}: {type(exc).__name__}: {exc}" + ) from exc + extra = {} + if isinstance(extra, dict): + for key, val in extra.items(): + if val is None: + if strict: + raise PromptBindError(f"prompt_bind returned None for {{{key}}} (task {task.get('id')})") + continue + text = str(val).strip() + if not text: + if strict: + raise PromptBindError(f"prompt_bind returned empty {{{key}}} for task {task.get('id')}") + continue + fields[key] = text + # Collect required placeholders from the template. + required = [name for _, name, _, _ in string.Formatter().parse(tpl) if name] + for name in required: + if name in fields and str(fields[name]).strip() and not str(fields[name]).startswith("<"): + continue + if strict: + raise PromptBindError(f"missing prompt field {{{name}}} for task {task.get('id')}") + fields.setdefault(name, f"<{name}>") + return tpl.format(**fields) + + +__all__ = ["PromptBindError", "format_task_prompt"] diff --git a/evals/tasks/read.py b/evals/tasks/read.py new file mode 100644 index 0000000..8e5c022 --- /dev/null +++ b/evals/tasks/read.py @@ -0,0 +1,253 @@ +"""Read-task definitions and their verifiers.""" + +from __future__ import annotations + +from collections import Counter +from typing import Any + +from evals.core.fixtures import R1_TITLE, R5_TITLE +from evals.core.state_oracle import state_name_group_pairs +from evals.tasks.answers import ( + answer_with_provenance, + contract_values, + get_final_text, + reports_contract_int, + reports_contract_value, + reports_contract_values, +) + + +async def verify_r1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R1: final text reports the API-confirmed seed state with call provenance.""" + expected = str(ctx.get("r1_state_name") or "") + if not expected: + return answer_with_provenance(False, "API-confirmed seed state missing", run) + + final_text = get_final_text(run) + answer_correct = reports_contract_value(final_text, "state", expected) + answer_note = ( + f"final text reports state {expected!r} via contract" + if answer_correct + else f"state values={contract_values(final_text, 'state')!r}; want [{expected!r}]" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +R1_TASK: dict[str, Any] = { + "id": "R1", + "tags": {"read"}, + "prompt": ( + "In project {project}, what is the current state of the work item titled " + f"'{R1_TITLE}'? Return exactly one line: 'state: '." + ), + "needs": {"items"}, + "verify": verify_r1, +} + + +async def verify_r2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R2: report the seed count with item-value or target-scoped total-count evidence.""" + expected = ctx.get("r2_urgent_open_count") + if not isinstance(expected, int): + return answer_with_provenance(False, "API-confirmed urgent-open seed count missing", run) + final_text = get_final_text(run) + answer_correct = reports_contract_int(final_text, expected) + answer_note = ( + f"final text reports urgent-open count {expected} via contract" + if answer_correct + else f"final text missing contract count: {expected} (need 'count: {expected}')" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +R2_TASK: dict[str, Any] = { + "id": "R2", + "tags": {"read"}, + "prompt": ( + "In project {project}, how many urgent open work items are there? " + "Return exactly one line of the form 'count: N', where N is the integer count." + ), + "needs": {"items"}, + "verify": verify_r2, +} + + +async def verify_r3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R3: ``item: TITLE`` lines exactly match the seeded due-title set.""" + titles = list(ctx.get("r3_due_titles") or []) + if not titles: + return answer_with_provenance(False, "no API-confirmed R3 due titles in seed ctx", run) + final_text = get_final_text(run) + answer_correct = reports_contract_values(final_text, "item", titles) + answer_note = ( + f"final text reports exactly {len(titles)} due-this-week assigned items" + if answer_correct + else f"item contract values={contract_values(final_text, 'item')!r}; want {titles!r}" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +R3_TASK: dict[str, Any] = { + "id": "R3", + "tags": {"read"}, + "prompt": ( + "In project {project}, list work items assigned to me that are due this week. " + "Return one line per result as 'item: ' and no other 'item:' lines." + ), + "needs": {"items"}, + "verify": verify_r3, +} + + +async def verify_r4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R4: contract reports the active cycle, all its items, and overdue items.""" + final_text = get_final_text(run) + notes: list[str] = [] + ok = True + cycle_name = str(ctx.get("r4_cycle_name") or "") + if not cycle_name: + ok = False + notes.append("API-confirmed active-cycle name missing from seed ctx") + elif not reports_contract_value(final_text, "cycle", cycle_name): + ok = False + notes.append(f"cycle values={contract_values(final_text, 'cycle')!r}; want [{cycle_name!r}]") + else: + notes.append(f"cycle={cycle_name!r}") + + active_titles = [str(value) for value in (ctx.get("r4_active_titles") or [])] + if not active_titles: + ok = False + notes.append("no active-cycle titles in seed ctx") + elif not reports_contract_values(final_text, "item", active_titles): + ok = False + notes.append(f"item values={contract_values(final_text, 'item')!r}; want {active_titles!r}") + else: + notes.append(f"{len(active_titles)} active-cycle items") + + overdue_titles = [str(value) for value in (ctx.get("r4_overdue_titles") or [])] + expected_overdue = overdue_titles or ["none"] + if not reports_contract_values(final_text, "overdue", expected_overdue): + ok = False + notes.append(f"overdue values={contract_values(final_text, 'overdue')!r}; want {expected_overdue!r}") + else: + notes.append(f"overdue={expected_overdue!r}") + return answer_with_provenance(ok, "; ".join(notes), run) + + +R4_TASK: dict[str, Any] = { + "id": "R4", + "tags": {"read"}, + "prompt": ( + "In project {project}, what is in the active cycle, and is anything overdue? " + "Use these exact contract lines: one 'cycle: ' line, one " + "'item: ' line for every item in that cycle, and one " + "'overdue: ' line for every overdue item. If none " + "are overdue, use 'overdue: none'." + ), + "needs": {"items", "cycles"}, + "verify": verify_r4, +} + + +async def verify_r5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R5: ``comment: TEXT`` lines exactly match the seeded comments.""" + phrases = list(ctx.get("r5_comment_phrases") or []) + if not phrases: + return answer_with_provenance(False, "no API-confirmed R5 comments in seed ctx", run) + final_text = get_final_text(run) + answer_correct = reports_contract_values(final_text, "comment", phrases) + answer_note = ( + f"final text reports exactly {len(phrases)} seeded comments" + if answer_correct + else f"comment values={contract_values(final_text, 'comment')!r}; want {phrases!r}" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +R5_TASK: dict[str, Any] = { + "id": "R5", + "tags": {"read"}, + "prompt": ( + f"In project {{project}}, summarize the discussion on the work item titled '{R5_TITLE}'. " + "You may summarize in prose, but end with one contract line per comment: " + "'comment: '. Copy the comment text exactly and include " + "no other 'comment:' lines." + ), + "needs": {"items"}, + "verify": verify_r5, +} + + +async def verify_r6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R6: report the winner with item-value or exact project-grouped-count evidence.""" + expected = str(ctx.get("r6_more_bugs_project") or "") + if not expected: + return answer_with_provenance(False, "API-confirmed R6 winner missing from seed ctx", run) + final_text = get_final_text(run) + answer_correct = reports_contract_value(final_text, "project", expected) + answer_note = ( + f"final text reports project with more bugs {expected!r}" + if answer_correct + else f"project values={contract_values(final_text, 'project')!r}; want [{expected!r}]" + ) + return answer_with_provenance(answer_correct, answer_note, run) + + +R6_TASK: dict[str, Any] = { + "id": "R6", + "tags": {"read"}, + "prompt": ( + "Across the eval projects created for this run (main project {project} and its " + "sibling 'B' project), which project has more open Bug-typed work items? " + "Return exactly one line: 'project: '." + ), + "needs": {"items", "bug_type", "second_project"}, + "verify": verify_r6, +} + + +async def verify_r7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """R7: report the immutable state/group baseline with target-response evidence.""" + baseline = [str(value) for value in (ctx.get("r7_state_pairs") or [])] + if not baseline: + return answer_with_provenance(False, "R7 fixture error: seeded state baseline is empty", run) + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + page = plane.states.list(workspace_slug=workspace_slug, project_id=project_id) + states = list(page.results or []) + live = state_name_group_pairs(states) + if Counter(live) != Counter(baseline): + return answer_with_provenance( + False, + f"state oracle was mutated after seeding: live={live!r}; baseline={baseline!r}", + run, + ) + + final_text = get_final_text(run) + if not reports_contract_values(final_text, "state", baseline): + reported = contract_values(final_text, "state") + return answer_with_provenance( + False, + f"state values={reported!r}; want seeded state/group pairs {baseline!r}", + run, + ) + return answer_with_provenance(True, f"final text reports all {len(baseline)} seeded state/group pairs", run) + + +R7_TASK: dict[str, Any] = { + "id": "R7", + "tags": {"read", "extra"}, + "prompt": ( + "List every workflow state in project {project} and its group. Return exactly " + "one line per state as 'state: | group: '." + ), + # Extra: exercises project state listing. + "needs": set(), + "verify": verify_r7, +} + + +READ_TASKS: list[dict[str, Any]] = [R1_TASK, R2_TASK, R3_TASK, R4_TASK, R5_TASK, R6_TASK, R7_TASK] + + +__all__ = ["READ_TASKS", "verify_r1", "verify_r2", "verify_r3", "verify_r4", "verify_r5", "verify_r6", "verify_r7"] diff --git a/evals/tasks/schema.py b/evals/tasks/schema.py new file mode 100644 index 0000000..f5f9fa5 --- /dev/null +++ b/evals/tasks/schema.py @@ -0,0 +1,373 @@ +"""Schema-task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.enums import PropertyType + +from evals.core.errors import TaskSkipped +from evals.core.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE, W8_TITLE +from evals.tasks.lookups import as_id, find_item_by_name +from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error + + +async def verify_s1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S1: Bug type has an OPTION property 'Severity' with Critical/Major/Minor. + + Only type-scoped property listing is accepted (no project/workspace fallbacks that + would pass an unattached Severity). Unexpected API errors propagate as harness errors. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + bug_type_id = ( + (ctx.get("bug_type") or {}).get("id") if isinstance(ctx.get("bug_type"), dict) else ctx.get("bug_type") + ) + if not bug_type_id: + raise TaskSkipped("bug_type not seeded") + + try: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(bug_type_id), + ) + or [] + ) + except HttpError as exc: + if is_verifier_not_found(exc): + # A type-scoped 404 is authoritative absence, so it is evidence of a failed end state. + return False, "Severity property not found on Bug type (type-scoped list empty/404)" + raise_verifier_read_error("S1", "listing Bug type properties", exc) + + severity = None + for p in props: + display = (getattr(p, "display_name", None) or getattr(p, "name", None) or "").strip() + if display.lower() == "severity": + # Prefer an explicit type link when the API exposes it. + issue_type = getattr(p, "issue_type", None) + if issue_type is not None and str(issue_type) not in ("", str(bug_type_id)): + continue + severity = p + break + if severity is None: + return False, "Severity property not found on Bug type" + + prop_type = getattr(severity, "property_type", None) + prop_type_val = prop_type.value if isinstance(prop_type, PropertyType) else prop_type + if str(prop_type_val or "").upper() != PropertyType.OPTION.value: + return False, f"Severity property_type={prop_type_val!r} (want OPTION)" + + option_names = { + (getattr(o, "name", None) or (o.get("name") if isinstance(o, dict) else "") or "").strip() + for o in (getattr(severity, "options", None) or []) + } + if not option_names: + try: + opts = plane.work_item_properties.options.list( + workspace_slug=workspace_slug, + project_id=project_id, + property_id=severity.id, + ) + option_names = {(getattr(o, "name", None) or "").strip() for o in (opts or [])} + except HttpError as exc: + if not is_verifier_not_found(exc): + raise_verifier_read_error("S1", "listing Severity options", exc) + # A missing options collection definitively cannot contain the required choices. + option_names = set() + + required = {"critical", "major", "minor"} + have = {n.casefold() for n in option_names if n} + missing = required - have + if missing: + return False, f"Severity options missing {sorted(missing)}; have {sorted(option_names)}" + return True, "Severity OPTION with Critical/Major/Minor present on Bug type" + + +S1_TASK: dict[str, Any] = { + "id": "S1", + "tags": {"setup"}, + "prompt": ( + "In project {project}, add a Severity dropdown property (options: Critical, " + "Major, Minor) to the Bug work item type." + ), + "needs": {"bug_type"}, + "verify": verify_s1, +} + + +async def verify_s2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S2: Fibonacci estimate scale exists and target item estimate_point is 5.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + # Resolve active estimate + points. + try: + est = plane.estimates.retrieve(workspace_slug=workspace_slug, project_id=project_id) + except Exception as exc: + if is_verifier_not_found(exc): + return False, "project estimate not found; requested Fibonacci scale was not created" + raise_verifier_read_error("S2", "retrieving the project estimate", exc) + est_id = getattr(est, "id", None) or as_id(est) + points = plane.estimates.list_points(workspace_slug=workspace_slug, project_id=project_id, estimate_id=est_id) + point_rows = points if isinstance(points, list) else (points.results if hasattr(points, "results") else points) + values = {(getattr(p, "value", None) or "").strip() for p in (point_rows or [])} + fib_like = {"1", "2", "3", "5", "8"} + if not fib_like.issubset(values): + ok = False + notes.append(f"estimate points missing fib subset; have {sorted(values)}") + else: + notes.append("fibonacci points present") + + five = next((p for p in (point_rows or []) if (getattr(p, "value", None) or "").strip() == "5"), None) + item = find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) + if item is None: + ok = False + notes.append(f"target item {W8_TITLE!r} missing") + else: + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + ep = getattr(detail, "estimate_point", None) + ep_id = as_id(ep) if not isinstance(ep, (int, float)) else None + # estimate_point may be expanded object or UUID. + if five is not None and ep_id and str(ep_id) == str(five.id): + notes.append("item estimate_point=5") + elif ep is not None and str(getattr(ep, "value", ep)) in ("5", "5.0"): + notes.append("item estimate value=5") + else: + ok = False + notes.append(f"item estimate_point={ep!r} (want 5)") + return ok, "; ".join(notes) + + +S2_TASK: dict[str, Any] = { + "id": "S2", + "tags": {"setup"}, + "prompt": ( + f"In project {{project}}, add a Fibonacci estimate scale (points 1,2,3,5,8) " + f"and set the work item '{W8_TITLE}' to 5 points." + ), + "needs": {"items"}, + "verify": verify_s2, +} + + +async def verify_s3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S3: Incident type exists with a required TEXT property. + + Workspace-owned types: probe get_features.is_work_item_types_enabled at verify + time (S3 needs is empty, so seed never sets bug_type_workspace_level). + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + # Find Incident type (project list first). + types = list(plane.work_item_types.list(workspace_slug=workspace_slug, project_id=project_id) or []) + incident = next((t for t in types if (t.name or "").strip().casefold() == "incident"), None) + if incident is None: + # Probe workspace feature at verify time — do not rely on seed ctx flags. + workspace_owns = False + try: + features = plane.workspaces.get_features(workspace_slug=workspace_slug) + dump = features.model_dump() if hasattr(features, "model_dump") else {} + workspace_owns = bool(dump.get("is_work_item_types_enabled")) + except Exception as exc: + raise_verifier_read_error("S3", "reading workspace work-item-type ownership", exc) + if workspace_owns: + try: + wtypes = list(plane.workspace_work_item_types.list(workspace_slug=workspace_slug) or []) + incident = next((t for t in wtypes if (t.name or "").strip().casefold() == "incident"), None) + except Exception as exc: + raise_verifier_read_error("S3", "listing workspace work-item types", exc) + if incident is None: + return False, "Incident work item type not found" + + try: + props = list( + plane.work_item_properties.list( + workspace_slug=workspace_slug, + project_id=project_id, + type_id=str(incident.id), + ) + or [] + ) + except HttpError as exc: + if is_verifier_not_found(exc): + # A type-scoped 404 is authoritative absence, not an unavailable read. + return False, "no properties on Incident type" + raise_verifier_read_error("S3", "listing Incident type properties", exc) + + required_text = None + for p in props: + prop_type = getattr(p, "property_type", None) + prop_type_val = str(prop_type.value if isinstance(prop_type, PropertyType) else prop_type or "").upper() + is_required = bool(getattr(p, "is_required", False)) + # TEXT only — no fallback to other required types (OPTION etc.). + if is_required and prop_type_val in (PropertyType.TEXT.value, "TEXT", "STRING"): + required_text = p + break + if required_text is None: + return False, f"no required TEXT property on Incident; props={len(props)}" + display = getattr(required_text, "display_name", None) or getattr(required_text, "name", None) + return True, f"Incident type + required TEXT property {display!r}" + + +S3_TASK: dict[str, Any] = { + "id": "S3", + "tags": {"setup"}, + "prompt": ( + "In project {project}, create a work item type named 'Incident' and add a " + "required text property (e.g. 'Impact summary') on it." + ), + "needs": set(), + "verify": verify_s3, +} + + +async def verify_s4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S4: billing intake accepted (status=1), spam declined (status=-1).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + intake = ctx.get("intake") or {} + billing = intake.get("billing") or {} + spam = intake.get("spam") or {} + notes: list[str] = [] + ok = True + + def _status_of(issue_id: str | None, title: str) -> int | None: + if not issue_id: + return None + try: + row = plane.intake.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=issue_id) + return getattr(row, "status", None) + except Exception: + # Retrieve is optional because the independently authoritative list can resolve the same row. + try: + rows = plane.intake.list(workspace_slug=workspace_slug, project_id=project_id) + results = rows.results if hasattr(rows, "results") else rows + for r in results or []: + detail = getattr(r, "issue_detail", None) + name = getattr(detail, "name", None) if detail is not None else None + if name and name.strip() == title: + return getattr(r, "status", None) + except Exception as exc: + raise_verifier_read_error("S4", f"listing intake while resolving {title!r}", exc) + return None + + b_status = _status_of(billing.get("issue_id"), INTAKE_BILLING_TITLE) + s_status = _status_of(spam.get("issue_id"), INTAKE_SPAM_TITLE) + # accept=1, decline=-1 per IntakeWorkItemStatusEnum + if b_status != 1: + ok = False + notes.append(f"billing status={b_status!r} (want 1/accepted)") + else: + notes.append("billing accepted") + if s_status != -1: + ok = False + notes.append(f"spam status={s_status!r} (want -1/declined)") + else: + notes.append("spam declined") + return ok, "; ".join(notes) + + +S4_TASK: dict[str, Any] = { + "id": "S4", + "tags": {"setup"}, + "prompt": ( + f"In project {{project}}, triage intake: accept the billing request " + f"'{INTAKE_BILLING_TITLE}' and reject/decline the spam item " + f"'{INTAKE_SPAM_TITLE}'." + ), + "needs": {"intake"}, + "verify": verify_s4, +} + + +async def verify_s5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """S5: project cycles + worklogs AND workspace customers enabled. + + Gates (plane-ee): + - project.cycle_view — cycles create/list + - project.is_time_tracking_enabled — worklogs + - WorkspaceFeature.is_customer_enabled (API field ``customers``) — customer create 403 + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + notes: list[str] = [] + ok = True + + proj = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) + cycle_view = bool(getattr(proj, "cycle_view", None)) + time_tracking = bool(getattr(proj, "is_time_tracking_enabled", None)) + if not cycle_view: + ok = False + notes.append(f"cycle_view={getattr(proj, 'cycle_view', None)!r} (want True)") + else: + notes.append("cycle_view=True") + if not time_tracking: + ok = False + notes.append(f"is_time_tracking_enabled={getattr(proj, 'is_time_tracking_enabled', None)!r} (want True)") + else: + notes.append("is_time_tracking_enabled=True") + + try: + feat = plane.projects.get_features(workspace_slug=workspace_slug, project_id=project_id) + dump = feat.model_dump() if hasattr(feat, "model_dump") else (feat if isinstance(feat, dict) else {}) + cycles_flag = dump.get("cycles") if isinstance(dump, dict) else getattr(feat, "cycles", None) + if not cycles_flag: + ok = False + notes.append(f"features.cycles={cycles_flag!r} (want True)") + else: + notes.append("features.cycles=True") + except Exception as exc: + raise_verifier_read_error("S5", "reading project feature flags", exc) + + # Workspace customers toggle (is_customer_enabled behind API field ``customers``). + try: + ws_feat = plane.workspaces.get_features(workspace_slug=workspace_slug) + ws_dump = ( + ws_feat.model_dump() if hasattr(ws_feat, "model_dump") else (ws_feat if isinstance(ws_feat, dict) else {}) + ) + customers_on = None + if isinstance(ws_dump, dict): + customers_on = ws_dump.get("customers") + if customers_on is None: + customers_on = ws_dump.get("is_customer_enabled") + if customers_on is None: + customers_on = getattr(ws_feat, "customers", None) + if customers_on is None: + customers_on = getattr(ws_feat, "is_customer_enabled", None) + if not customers_on: + ok = False + notes.append(f"workspace.customers={customers_on!r} (want True)") + else: + notes.append("workspace.customers=True") + except Exception as exc: + raise_verifier_read_error("S5", "reading workspace customer feature flags", exc) + + return ok, "; ".join(notes) + + +S5_TASK: dict[str, Any] = { + "id": "S5", + "tags": {"setup"}, + "prompt": ( + "Enable cycles and time tracking (worklogs) for project {project}, " + "and enable the customers feature for the workspace." + ), + # Minimal legacy path (2 calls): + # 1. update_project(cycle_view=True, is_time_tracking_enabled=True) + # 2. update_workspace_features(customers=True) + # (features PATCH can set cycles→cycle_view but cannot set worklogs.) + # Seed leaves project cycles+worklogs and workspace customers off. + "needs": {"leave_cycles_worklogs_off"}, + "verify": verify_s5, +} + + +SCHEMA_TASKS: list[dict[str, Any]] = [S1_TASK, S2_TASK, S3_TASK, S4_TASK, S5_TASK] + + +__all__ = ["SCHEMA_TASKS", "verify_s1", "verify_s2", "verify_s3", "verify_s4", "verify_s5"] diff --git a/evals/tasks/skip.py b/evals/tasks/skip.py new file mode 100644 index 0000000..19b9abd --- /dev/null +++ b/evals/tasks/skip.py @@ -0,0 +1,11 @@ +"""Retired import path for :class:`evals.core.errors.TaskSkipped`. + +Kept because it shipped and ``tests/evals/test_import_compat.py`` pins it. Nothing in the +package imports it: the canonical home is ``evals.core.errors``, which depends on nothing. Every +source module routed through here for a while, which left the neutral module unused and made +a compat shim look like the real one. +""" + +from evals.core.errors import TaskSkipped as TaskSkipped + +__all__ = ["TaskSkipped"] diff --git a/evals/tasks/verification.py b/evals/tasks/verification.py new file mode 100644 index 0000000..dd8bfa4 --- /dev/null +++ b/evals/tasks/verification.py @@ -0,0 +1,24 @@ +"""Shared verifier failure semantics.""" + +from __future__ import annotations + +from typing import NoReturn + +from plane.errors.errors import HttpError + + +class VerifierReadError(RuntimeError): + """An infrastructure failure while a verifier was reading authoritative state.""" + + +def is_verifier_not_found(exc: BaseException) -> bool: + """Return whether a verifier read got an authoritative HTTP 404 response.""" + return isinstance(exc, HttpError) and exc.status_code == 404 + + +def raise_verifier_read_error(task_id: str, reading: str, exc: BaseException) -> NoReturn: + """Raise a diagnosable infrastructure error for a required verifier API read.""" + raise VerifierReadError(f"{task_id} verifier read failed while {reading}: {type(exc).__name__}: {exc}") from exc + + +__all__ = ["VerifierReadError", "is_verifier_not_found", "raise_verifier_read_error"] diff --git a/evals/tasks/write.py b/evals/tasks/write.py new file mode 100644 index 0000000..6a93232 --- /dev/null +++ b/evals/tasks/write.py @@ -0,0 +1,661 @@ +"""Write-task definitions and their verifiers.""" + +from __future__ import annotations + +from typing import Any + +from plane.errors.errors import HttpError +from plane.models.query_params import PaginatedQueryParams, RetrieveQueryParams, WorkItemQueryParams + +from evals.core.fixtures import ( + CYCLE_CURRENT, + CYCLE_PAST, + MODULE_COMPLETED_TITLES, + MODULE_NAME, + W2_TITLE, + W3_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, + W7_URL, + W8_TITLE, +) +from evals.tasks.answers import normalize_rich_text +from evals.tasks.lookups import ( + collect_paginated, + find_item_by_name, + find_items_by_name, + ids, + state_name, +) +from evals.tasks.verification import is_verifier_not_found, raise_verifier_read_error + +W3_COMMENT_TEXT = "Reviewed contrast tokens — needs design pass" +W10_PAGE_NAME = "Eval Runbook" +W10_PAGE_BODY = "Rollback steps for eval harness" + + +async def verify_w1(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W1: assert end-state via Plane API (title, priority, assignee, auth label).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + title = "Login page 500s on empty password" + matches = find_items_by_name(plane, workspace_slug, project_id, title) + if not matches: + return False, f"work item {title!r} not found" + item = matches[0] # newest first + notes: list[str] = [] + if len(matches) > 1: + notes.append(f"warning: {len(matches)} items with title (verifying newest)") + + detail = plane.work_items.retrieve( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + params=RetrieveQueryParams(expand="assignees,labels"), + ) + + ok = True + + priority = (detail.priority or "").lower() if detail.priority else "" + if priority != "urgent": + ok = False + notes.append(f"priority={priority!r} (want urgent)") + else: + notes.append("priority=urgent") + + me = plane.users.get_me() + me_id = str(me.id) + assignee_ids = ids(detail.assignees) + if me_id not in assignee_ids: + ok = False + notes.append(f"assignees={sorted(assignee_ids)} missing me={me_id}") + else: + notes.append("assigned to me") + + auth_label_id = (ctx.get("labels") or {}).get("auth") + label_ids = ids(detail.labels) + if not auth_label_id: + ok = False + notes.append("auth label id missing from seed ctx") + elif str(auth_label_id) not in label_ids: + ok = False + notes.append(f"labels={sorted(label_ids)} missing auth={auth_label_id}") + else: + notes.append("auth label attached") + + return ok, "; ".join(notes) + + +W1_TASK: dict[str, Any] = { + "id": "W1", + "tags": {"write"}, + "prompt": ( + "Create a work item in project {project}: title 'Login page 500s on empty " + "password', priority urgent, assign it to me, and add the 'auth' label." + ), + "needs": {"labels"}, + "verify": verify_w1, +} + + +async def verify_w2(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W2: target item is in the exact state named by the prompt: Done.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W2_TITLE) + if item is None: + return False, f"item {W2_TITLE!r} not found" + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + name = (state_name(plane, workspace_slug, project_id, detail.state) or "").strip() + if name == "Done": + return True, f"state exactly matches {name!r}" + return False, f"state={name!r} (want exact 'Done')" + + +W2_TASK: dict[str, Any] = { + "id": "W2", + "tags": {"write"}, + "prompt": (f"In project {{project}}, move the work item titled '{W2_TITLE}' to the Done state."), + "needs": {"items"}, + "verify": verify_w2, +} + + +async def verify_w3(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W3: target item has a comment whose normalized text exactly matches the ask.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W3_TITLE) + if item is None: + return False, f"item {W3_TITLE!r} not found" + resp = plane.work_items.comments.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + results = list(resp.results if hasattr(resp, "results") else resp or []) + if not results: + return False, "no comments on target item" + expected = normalize_rich_text(W3_COMMENT_TEXT) + for c in results: + actual = normalize_rich_text(c) + if actual == expected: + return True, f"comment text exactly matches {expected!r}" + actual_texts = [normalize_rich_text(comment) for comment in results] + return False, f"no exact normalized comment {expected!r}; have {actual_texts!r}" + + +W3_TASK: dict[str, Any] = { + "id": "W3", + "tags": {"write"}, + "prompt": ( + f"In project {{project}}, add a comment on the work item titled '{W3_TITLE}' saying '{W3_COMMENT_TEXT}'." + ), + "needs": {"items"}, + "verify": verify_w3, +} + + +async def verify_w4(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W4: the seeded triage label id is now named needs-triage. + + Authoritative path: retrieve ctx['labels']['triage'] by id. Name-scan is + only a fallback when the seed id is missing from ctx. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + triage_id = (ctx.get("labels") or {}).get("triage") + if triage_id: + try: + lb = plane.labels.retrieve(workspace_slug=workspace_slug, project_id=project_id, label_id=triage_id) + name = (lb.name or "").strip() + if name == "needs-triage": + return True, f"label id {triage_id} now named {lb.name!r}" + return False, f"label id {triage_id} named {lb.name!r} (want exact 'needs-triage')" + except HttpError as exc: + if not is_verifier_not_found(exc): + raise_verifier_read_error("W4", f"retrieving seeded triage label {triage_id}", exc) + # The seed ID returning 404 proves the requested rename end state does not exist. + return False, f"seeded triage label id {triage_id} not found (deleted?)" + + # Fallback only when seed id is absent from ctx. + page = plane.labels.list(workspace_slug=workspace_slug, project_id=project_id) + names = {(lb.name or "").strip() for lb in (page.results or [])} + if "needs-triage" in names: + if "triage" in names: + return False, "both triage and needs-triage still present" + return True, "label renamed to needs-triage (no seed id; name-scan fallback)" + return False, f"exact needs-triage label not found; labels={sorted(names)}" + + +W4_TASK: dict[str, Any] = { + "id": "W4", + "tags": {"write"}, + "prompt": ("In project {project}, rename the label 'triage' to 'needs-triage'."), + "needs": {"labels"}, + "verify": verify_w4, +} + + +async def verify_w5(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W5: all seeded module completed items are archived (not merely deleted).""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + ids = [str(i) for i in (ctx.get("module_completed_ids") or [])] + if not ids: + # Fall back to titles. + for title in MODULE_COMPLETED_TITLES: + item = find_item_by_name(plane, workspace_slug, project_id, title) + if item: + ids.append(str(item.id)) + if not ids: + return False, "no module completed item ids" + + not_archived: list[str] = [] + need_archive_list: list[str] = [] # 404 on retrieve — must appear in archived list + for wid in ids: + try: + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=wid) + except HttpError as exc: + if is_verifier_not_found(exc): + # Retrieve 404 is ambiguous by design; the archived list is an authoritative fallback. + need_archive_list.append(str(wid)) + continue + raise_verifier_read_error("W5", f"retrieving module work item {wid}", exc) + archived_at = getattr(detail, "archived_at", None) + if not archived_at: + not_archived.append(str(wid)) + + arch_ids: set[str] = set() + if need_archive_list or not_archived: + try: + archived_rows = collect_paginated( + lambda cursor: plane.work_items.list_archived( + workspace_slug=workspace_slug, + project_id=project_id, + params=( + WorkItemQueryParams(cursor=cursor, per_page=100) + if cursor + else WorkItemQueryParams(per_page=100) + ), + ) + ) + arch_ids = {str(i.id) for i in archived_rows} + except Exception as exc: + if need_archive_list: + raise_verifier_read_error("W5", "listing archived items to resolve retrieve 404s", exc) + # Optional cross-check only: successful retrieves already prove these rows are unarchived. + pass + + # 404s only count as archived if present on the archived list (deletes fail). + for wid in need_archive_list: + if wid not in arch_ids: + not_archived.append(wid) + not_archived = [i for i in not_archived if i not in arch_ids] + + if not_archived: + return False, f"{len(not_archived)} module items not archived: {not_archived}" + return True, f"{len(ids)} module completed items archived" + + +W5_TASK: dict[str, Any] = { + "id": "W5", + "tags": {"write"}, + "prompt": (f"In project {{project}}, archive all completed work items in the module '{MODULE_NAME}'."), + "needs": {"module"}, + "verify": verify_w5, +} + + +async def verify_w6(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W6: Sprint 12 closed by a real completion signal + unfinished items on Sprint 13. + + complete_cycle (SDK) sets end_date to *today* — a no-op agent leaves the seeded + past end_date unchanged, so requiring end_date==today (or archived_at set) is + non-vacuous. progress_snapshot non-null is also accepted when the API flips it. + """ + from datetime import date as _date + + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + past_id = ctx.get("cycle_past_id") or (ctx.get("cycles") or {}).get(CYCLE_PAST) + cur_id = ctx.get("cycle_current_id") or (ctx.get("cycles") or {}).get(CYCLE_CURRENT) + if not past_id: + raise RuntimeError(f"W6 fixture error: {CYCLE_PAST} id missing from seed") + if not cur_id: + raise RuntimeError(f"W6 fixture error: {CYCLE_CURRENT} id missing from seed") + unfinished = list(ctx.get("w6_unfinished_titles") or []) + if not unfinished: + raise RuntimeError(f"W6 fixture error: expected unfinished items for {CYCLE_CURRENT} are empty") + notes: list[str] = [] + ok = True + past = plane.cycles.retrieve(workspace_slug=workspace_slug, project_id=project_id, cycle_id=past_id) + end = getattr(past, "end_date", None) + archived_at = getattr(past, "archived_at", None) + snapshot = getattr(past, "progress_snapshot", None) + today = _date.today().isoformat() + seed_end = ctx.get("cycle_past_seed_end_date") + + # Real close signals (any one suffices): + # 1) complete_cycle → end_date becomes today + # 2) manage_cycle_archive → archived_at set + # 3) progress_snapshot populated (Plane completion snapshot) + # end_date comes back as a timestamp ('2026-08-12T00:00:00Z'), so compare the + # date part — a whole-string match against today's date can never be true. + end_day = str(end or "")[:10] + closed = False + if archived_at: + closed = True + notes.append(f"Sprint 12 archived_at={archived_at}") + elif end_day == today: + closed = True + notes.append(f"Sprint 12 end_date={end} (complete_cycle today)") + elif snapshot not in (None, {}, []): + closed = True + notes.append("Sprint 12 progress_snapshot set") + if not closed: + ok = False + notes.append( + f"Sprint 12 not closed: end_date={end!r} seed_end={seed_end!r} " + f"archived_at={archived_at!r} snapshot={snapshot!r} " + f"(want end_date={today!r} or archived_at or progress_snapshot)" + ) + + try: + on13 = plane.cycles.list_work_items( + workspace_slug=workspace_slug, + project_id=project_id, + cycle_id=cur_id, + params=WorkItemQueryParams(per_page=100), + ) + names = {(i.name or "").strip() for i in (on13.results or [])} + except Exception as exc: + if is_verifier_not_found(exc): + return False, f"{CYCLE_CURRENT} not found while checking unfinished-item rollover" + raise_verifier_read_error("W6", f"listing {CYCLE_CURRENT} work items", exc) + missing = [t for t in unfinished if t not in names] + if missing: + ok = False + notes.append(f"unfinished not on Sprint 13: {missing}") + else: + notes.append(f"{len(unfinished)} unfinished on Sprint 13") + return ok, "; ".join(notes) + + +W6_TASK: dict[str, Any] = { + "id": "W6", + "tags": {"write"}, + "prompt": ( + f"In project {{project}}, '{CYCLE_PAST}' is wrapping up. Close it and make sure " + f"its unfinished work items end up on '{CYCLE_CURRENT}'." + ), + # cycles_open_past: Sprint 12 must still be open, or "close it" is impossible — + # Plane rejects every edit to an ended cycle. See _seed_cycles. + "needs": {"items", "cycles", "cycles_open_past"}, + "verify": verify_w6, +} + + +async def verify_w7(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W7: source blocks target (dependency) AND reference URL link exists on source. + + Only dump['blocking'] ids count — a reverse blocked_by match must not pass. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + src = find_item_by_name(plane, workspace_slug, project_id, W7_SOURCE_TITLE) + tgt = find_item_by_name(plane, workspace_slug, project_id, W7_TARGET_TITLE) + if not src or not tgt: + return False, "W7 source/target items not found" + notes: list[str] = [] + ok = True + + # Dependencies — require tgt in blocking specifically. + try: + deps = plane.work_items.dependencies.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=src.id, + ) + dump = deps.model_dump() if hasattr(deps, "model_dump") else (deps if isinstance(deps, dict) else {}) + blocking = dump.get("blocking") or [] + if isinstance(blocking, dict): + blocking = blocking.get("results") or list(blocking.values()) + blocking_ids = ids(blocking) + # blocking may also be plain UUID strings + for b in blocking if isinstance(blocking, list) else []: + if isinstance(b, str): + blocking_ids.add(b) + if str(tgt.id) not in blocking_ids: + ok = False + blob_hit = str(tgt.id) in str(dump) + note = f"no blocking relation from source to {tgt.id}; blocking_ids={sorted(blocking_ids)}" + if blob_hit: + note += " (target id appears elsewhere in dump — wrong direction)" + notes.append(note) + else: + notes.append("blocking relation present") + except Exception as exc: + raise_verifier_read_error("W7", f"listing dependencies for source item {src.id}", exc) + + # Links + try: + links = plane.work_items.links.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=src.id, + ) + rows = links.results if hasattr(links, "results") else links + urls = {(getattr(ln, "url", None) or "").strip() for ln in (rows or [])} + if W7_URL not in urls: + ok = False + notes.append(f"link {W7_URL!r} missing; have {sorted(urls)}") + else: + notes.append("reference URL present") + except Exception as exc: + raise_verifier_read_error("W7", f"listing links for source item {src.id}", exc) + + return ok, "; ".join(notes) + + +W7_TASK: dict[str, Any] = { + "id": "W7", + "tags": {"write"}, + "prompt": ( + f"In project {{project}}, mark the work item '{W7_SOURCE_TITLE}' as blocking " + f"'{W7_TARGET_TITLE}', and add the reference URL {W7_URL} on the blocking item." + ), + "needs": {"items"}, + "verify": verify_w7, +} + + +async def verify_w8(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W8: work log of exactly 120 minutes exists on the target item.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W8_TITLE) + if item is None: + return False, f"item {W8_TITLE!r} not found" + logs = plane.work_items.work_logs.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 120 in durations: + return True, "work log duration=120 present" + return False, f"no 120-minute work log; durations={durations}" + + +W8_TASK: dict[str, Any] = { + "id": "W8", + "tags": {"write"}, + "prompt": (f"In project {{project}}, log 2 hours of work on the item titled '{W8_TITLE}'."), + "needs": {"items"}, + "verify": verify_w8, +} + + +W11_TITLE = W8_TITLE + + +def _time_tracking_enabled(plane: Any, workspace_slug: str, project_id: str) -> bool | None: + """Whether the project has time tracking on, or None when the read itself failed.""" + try: + project = plane.projects.retrieve(workspace_slug=workspace_slug, project_id=project_id) + except Exception: + # Optional diagnosis only: the already-successful worklog read determines failure. + return None + value = getattr(project, "is_time_tracking_enabled", None) + return None if value is None else bool(value) + + +async def verify_w11(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W11: the work log lands even though the project shipped with time tracking off. + + W8's end state reached from an obstacle — the worklog endpoint 404s until the feature + is on, which the prompt authorises. Failure notes separate the routes to failure, + since "no work log" alone does not say whether the agent gave up, half-finished, or + claimed a success it never earned. + """ + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + item = find_item_by_name(plane, workspace_slug, project_id, W11_TITLE) + if item is None: + return False, f"item {W11_TITLE!r} not found" + + durations: list[int] = [] + try: + logs = plane.work_items.work_logs.list( + workspace_slug=workspace_slug, + project_id=project_id, + work_item_id=item.id, + ) + except Exception as exc: + # The list endpoint is gated by the same toggle, so this is the diagnosis itself. + if not is_verifier_not_found(exc): + raise_verifier_read_error("W11", f"listing work logs for item {item.id}", exc) + # This endpoint's documented gate is a 404, which proves time tracking remains disabled. + return False, f"work log list still refused, so time tracking was never enabled ({exc})" + rows = logs if isinstance(logs, list) else (logs.results if hasattr(logs, "results") else logs) + durations = [int(getattr(w, "duration", 0) or 0) for w in (rows or [])] + if 120 in durations: + return True, "work log duration=120 present; agent cleared the disabled feature" + + enabled = _time_tracking_enabled(plane, workspace_slug, project_id) + if enabled: + return False, f"time tracking enabled but no 120-minute log; durations={durations}" + + final_text = str(run.get("final_text") or "").lower() + mentioned = any(phrase in final_text for phrase in ("time tracking", "not enabled", "disabled")) + if mentioned: + return False, "reported the feature was off rather than enabling it, which the prompt allowed" + return False, "no work log, and the answer never mentions the disabled feature" + + +W11_TASK: dict[str, Any] = { + "id": "W11", + "tags": {"write", "gate"}, + "prompt": ( + f"In project {{project}}, log 2 hours of work on the item titled '{W11_TITLE}'. " + "Time tracking may be switched off for this project; you have permission to turn " + "it on if you need to." + ), + # enable time tracking → find the item → create the work log. + "needs": {"items", "leave_worklogs_off"}, + "verify": verify_w11, +} + + +async def verify_w9(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W9 (extra): bulk priority change — the three non-R1 urgent titles are now high.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + # All urgent fixtures except we ask agent to set medium-priority batch targets. + # Prompt targets the three titles starting with Session/Inventory/Checkout (non-R1 urgent). + targets = [ + "Checkout times out on 3DS challenge", + "Session cookie not rotated after login", + "Inventory count goes negative under load", + ] + wrong: list[str] = [] + for title in targets: + item = find_item_by_name(plane, workspace_slug, project_id, title) + if not item: + wrong.append(f"{title}: missing") + continue + detail = plane.work_items.retrieve(workspace_slug=workspace_slug, project_id=project_id, work_item_id=item.id) + pr = (detail.priority or "").lower() + if pr != "high": + wrong.append(f"{title}: priority={pr!r}") + if wrong: + return False, "; ".join(wrong) + return True, "3 items priority=high" + + +W9_TASK: dict[str, Any] = { + "id": "W9", + "tags": {"write", "extra"}, + "prompt": ( + "In project {project}, set priority to high on these three work items: " + "'Checkout times out on 3DS challenge', " + "'Session cookie not rotated after login', " + "'Inventory count goes negative under load'." + ), + # Extra: call distributions describe whether agents batch this multi-item mutation. + "needs": {"items"}, + "verify": verify_w9, +} + + +async def verify_w10(plane: Any, ctx: dict[str, Any], run: dict[str, Any]) -> tuple[bool, str]: + """W10 (extra): named project page has the exact normalized requested body.""" + workspace_slug = ctx["workspace_slug"] + project_id = ctx["project_id"] + try: + rows = collect_paginated( + lambda cursor: plane.pages.list_project_pages( + workspace_slug=workspace_slug, + project_id=project_id, + params=( + PaginatedQueryParams(cursor=cursor, per_page=100) if cursor else PaginatedQueryParams(per_page=100) + ), + ) + ) + except Exception as exc: + raise_verifier_read_error("W10", "listing project pages", exc) + candidates = [page for page in rows if (getattr(page, "name", None) or "").strip() == W10_PAGE_NAME] + if not candidates: + names = sorted({(getattr(page, "name", None) or "").strip() for page in rows}) + return False, f"page {W10_PAGE_NAME!r} missing; have {names}" + actual_bodies: list[str] = [] + for page in candidates: + page_id = getattr(page, "id", None) + if not page_id: + actual_bodies.append("") + continue + try: + detail = plane.pages.retrieve_project_page( + workspace_slug=workspace_slug, + project_id=project_id, + page_id=page_id, + ) + except Exception as exc: + if is_verifier_not_found(exc): + actual_bodies.append(f"") + continue + raise_verifier_read_error("W10", f"retrieving project page {page_id}", exc) + actual = normalize_rich_text(detail) + actual_bodies.append(actual) + if actual == W10_PAGE_BODY: + return True, f"page {W10_PAGE_NAME!r} has exact normalized body" + return False, f"page {W10_PAGE_NAME!r} body mismatch: have {actual_bodies!r}; want {W10_PAGE_BODY!r}" + + +W10_TASK: dict[str, Any] = { + "id": "W10", + "tags": {"write", "extra"}, + "prompt": ( + f"In project {{project}}, create a project page named '{W10_PAGE_NAME}' with body text '{W10_PAGE_BODY}'." + ), + # Extra: exercises pages family (create_page / get_page). + "needs": set(), + "verify": verify_w10, +} + + +WRITE_TASKS: list[dict[str, Any]] = [ + W1_TASK, + W2_TASK, + W3_TASK, + W4_TASK, + W5_TASK, + W6_TASK, + W7_TASK, + W8_TASK, + W9_TASK, + W10_TASK, + W11_TASK, +] + + +__all__ = [ + "W3_COMMENT_TEXT", + "W10_PAGE_BODY", + "W10_PAGE_NAME", + "W11_TITLE", + "WRITE_TASKS", + "verify_w1", + "verify_w2", + "verify_w3", + "verify_w4", + "verify_w5", + "verify_w6", + "verify_w7", + "verify_w8", + "verify_w9", + "verify_w10", + "verify_w11", +] diff --git a/pyproject.toml b/pyproject.toml index e5c59f0..5fe9068 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,17 @@ dev = [ "pytest>=7.0.0", "ruff>=0.1.0", ] +# The default API eval provider uses the stable Anthropic Messages client. +evals = [ + "anthropic>=0.121.0", +] +# The OpenAI API provider is a second vendor's client, so it stays opt-in rather than being +# imposed on an anthropic-only install. Declared all the same: without it, `--provider openai` +# is a provider the harness advertises in KNOWN_API_PROVIDERS and cannot run, and a result +# anyone else has to reproduce should not depend on remembering an undeclared package. +evals-openai = [ + "openai>=1.0.0", +] [project.scripts] plane-mcp-server = "plane_mcp.__main__:main" diff --git a/tests/evals/__init__.py b/tests/evals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/conftest.py b/tests/evals/conftest.py new file mode 100644 index 0000000..9cf0f3f --- /dev/null +++ b/tests/evals/conftest.py @@ -0,0 +1,35 @@ +"""Shared fixtures and helpers for eval harness tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + + +def case_params(*cases): + """Build readable pytest cases from consolidated case helpers.""" + return [pytest.param(case, id=case.__name__.removeprefix("_").replace("_", "-")) for case in cases] + + +@pytest.fixture(autouse=True) +def _eval_creds(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "test-key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + +def _data_rows(path: Path) -> list[dict]: + """Parse JSONL skipping meta / non-task lines.""" + out: list[dict] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if row.get("row_type") == "meta" or row.get("task_id") is None: + continue + out.append(row) + return out diff --git a/tests/evals/drivers/__init__.py b/tests/evals/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/drivers/test_api_driver.py b/tests/evals/drivers/test_api_driver.py new file mode 100644 index 0000000..0fa7890 --- /dev/null +++ b/tests/evals/drivers/test_api_driver.py @@ -0,0 +1,929 @@ +"""Offline eval tests for api driver.""" + +from __future__ import annotations + +import copy +from collections import deque +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import Any + +import pytest + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.token_counting import estimate_result_tokens +from evals.core.tool_manifest import tool_manifest_fingerprint +from evals.drivers.api import ( + KNOWN_API_PROVIDERS, + AnthropicBackend, + OpenAIBackend, + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, + UnmappedModelTierError, + Usage, + register_backend, + resolve_backend_model, + unregister_backend, +) +from evals.drivers.api.driver import ApiDriver +from tests.evals.conftest import case_params + + +class FakeBackend: + provider = "fake" + model = "fake-requested" + actual_model = "fake-actual" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = deque(turns) + self.started: tuple[str | None, str, list[ToolSpec]] | None = None + self.added_results: list[list[ToolResult]] = [] + self.num_turns = 0 + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.started = (system, prompt, tools) + + def next_turn(self) -> Turn: + self.num_turns += 1 + if not self.turns: + raise AssertionError("driver requested an unexpected backend turn") + return self.turns.popleft() + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.added_results.append(results) + + +class FakeMcpSession: + def __init__(self, results: list[Any] | None = None, tool_pages: list[Any] | None = None) -> None: + self.results = deque(results or []) + self.tool_pages = deque(tool_pages or []) + self.initialized = False + self.called: list[tuple[str, dict[str, Any]]] = [] + self.list_cursors: list[str | None] = [] + + async def initialize(self) -> None: + self.initialized = True + + async def list_tools(self, cursor: str | None = None) -> Any: + self.list_cursors.append(cursor) + if self.tool_pages: + return self.tool_pages.popleft() + return SimpleNamespace( + tools=[ + SimpleNamespace( + name="lookup", + description="Look something up", + inputSchema={"type": "object", "properties": {"q": {"type": "string"}}}, + ), + SimpleNamespace( + name="write", + description="Write something", + inputSchema={"type": "object"}, + ), + ] + ) + + async def call_tool(self, name: str, arguments: dict[str, Any]) -> Any: + self.called.append((name, arguments)) + if not self.results: + raise AssertionError(f"no fake result left for {name}") + return self.results.popleft() + + +def make_driver(backend: FakeBackend, session: FakeMcpSession) -> ApiDriver: + @asynccontextmanager + async def session_factory(_params): + yield session + + return ApiDriver( + provider="anthropic", + backend_factory=lambda _model, _max_tokens: backend, + mcp_session_factory=session_factory, + ) + + +def run_driver( + driver: ApiDriver, + *, + max_turns: int = 5, + evidence_sentinels=None, + evidence_targets=None, + evidence_aggregates=None, +): + return driver.run_task( + "do it", + {"SAFE": "1"}, + "fake-requested", + max_turns, + system="system", + evidence_sentinels=evidence_sentinels, + evidence_targets=evidence_targets, + evidence_aggregates=evidence_aggregates, + ) + + +class FakeAnthropicMessages: + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = deque(responses) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs): + self.requests.append(copy.deepcopy(kwargs)) + return self.responses.popleft() + + +class FakeOpenAIResponses: + """Stands in for ``client.responses``. Deep-copies each request, because the backend + appends the model's own output items to the same input list it sends.""" + + def __init__(self, responses: list[dict[str, Any]]) -> None: + self.responses = deque(responses) + self.requests: list[dict[str, Any]] = [] + + def create(self, **kwargs): + self.requests.append(copy.deepcopy(kwargs)) + return self.responses.popleft() + + +def openai_client(responses: FakeOpenAIResponses) -> SimpleNamespace: + return SimpleNamespace(responses=responses) + + +def test_registered_third_party_backend_runs_without_driver_changes(): + created: list[FakeBackend] = [] + + class DummyBackend(FakeBackend): + provider = "dummy" + + def __init__(self, model: str, *, max_tokens: int, client: Any | None = None) -> None: + super().__init__( + [ + Turn( + text=f"done in {max_tokens}", + tool_calls=[], + usage=Usage(input_tokens=7, output_tokens=2), + stop_reason=StopReason.END_TURN, + provider_stop_reason="dummy_complete", + ) + ] + ) + self.model = model + self.actual_model = f"{model}-actual" + self.client = client + created.append(self) + + session = FakeMcpSession() + + @asynccontextmanager + async def session_factory(_params): + yield session + + register_backend("dummy", DummyBackend) + try: + assert "dummy" in KNOWN_API_PROVIDERS + with pytest.raises(UnmappedModelTierError, match=r"standard.*explicit model ID"): + resolve_backend_model("dummy", "standard") + assert resolve_backend_model("dummy", "dummy-explicit") == "dummy-explicit" + driver = ApiDriver( + provider="dummy", + client=object(), + mcp_session_factory=session_factory, + max_tokens=99, + ) + run = driver.run_task("do it", {"SAFE": "1"}, "dummy-model", 1) + finally: + unregister_backend("dummy") + + assert created[0].started is not None + assert run.final_text == "done in 99" + assert run.provider == "dummy" + assert run.model == "dummy-model-actual" + assert run.stopped_reason == "end_turn" + assert run.provider_stop_reason == "dummy_complete" + assert run.usage_per_iteration == [Usage(7, 2, 0, 0)] + + +def _api_driver_multi_turn_tool_loop_and_usage_accumulation(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("call-1", "lookup", {"q": "one"})], + usage=Usage(10, 2, 3, 1), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("call-2", "lookup", {"q": "two"})], + usage=Usage(20, 4, 6, 0), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=Usage(30, 6, 9, 0), + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", + ), + ] + ) + session = FakeMcpSession( + [ + {"content": [{"type": "text", "text": "first result"}], "isError": False}, + {"content": [{"type": "text", "text": "second"}], "isError": True}, + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert session.initialized is True + assert session.called == [("lookup", {"q": "one"}), ("lookup", {"q": "two"})] + assert backend.started is not None + assert [tool.name for tool in backend.started[2]] == ["lookup", "write"] + assert [[result.call_id for result in turn] for turn in backend.added_results] == [["call-1"], ["call-2"]] + assert run.final_text == "done" + assert run.stopped_reason == "end_turn" + assert run.cum_input_tokens == 60 + assert run.usage_per_iteration == [Usage(10, 2, 3, 1), Usage(20, 4, 6, 0), Usage(30, 6, 9, 0)] + assert [call["result_chars"] for call in run.calls] == [len("first result"), len("second")] + assert [call["result_tokens"] for call in run.calls] == [ + estimate_result_tokens(len("first result")), + estimate_result_tokens(len("second")), + ] + assert [call["is_error"] for call in run.calls] == [False, True] + assert run.result_tokens_estimated is True + assert run.token_count_failures == 0 + assert run.provider == "fake" + assert run.model == "fake-actual" + assert run.provider_stop_reason == "fake_done" + + +def _api_driver_refusal_records_calls_but_executes_nothing(): + backend = FakeBackend( + [ + Turn( + text="declined", + tool_calls=[ToolCall("write-1", "write", {"value": "x"})], + usage=Usage(1, 1), + stop_reason=StopReason.REFUSAL, + ) + ] + ) + session = FakeMcpSession() + + run = run_driver(make_driver(backend, session)) + + assert [call["tool"] for call in run.calls] == ["write"] + assert session.called == [] + assert backend.added_results == [] + assert run.stopped_reason == "refusal" + assert run.hit_max_turns is False + + +def _api_driver_pairs_results_by_id_not_ordinal(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + # The fake session deliberately returns tagged results in reverse ID order. + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="a", text="A"), + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert [call["result_chars"] for call in run.calls] == [1, 4] + assert run.result_pair_mismatch is False + + +def _api_driver_flags_result_id_mismatch(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"}), ToolCall("b", "lookup", {"q": "b"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + ), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="b", text="BBBB"), + ToolResult(call_id="unknown", text="lost"), + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert run.result_pair_mismatch is True + assert run.trace_integrity is False + assert run.trace_integrity_reason == "result_pair_mismatch" + assert [call["result_chars"] for call in run.calls] == [0, 4] + + +def test_api_driver_aggregates_every_tools_list_page_before_fingerprinting(): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) + tools = [ + {"name": "alpha", "inputSchema": {"type": "object"}}, + {"name": "beta", "inputSchema": {"type": "object"}}, + ] + session = FakeMcpSession( + tool_pages=[ + {"tools": [tools[0]], "nextCursor": "page-2"}, + {"tools": [tools[1]]}, + ] + ) + + run = run_driver(make_driver(backend, session)) + + assert session.list_cursors == [None, "page-2"] + assert run.tool_manifest_fingerprint == tool_manifest_fingerprint(tools) + assert backend.started is not None + assert [tool.name for tool in backend.started[2]] == ["alpha", "beta"] + + +def test_api_driver_invalidates_manifest_after_tools_list_changed(monkeypatch): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) + session = FakeMcpSession() + + @asynccontextmanager + async def fake_stdio_client(_params): + yield object(), object() + + class FakeClientSessionContext: + def __init__(self, _read, _write, *, message_handler): + self.message_handler = message_handler + + async def __aenter__(self): + return session + + async def __aexit__(self, *_args): + await self.message_handler(SimpleNamespace(root=SimpleNamespace(method="notifications/tools/list_changed"))) + return None + + monkeypatch.setattr("evals.drivers.api.driver.stdio_client", fake_stdio_client) + monkeypatch.setattr("evals.drivers.api.driver.ClientSession", FakeClientSessionContext) + driver = ApiDriver( + provider="anthropic", + backend_factory=lambda _model, _max_tokens: backend, + server_command=["fake-server"], + ) + + run = run_driver(driver) + + assert run.tool_manifest_fingerprint is None + + +def _api_driver_iteration_cap_only_flags_mid_tool_loop(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="must not be read", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + ), + ] + ) + session = FakeMcpSession([ToolResult(call_id="a", text="result")]) + + run = run_driver(make_driver(backend, session), max_turns=1) + + assert session.called == [("lookup", {"q": "a"})] + assert len(backend.added_results) == 1 + assert backend.num_turns == 1 + assert run.hit_max_turns is True + assert run.stopped_reason == "tool_use" + + +def _api_driver_clean_end_on_last_iteration_is_not_capped(): + backend = FakeBackend([Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN)]) + + run = run_driver(make_driver(backend, FakeMcpSession()), max_turns=1) + + assert run.hit_max_turns is False + assert run.stopped_reason == "end_turn" + + +def _api_driver_uses_optional_backend_token_counter(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + backend.count_tokens = lambda text: len(text) + 10 + + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="abc")]))) + + assert run.calls[0]["result_tokens"] == 13 + assert run.result_tokens_estimated is False + assert run.token_count_failures == 0 + + +def _api_driver_records_only_matching_evidence_labels(): + sentinel = "hidden-target-fact-2f81a0cd" + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "unrelated"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("b", "lookup", {"q": "target"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="a", text=f"state={sentinel}"), + ToolResult(call_id="b", text="no seeded value in this response"), + ] + ) + + run = run_driver( + make_driver(backend, session), + evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + ) + + # The sentinel only exists inside Plane, so the response carrying it is proof of + # surface use even though its request named an unrelated entity. The response without + # it is not evidence, whatever it was asked about. + assert run.evidence_trace_available is True + assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.calls[1]["observed_sentinels"] == [] + assert "result_text" not in run.calls[1] + + +def test_api_driver_records_only_exact_target_bound_aggregate_evidence(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "count_work_items", {"pql": 'project = "project-other"'})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("b", "count_work_items", {"pql": 'project = "project-1"'})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("c", "count_work_items", {"pql": 'project = "project-1"'})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ToolCall("d", "count_work_items", {"group_by": "project_id"})], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="", + tool_calls=[ + ToolCall( + "e", + "count_work_items", + {"group_by": "project_id", "project_ids": ["project-1", "project-2"]}, + ) + ], + usage=None, + stop_reason=StopReason.TOOL_USE, + ), + Turn(text="done", tool_calls=[], usage=None, stop_reason=StopReason.END_TURN), + ] + ) + session = FakeMcpSession( + [ + ToolResult(call_id="a", text='{"total_count": 4}'), + ToolResult(call_id="b", text='{"total_count": 3}'), + ToolResult(call_id="c", text='{"total_count": 4}'), + ToolResult( + call_id="d", + text=('{"grouped_counts": {"project-1": {"count": 2}, "project-2": {"count": 5}}}'), + ), + ToolResult( + call_id="e", + text=('{"grouped_counts": {"project-1": {"count": 2}, "project-2": {"count": 5}}}'), + ), + ] + ) + + run = run_driver( + make_driver(backend, session), + evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1", "project-2"]}, + evidence_aggregates={ + TARGET_ENTITY_EVIDENCE: [ + {"kind": "total_count", "value": 4}, + {"kind": "grouped_counts", "values": {"project-1": 2, "project-2": 5}}, + ] + }, + max_turns=6, + ) + + assert run.evidence_trace_available is True + assert run.calls[0]["observed_sentinels"] == [] + assert run.calls[1]["observed_sentinels"] == [] + assert run.calls[2]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.calls[3]["observed_sentinels"] == [] + assert run.calls[4]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + + +_API_DRIVER_CASES = case_params( + _api_driver_multi_turn_tool_loop_and_usage_accumulation, + _api_driver_refusal_records_calls_but_executes_nothing, + _api_driver_pairs_results_by_id_not_ordinal, + _api_driver_flags_result_id_mismatch, + _api_driver_iteration_cap_only_flags_mid_tool_loop, + _api_driver_clean_end_on_last_iteration_is_not_capped, + _api_driver_uses_optional_backend_token_counter, + _api_driver_records_only_matching_evidence_labels, +) + + +@pytest.mark.parametrize("case", _API_DRIVER_CASES) +def test_api_driver_behaviours(case): + case() + + +@pytest.mark.parametrize( + ("raw_reason", "expected"), + [ + ("end_turn", StopReason.END_TURN), + ("tool_use", StopReason.TOOL_USE), + ("max_tokens", StopReason.MAX_TOKENS), + ("refusal", StopReason.REFUSAL), + ("pause_turn", StopReason.PAUSE_TURN), + ("model_context_window_exceeded", StopReason.MODEL_CONTEXT_WINDOW_EXCEEDED), + ("future_reason", StopReason.UNKNOWN), + ], +) +def test_anthropic_backend_normalizes_and_preserves_stop_reason(raw_reason, expected): + messages = FakeAnthropicMessages([{"model": "claude", "content": [], "usage": None, "stop_reason": raw_reason}]) + backend = AnthropicBackend("claude", max_tokens=10, client=SimpleNamespace(messages=messages)) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.stop_reason is expected + assert turn.provider_stop_reason == raw_reason + + +def test_anthropic_backend_translates_tools_turns_and_results(): + responses = [ + { + "model": "claude-actual", + "content": [ + {"type": "text", "text": "checking"}, + {"type": "tool_use", "id": "toolu-1", "name": "lookup", "input": {"q": "x"}}, + ], + "usage": { + "input_tokens": 10, + "output_tokens": 2, + "cache_read_input_tokens": 3, + "cache_creation_input_tokens": 4, + }, + "stop_reason": "tool_use", + }, + { + "model": "claude-actual", + "content": [{"type": "text", "text": "done"}], + "usage": {"input_tokens": 20, "output_tokens": 5}, + "stop_reason": "end_turn", + }, + ] + messages = FakeAnthropicMessages(responses) + backend = AnthropicBackend( + "claude-requested", + max_tokens=123, + client=SimpleNamespace(messages=messages), + ) + tool = ToolSpec("lookup", "Look up", {"type": "object", "required": ["q"]}) + + backend.start("system", "prompt", [tool]) + first = backend.next_turn() + backend.add_tool_results([ToolResult("toolu-1", "value", is_error=True)]) + second = backend.next_turn() + + assert messages.requests[0]["system"] == "system" + assert messages.requests[0]["max_tokens"] == 123 + assert messages.requests[0]["tools"] == [ + {"name": "lookup", "description": "Look up", "input_schema": {"type": "object", "required": ["q"]}} + ] + assert first.tool_calls == [ToolCall("toolu-1", "lookup", {"q": "x"})] + assert first.usage == Usage(10, 2, 3, 4) + assert first.stop_reason is StopReason.TOOL_USE + assert first.provider_stop_reason == "tool_use" + replay = messages.requests[1]["messages"] + assert replay[1] == {"role": "assistant", "content": responses[0]["content"]} + assert replay[2] == { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu-1", + "content": "value", + "is_error": True, + } + ], + } + assert second.text == "done" + assert backend.actual_model == "claude-actual" + + +@pytest.mark.parametrize( + ("status", "incomplete_reason", "expected"), + [ + ("completed", None, StopReason.END_TURN), + ("incomplete", "max_output_tokens", StopReason.MAX_TOKENS), + ("incomplete", "content_filter", StopReason.REFUSAL), + ("failed", None, StopReason.UNKNOWN), + ("queued", None, StopReason.UNKNOWN), + ], +) +def test_openai_backend_derives_stop_reason_from_status(status, incomplete_reason, expected): + """Responses has no finish_reason: the outcome is status plus what the turn emitted.""" + responses = FakeOpenAIResponses( + [ + { + "model": "gpt", + "status": status, + "incomplete_details": ({"reason": incomplete_reason} if incomplete_reason else None), + "output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}], + "usage": None, + } + ] + ) + backend = OpenAIBackend("gpt", max_tokens=10, client=openai_client(responses)) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.stop_reason is expected + assert turn.provider_stop_reason == (incomplete_reason or status) + + +def test_anthropic_backend_requests_automatic_prompt_caching(): + """Every turn resends the whole tool surface plus the transcript, so caching is not optional. + + Anthropic caching is opt-in where OpenAI's Responses API caches unasked. Without this field + an Anthropic arm paid full input price on content the measured OpenAI arm read 88% of from + cache, which made a provider cost comparison read as pricing rather than a missing field. + """ + responses = [ + { + "model": "claude-actual", + "stop_reason": "end_turn", + "content": [{"type": "text", "text": "done"}], + "usage": {"input_tokens": 5, "output_tokens": 1}, + } + ] + messages = FakeAnthropicMessages(responses) + backend = AnthropicBackend("claude", max_tokens=10, client=SimpleNamespace(messages=messages)) + tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {}}) + backend.start("system", "prompt", [tool]) + + backend.next_turn() + + request = messages.requests[0] + # Top level, not on a content block: that is the automatic form, where the breakpoint + # advances by itself as the conversation grows. + assert request["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in request["messages"][0] + + +def _openai_backend_translates_tools_calls_and_outputs(): + responses = [ + { + "model": "gpt-actual", + "status": "completed", + "output": [ + {"type": "reasoning", "id": "rs-1", "summary": []}, + { + "type": "function_call", + "id": "fc-1", + "call_id": "call-1", + "name": "lookup", + "arguments": '{"q":"x"}', + }, + ], + "usage": { + "input_tokens": 12, + "output_tokens": 3, + "input_tokens_details": {"cached_tokens": 5}, + }, + }, + { + "model": "gpt-actual", + "status": "completed", + "output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}], + "usage": {"input_tokens": 20, "output_tokens": 4}, + }, + ] + fake = FakeOpenAIResponses(responses) + backend = OpenAIBackend("gpt-requested", max_tokens=321, client=openai_client(fake)) + tool = ToolSpec("lookup", "Look up", {"type": "object", "properties": {"q": {"type": "string"}}}) + + backend.start("system", "prompt", [tool]) + first = backend.next_turn() + backend.add_tool_results([ToolResult("call-1", "value")]) + second = backend.next_turn() + + first_request = fake.requests[0] + # Responses names the cap max_output_tokens and carries the system prompt as instructions. + assert first_request["max_output_tokens"] == 321 + assert first_request["instructions"] == "system" + assert first_request["input"] == [{"role": "user", "content": "prompt"}] + # A function tool is declared flat here; Chat Completions nested it under "function". + assert first_request["tools"] == [ + { + "type": "function", + "name": "lookup", + "description": "Look up", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + } + ] + assert first.tool_calls == [ToolCall("call-1", "lookup", {"q": "x"})] + assert first.stop_reason is StopReason.TOOL_USE + assert first.usage == Usage(12, 3, 5, 0) + + second_input = fake.requests[1]["input"] + # The model's own items are replayed verbatim, reasoning included: dropping a reasoning + # item breaks the chain these models expect on the next request. + assert second_input[1]["type"] == "reasoning" + assert second_input[2]["type"] == "function_call" + assert second_input[2]["call_id"] == "call-1" + # The result is a function_call_output keyed by call_id, not a role-tagged tool message. + assert second_input[3] == {"type": "function_call_output", "call_id": "call-1", "output": "value"} + assert second.text == "done" + assert second.stop_reason is StopReason.END_TURN + assert backend.actual_model == "gpt-actual" + + +def _openai_backend_normalizes_refusal_for_driver_guard(): + """A refusal must outrank the tool calls beside it, or the driver executes a refused write.""" + fake = FakeOpenAIResponses( + [ + { + "model": "gpt", + "status": "completed", + "output": [ + {"type": "message", "content": [{"type": "refusal", "refusal": "declined"}]}, + { + "type": "function_call", + "call_id": "danger", + "name": "write", + "arguments": "{}", + }, + ], + "usage": None, + } + ] + ) + backend = OpenAIBackend("gpt", max_tokens=10, client=openai_client(fake)) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.stop_reason is StopReason.REFUSAL + assert turn.text == "declined" + assert turn.tool_calls == [ToolCall("danger", "write", {})] + + +def _openai_backend_preserves_malformed_arguments(): + """Unparseable arguments are kept as _raw, never silently dropped to an empty call.""" + fake = FakeOpenAIResponses( + [ + { + "model": "gpt", + "status": "completed", + "output": [{"type": "function_call", "call_id": "c1", "name": "lookup", "arguments": "{not json"}], + "usage": None, + } + ] + ) + backend = OpenAIBackend("gpt", max_tokens=10, client=openai_client(fake)) + backend.start(None, "prompt", []) + + turn = backend.next_turn() + + assert turn.tool_calls == [ToolCall("c1", "lookup", {"_raw": "{not json"})] + + +@pytest.mark.parametrize( + "case", + case_params( + _openai_backend_translates_tools_calls_and_outputs, + _openai_backend_normalizes_refusal_for_driver_guard, + _openai_backend_preserves_malformed_arguments, + ), +) +def test_openai_backend_behaviours(case): + case() + + +def _tool_spec_from_mcp_reads_dict_and_object_entries(): + from evals.drivers.api.driver import tool_spec_from_mcp + + as_dict = tool_spec_from_mcp( + {"name": "list_work_items", "description": "List them", "inputSchema": {"type": "object", "x": 1}} + ) + assert (as_dict.name, as_dict.description) == ("list_work_items", "List them") + assert as_dict.input_schema == {"type": "object", "x": 1} + + as_object = tool_spec_from_mcp(SimpleNamespace(name="create_cycle", description="", input_schema=None)) + assert as_object.name == "create_cycle" + # A missing or non-dict schema must still yield a usable object schema. + assert as_object.input_schema == {"type": "object"} + assert tool_spec_from_mcp({"name": "x", "inputSchema": "not-a-schema"}).input_schema == {"type": "object"} + + +def _tool_result_from_mcp_text_only_joins_blocks(): + from evals.drivers.api.driver import tool_result_from_mcp + + result = tool_result_from_mcp( + "call_1", + {"content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + ) + assert (result.call_id, result.text, result.kind, result.is_error) == ( + "call_1", + "first\nsecond", + "text", + False, + ) + + +def _tool_result_from_mcp_serializes_non_text_blocks(): + from evals.drivers.api.driver import tool_result_from_mcp + + mixed = tool_result_from_mcp( + "call_2", + {"content": [{"type": "text", "text": "chart:"}, {"type": "image", "data": "AAAA"}]}, + ) + assert mixed.kind == "mixed" + assert '"image"' in mixed.text and "chart:" in mixed.text + + image_only = tool_result_from_mcp("call_3", {"content": [{"type": "image", "data": "AAAA"}]}) + assert image_only.kind == "image" + assert '"data":"AAAA"' in image_only.text + + +def _tool_result_from_mcp_propagates_error_flag_in_both_spellings(): + from evals.drivers.api.driver import tool_result_from_mcp + + assert tool_result_from_mcp("c", {"content": "boom", "isError": True}).is_error is True + assert tool_result_from_mcp("c", SimpleNamespace(content="boom", is_error=True)).is_error is True + + +@pytest.mark.parametrize( + "case", + case_params( + _tool_spec_from_mcp_reads_dict_and_object_entries, + _tool_result_from_mcp_text_only_joins_blocks, + _tool_result_from_mcp_serializes_non_text_blocks, + _tool_result_from_mcp_propagates_error_flag_in_both_spellings, + ), +) +def test_tool_behaviours(case): + case() diff --git a/tests/evals/drivers/test_cli_driver.py b/tests/evals/drivers/test_cli_driver.py new file mode 100644 index 0000000..5405640 --- /dev/null +++ b/tests/evals/drivers/test_cli_driver.py @@ -0,0 +1,1650 @@ +"""Offline eval tests for cli driver.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import textwrap +import time +from pathlib import Path + +import pytest +import tomllib + +from evals.core.evidence import EVIDENCE_SENTINELS_ENV, TARGET_ENTITY_EVIDENCE +from evals.drivers.cli.antigravity import AntigravityCliDriver +from evals.drivers.cli.base import CliDriver, CliLaunch, CliOutput, CliOutputError +from evals.drivers.cli.claude import ClaudeCliDriver +from evals.drivers.cli.codex import CodexCliDriver, prepare_codex_home +from evals.drivers.cli.opencode import OpencodeCliDriver +from evals.drivers.cli.process import run_cli_subprocess +from evals.drivers.cli.sidecar import ( + apply_proxy_sidecar, + ensure_proxy_pythonpath, + harvest_proxy_after_cli_timeout, + load_proxy_sidecar, + load_proxy_sidecar_calls, + proxy_pid_path, + proxy_wrap_server_command, + wait_for_proxy_meta, +) +from tests.evals.conftest import case_params + + +def _pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # exists but not owned by us + return True + + +REPO = Path(__file__).resolve().parents[3] + + +def _run_cli_subprocess_kills_process_group_on_timeout(tmp_path, _monkeypatch): + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_cli.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + # Grandchild stays in the same process group (no start_new_session). + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + # Hold our stdout open forever (simulates grandchild pipe hold). + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + t0 = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired) as ei: + run_cli_subprocess( + [sys.executable, str(script)], + timeout=1.0, + capture_output=True, + text=True, + ) + elapsed = time.monotonic() - t0 + assert elapsed < 6.0, f"timeout path took {elapsed:.1f}s (unbounded communicate hang?)" + assert getattr(ei.value, "killed_process_group", False) is True + + # Wait briefly for reaping + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2, f"pidfile incomplete: {pidfile} {pids}" + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"process group members still alive: {alive}" + + +def _run_cli_subprocess_baseexception_kills_group(tmp_path, monkeypatch): + from evals.drivers.cli import process as process_mod + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(9999)"]) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + real_comm = subprocess.Popen.communicate + calls = {"n": 0} + + def boom_communicate(self, *a, **k): + calls["n"] += 1 + if calls["n"] == 1: + # Wait until pidfile is written so we can assert both die. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if pidfile.is_file() and len(pidfile.read_text().splitlines()) >= 2: + break + time.sleep(0.02) + raise KeyboardInterrupt("injected mid-communicate") + return real_comm(self, *a, **k) + + monkeypatch.setattr(subprocess.Popen, "communicate", boom_communicate) + + t0 = time.monotonic() + with pytest.raises(KeyboardInterrupt): + run_cli_subprocess( + [sys.executable, str(script)], + timeout=30.0, + capture_output=True, + text=True, + ) + assert time.monotonic() - t0 < 6.0 + + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2 and not any(_pid_alive(p) for p in pids): + break + time.sleep(0.05) + assert len(pids) == 2 + alive = [p for p in pids if _pid_alive(p)] + assert not alive, f"group survived BaseException path: {alive}" + # silence unused import lint if any + assert process_mod.run_cli_subprocess is run_cli_subprocess + + +@pytest.mark.parametrize( + "case", + case_params(_run_cli_subprocess_kills_process_group_on_timeout, _run_cli_subprocess_baseexception_kills_group), +) +def test_run_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def test_killpg_reaps_grandchild_when_leader_already_dead(tmp_path: Path): + """killpg(leader_pid) works after the leader is reaped (no getpgid / no proc.kill fallback). + + Simulates: leader already gone, only grandchild remains in the process group. + """ + import signal + from types import SimpleNamespace + + from evals.drivers.cli.process import kill_process_group + + pidfile = tmp_path / "pids.txt" + script = tmp_path / "sticky_leader.py" + script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + pidfile = Path({str(pidfile)!r}) + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(9999)"], + ) + pidfile.write_text(f"{{os.getpid()}}\\n{{child.pid}}\\n") + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + leader = subprocess.Popen( + [sys.executable, str(script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 3.0 + pids: list[int] = [] + while time.monotonic() < deadline: + if pidfile.is_file(): + pids = [int(x) for x in pidfile.read_text().splitlines() if x.strip()] + if len(pids) == 2: + break + time.sleep(0.02) + assert len(pids) == 2, pids + leader_pid, child_pid = pids + + # Kill ONLY the leader (not the group) — grandchild survives in the group. + os.kill(leader_pid, signal.SIGKILL) + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + pass + assert not _pid_alive(leader_pid) + assert _pid_alive(child_pid), "precondition: grandchild must still be alive" + + t0 = time.monotonic() + # Direct killpg(leader_pid) — pgid == original leader pid under start_new_session. + ok = kill_process_group(SimpleNamespace(pid=leader_pid)) + assert ok is True + assert time.monotonic() - t0 < 3.0 + + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline and _pid_alive(child_pid): + time.sleep(0.05) + assert not _pid_alive(child_pid), "grandchild survived killpg after leader death" + finally: + if leader.poll() is None: + try: + os.killpg(leader.pid, signal.SIGKILL) + except Exception: + leader.kill() + try: + leader.wait(timeout=2.0) + except Exception: + pass + + +def _cli_driver_timeout_notes_process_group_kill(tmp_path, _monkeypatch): + script = tmp_path / "slow.py" + script.write_text( + textwrap.dedent( + """ + import time + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + # Use real run_cli_subprocess with a tiny timeout via fake that wraps it. + from evals.drivers.cli.process import run_cli_subprocess as real_runner + + def short_timeout_runner(cmd, **kwargs): + kwargs = dict(kwargs) + kwargs["timeout"] = 0.5 + # Replace the CLI binary with our sticky sleeper + return real_runner([sys.executable, str(script)], **kwargs) + + driver = ClaudeCliDriver(runner=short_timeout_runner, use_proxy=False) + t0 = time.monotonic() + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert time.monotonic() - t0 < 6.0 + assert run.stopped_reason == "timeout" + assert "timeout_killed_process_group" in run.notes + + +def _cli_driver_template_inherits_proxy_first_and_timeout_harvest(tmp_path, monkeypatch): + clock = {"now": 0.0} + monkeypatch.setattr("evals.drivers.cli.base.time.perf_counter", lambda: clock["now"]) + + class MinimalCliDriver(CliDriver): + name = "minimal-cli" + temp_dir_prefix = "plane-eval-minimal-" + + def write_mcp_config( + self, + temp_dir: Path, + *, + task_cwd: Path, + server_command: list[str], + child_env: dict[str, str], + ) -> CliLaunch: + del temp_dir, child_env + # Harness-owned setup takes five seconds on the fake clock. The + # persisted wall time must start after this hook returns. + clock["now"] = 5.0 + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command( + self, + prompt: str, + *, + model: str | None, + max_turns: int, + system: str | None, + launch: CliLaunch, + ) -> list[str]: + del model, max_turns, system, launch + return ["minimal", prompt] + + def parse_output( + self, + proc: subprocess.CompletedProcess[str], + *, + launch: CliLaunch, + task_cwd: Path, + max_turns: int, + notes: list[str], + ) -> CliOutput: + del proc, launch, task_cwd, max_turns, notes + return CliOutput( + final_text="done", + calls=[ + {"tool": "cli_fallback_one", "args": {}, "origin": "plane"}, + {"tool": "cli_fallback_two", "args": {}, "origin": "plane"}, + ], + ) + + def write_complete_sidecar(path: Path, tool: str) -> None: + rows = [ + { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + }, + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + success_driver: MinimalCliDriver + + def success_runner(cmd, **kwargs): + write_complete_sidecar(success_driver.sidecar_path, "proxy_first") + clock["now"] = 7.0 + return subprocess.CompletedProcess(cmd, 0, stdout="ignored", stderr="") + + success_driver = MinimalCliDriver(runner=success_runner, use_proxy=True) + success = success_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert success.call_source == "proxy" + assert [call["tool"] for call in success.calls] == ["proxy_first"] + assert success.wall_time_s == 2.0 + + timeout_driver: MinimalCliDriver + + def timeout_runner(cmd, **kwargs): + write_complete_sidecar(timeout_driver.sidecar_path, "before_timeout") + clock["now"] = 8.0 + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs["timeout"]) + + timeout_driver = MinimalCliDriver(runner=timeout_runner, use_proxy=True) + timed_out = timeout_driver.run_task("go", {}, None, 1, cwd=tmp_path) + assert timed_out.stopped_reason == "timeout" + assert timed_out.call_source == "proxy" + assert [call["tool"] for call in timed_out.calls] == ["before_timeout"] + assert timed_out.wall_time_s == 3.0 + + +@pytest.mark.parametrize( + "case", + case_params( + _cli_driver_timeout_notes_process_group_kill, + _cli_driver_template_inherits_proxy_first_and_timeout_harvest, + ), +) +def test_cli_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def test_old_payload_free_sidecar_still_parses(tmp_path: Path): + path = tmp_path / "old.jsonl" + path.write_text( + json.dumps( + { + "tool": "legacy", + "args": {}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 1, + } + ) + + "\n" + + json.dumps( + { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + } + ) + + "\n", + encoding="utf-8", + ) + + calls, status = load_proxy_sidecar(path) + assert status["state"] == "complete" + assert calls[0]["result_chars"] == 17 + assert "result_text" not in calls[0] + + +def test_cli_parse_failure_retains_lossy_sidecar_integrity(tmp_path: Path): + class BrokenOutputDriver(CliDriver): + name = "broken-output-cli" + + def write_mcp_config(self, temp_dir, *, task_cwd, server_command, child_env): + del temp_dir, child_env + self.sidecar_path = Path(server_command[server_command.index("--log") + 1]) + return CliLaunch(cwd=task_cwd) + + def build_command(self, prompt, *, model, max_turns, system, launch): + del prompt, model, max_turns, system, launch + return ["broken-output"] + + def parse_output(self, proc, *, launch, task_cwd, max_turns, notes): + del proc, launch, task_cwd, max_turns, notes + raise CliOutputError("cannot parse output") + + driver: BrokenOutputDriver + + def fake_run(command, **kwargs): + del kwargs + rows = [ + { + "row_type": "proxy_meta", + "unmatched_responses": 1, + "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 0, + "tool_request_count": 0, + } + ] + driver.sidecar_path.write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0, stdout="bad", stderr="") + + driver = BrokenOutputDriver(runner=fake_run, use_proxy=True) + + with pytest.raises(RuntimeError, match="cannot parse output") as exc_info: + driver.run_task("go", {}, None, 1, cwd=tmp_path) + + assert exc_info.value.trace_integrity is False + assert exc_info.value.trace_integrity_reason == "recorder_loss" + + +def _apply_proxy_sidecar_replaces_when_nonempty(tmp_path): + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps( + { + "tool": "find_work_items", + "args": {"q": "x"}, + "is_error": False, + "result_chars": 12, + "duration_ms": 5, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, client, src = apply_proxy_sidecar( + [{"tool": "old", "args": {}, "origin": "plane"}], + [], + side, + notes, + max_wait_s=0, + ) + assert src == "proxy" + assert calls[0]["tool"] == "find_work_items" + assert calls[0]["duration_ms"] == 5 + assert any("calls_from_proxy" in n for n in notes) + + +def _apply_proxy_sidecar_empty_fallback(tmp_path): + side = tmp_path / "empty.jsonl" + side.write_text("", encoding="utf-8") + notes: list[str] = [] + original = [{"tool": "from_cli", "args": {}, "origin": "plane"}] + calls, _client, src = apply_proxy_sidecar(original, [], side, notes, max_wait_s=0) + assert calls is original or calls == original + assert "proxy_sidecar_empty" in notes + assert src != "proxy" or calls == original + + +def _apply_proxy_incomplete_defers_to_richer_cli(tmp_path): + p = tmp_path / "s.jsonl" + # Incomplete: one proxy call, no meta. + p.write_text( + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + ) + + "\n", + encoding="utf-8", + ) + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes, max_wait_s=0) + assert src != "proxy" + assert [c["tool"] for c in calls] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + +def _apply_proxy_with_skipped_row_defers_to_richer_cli(tmp_path): + p = tmp_path / "s.jsonl" + rows = [ + json.dumps( + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + ), + "{corrupted mid-stream row", + json.dumps( + { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + } + ), + ] + p.write_text("\n".join(rows) + "\n", encoding="utf-8") + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + + calls, _client, src = apply_proxy_sidecar(cli, [], p, notes) + + assert src == "json" + assert [call["tool"] for call in calls] == ["c1", "c2"] + assert "proxy_sidecar_incomplete:skipped_rows=1" in notes + assert "proxy_sidecar_deferred_to_cli_trace" in notes + + +@pytest.mark.parametrize( + "case", + case_params( + _apply_proxy_sidecar_replaces_when_nonempty, + _apply_proxy_sidecar_empty_fallback, + _apply_proxy_incomplete_defers_to_richer_cli, + _apply_proxy_with_skipped_row_defers_to_richer_cli, + ), +) +def test_apply_behaviours(case, tmp_path): + case(tmp_path) + + +def test_proxy_wrap_server_command(): + out = proxy_wrap_server_command( + ["python", "-m", "plane_mcp", "stdio"], + sidecar_path=Path("/tmp/s.jsonl"), + python_bin="/venv/bin/python", + ) + assert out[:5] == ["/venv/bin/python", "-m", "evals.proxy", "--log", "/tmp/s.jsonl"] + assert out[5] == "--" + assert out[6:] == ["python", "-m", "plane_mcp", "stdio"] + + with_payloads = proxy_wrap_server_command( + ["server"], + sidecar_path=Path("/tmp/s.jsonl"), + python_bin="python", + record_result_payloads=True, + ) + assert with_payloads[5:7] == ["--record-result-payloads", "--"] + + +def _load_proxy_sidecar_sorts_by_seq(tmp_path): + p = tmp_path / "s.jsonl" + # Append in reverse response order. + rows = [ + {"tool": "b", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 2}, + {"tool": "a", "args": {}, "is_error": False, "result_chars": 1, "duration_ms": 1, "seq": 1}, + { + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 1, + "tool_request_count": 1, + "child_killed": False, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls = load_proxy_sidecar_calls(p) + assert [c["tool"] for c in calls] == ["a", "b"] + + +def _load_proxy_sidecar_torn_final_line(tmp_path): + p = tmp_path / "s.jsonl" + good = { + "tool": "a", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + # Complete call row + torn final line (no proxy_meta). + p.write_text(json.dumps(good) + "\n" + '{"tool": "b", "args":', encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status["torn_line"] is True + assert status["meta"] is None + assert [c["tool"] for c in calls] == ["a"] + + +@pytest.mark.parametrize( + "case", + case_params(_load_proxy_sidecar_sorts_by_seq, _load_proxy_sidecar_torn_final_line), +) +def test_load_behaviours(case, tmp_path): + case(tmp_path) + + +@pytest.mark.parametrize( + ("bad_row", "case_id"), + [ + ("{corrupted mid-stream row", "invalid-json"), + (json.dumps(["not", "an", "object"]), "non-object-json"), + (json.dumps({"args": {"lost": "tool"}}), "missing-tool"), + ], + ids=lambda value: value if value in {"invalid-json", "non-object-json", "missing-tool"} else None, +) +def test_load_proxy_sidecar_skipped_row_makes_trace_incomplete(tmp_path: Path, bad_row: str, case_id: str): + path = tmp_path / f"{case_id}.jsonl" + call = { + "tool": "visible", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = {"row_type": "proxy_meta", "pending_left": 0, "pumps_alive": False} + path.write_text("\n".join((json.dumps(call), bad_row, json.dumps(meta))) + "\n", encoding="utf-8") + + calls, status = load_proxy_sidecar(path) + + assert [row["tool"] for row in calls] == ["visible"] + assert status["skipped_rows"] == 1 + assert status["state"] == "incomplete" + + +def test_server_cmd_reaches_all_cli_drivers(tmp_path: Path): + def make_fake(driver_cls: type, bag: dict): + def fake_run(cmd, **kwargs): + bag["cmd"] = cmd + if driver_cls is ClaudeCliDriver and "--mcp-config" in cmd: + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + bag["cfg"] = json.loads(cfg.read_text()) + elif driver_cls is OpencodeCliDriver: + cwd = kwargs.get("cwd") + if cwd: + cfg = Path(cwd) / "opencode.json" + if cfg.is_file(): + bag["cfg"] = json.loads(cfg.read_text()) + elif driver_cls is AntigravityCliDriver: + flag = next((a for a in cmd if a.startswith("--gemini_dir=")), None) + if flag: + gemini_dir = Path(flag.split("=", 1)[1]) + for rel in ( + Path("config") / "mcp_config.json", + Path("antigravity-cli") / "mcp_config.json", + ): + p = gemini_dir / rel + if p.is_file(): + bag.setdefault("cfgs", []).append(json.loads(p.read_text())) + elif driver_cls is CodexCliDriver: + codex_home = Path(kwargs["env"]["CODEX_HOME"]) + with (codex_home / "config.toml").open("rb") as stream: + bag["cfg"] = tomllib.load(stream) + out = ( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + if driver_cls is ClaudeCliDriver + else "{}" + ) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + return fake_run + + for Driver, bin_key in ( + (ClaudeCliDriver, "claude_bin"), + (CodexCliDriver, "codex_bin"), + (AntigravityCliDriver, "agy_bin"), + (OpencodeCliDriver, "opencode_bin"), + ): + seen: dict = {} + kwargs = { + "runner": make_fake(Driver, seen), + "use_proxy": True, + "record_result_payloads": True, + "python_bin": sys.executable, + "server_command": ["/ext/bin/foreign-mcp", "stdio", "--mode", "candidate"], + } + if Driver is CodexCliDriver: + kwargs["allow_live"] = True + kwargs[bin_key] = "fake-bin" + driver = Driver(**kwargs) + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + blob = json.dumps(seen) + assert "foreign-mcp" in blob or "foreign-mcp" in seen.get("cmd_joined", "") + assert "record-result-payloads" in blob or "record-result-payloads" in seen.get("cmd_joined", "") + + +def test_codex_isolated_home_effective_mcp_server_list_is_exactly_plane(tmp_path: Path): + codex_bin = shutil.which("codex") + assert codex_bin is not None, "Codex CLI is required to observe its effective MCP configuration" + + fake_user_home = tmp_path / "fake-user" + global_config = fake_user_home / ".codex" / "config.toml" + global_config.parent.mkdir(parents=True) + global_config.write_text( + '[mcp_servers.forbidden_global]\ncommand = "/usr/bin/false"\n', + encoding="utf-8", + ) + project_config = tmp_path / ".codex" / "config.toml" + project_config.parent.mkdir() + project_config.write_text( + '[mcp_servers.forbidden_project]\ncommand = "/usr/bin/false"\n', + encoding="utf-8", + ) + driver = CodexCliDriver(codex_bin=codex_bin, runner=lambda *_args, **_kwargs: None, allow_live=True) + launch = driver.write_mcp_config( + tmp_path / "task-state", + task_cwd=tmp_path, + server_command=["/usr/bin/true"], + child_env={"PATH": os.environ["PATH"]}, + ) + assert launch.env is not None + effective_env = {**launch.env, "HOME": str(fake_user_home)} + + observed = subprocess.run( + [codex_bin, "mcp", "list", "--json"], + cwd=tmp_path, + env=effective_env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + server_names = sorted(item["name"] for item in json.loads(observed.stdout)) + + assert server_names == ["plane"] + + +def test_claude_cli_runs_with_isolated_environment(tmp_path: Path, monkeypatch): + ambient_config = tmp_path / "ambient-claude" + ambient_config.mkdir() + credentials = ambient_config / ".credentials.json" + credentials.write_text('{"sessionKey":"copied-login-only"}\n', encoding="utf-8") + (ambient_config / "settings.json").write_text('{"ambient":true}\n', encoding="utf-8") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(ambient_config)) + + driver = ClaudeCliDriver(runner=lambda *_args, **_kwargs: None) + launch = driver.write_mcp_config( + tmp_path / "task-state", + task_cwd=tmp_path, + server_command=["/usr/bin/true"], + child_env={"PATH": os.environ["PATH"]}, + ) + + assert launch.env is not None + isolated_roots = [ + Path(launch.env[name]) + for name in ( + "HOME", + "CLAUDE_CONFIG_DIR", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "XDG_STATE_HOME", + ) + ] + assert all(root.is_dir() and root.is_relative_to(tmp_path / "task-state") for root in isolated_roots) + isolated_config = Path(launch.env["CLAUDE_CONFIG_DIR"]) + assert (isolated_config / ".credentials.json").read_text(encoding="utf-8") == credentials.read_text( + encoding="utf-8" + ) + assert not (isolated_config / "settings.json").exists() + assert Path(launch.config_args[1]) == isolated_config / ".claude.json" + command = driver.build_command("prompt", model=None, max_turns=1, system=None, launch=launch) + assert "--strict-mcp-config" in command + + +def test_claude_credentials_copy_failure_aborts_before_cli(tmp_path: Path, monkeypatch): + ambient_config = tmp_path / "ambient-claude" + ambient_config.mkdir() + (ambient_config / ".credentials.json").write_text('{"sessionKey":"source"}\n', encoding="utf-8") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(ambient_config)) + invoked = False + + def fake_run(*_args, **_kwargs): + nonlocal invoked + invoked = True + return subprocess.CompletedProcess([], 0, stdout='{"result":"unexpected"}', stderr="") + + def fail_copy(*_args, **_kwargs): + raise OSError("injected credential copy failure") + + monkeypatch.setattr("evals.drivers.cli.claude.shutil.copy2", fail_copy) + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False) + + with pytest.raises(RuntimeError, match="failed to copy Claude credentials into isolated config"): + driver.run_task("prompt", {}, None, 1, cwd=tmp_path) + assert invoked is False + + +def test_claude_credentials_refresh_limitation_reaches_run_notes(tmp_path: Path): + def fake_run(cmd, **_kwargs): + output = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "refresh-limit-session", + "num_turns": 1, + } + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(output), stderr="") + + run = ClaudeCliDriver(runner=fake_run, use_proxy=False).run_task("prompt", {}, None, 1, cwd=tmp_path) + + assert any(note.startswith("known_limitation:claude_file_credentials_refresh_discarded:") for note in run.notes) + + +def test_claude_isolated_environment_management_readback_lists_exactly_plane(tmp_path: Path): + claude_bin = shutil.which("claude") + assert claude_bin is not None, "Claude CLI is required to observe its effective MCP configuration" + + stub = tmp_path / "mcp_stub.py" + stub.write_text( + textwrap.dedent( + """ + import json + import sys + + def send(message): + sys.stdout.write(json.dumps(message) + "\\n") + sys.stdout.flush() + + for line in sys.stdin: + message = json.loads(line) + if "id" not in message: + continue + if message.get("method") == "initialize": + result = { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "serverInfo": {"name": "readback-stub", "version": "1"}, + } + elif message.get("method") == "tools/list": + result = {"tools": []} + else: + result = {} + send({"jsonrpc": "2.0", "id": message["id"], "result": result}) + """ + ), + encoding="utf-8", + ) + driver = ClaudeCliDriver(claude_bin=claude_bin, runner=lambda *_args, **_kwargs: None) + launch = driver.write_mcp_config( + tmp_path / "task-state", + task_cwd=tmp_path, + server_command=[sys.executable, str(stub)], + child_env={"PATH": os.environ["PATH"]}, + ) + assert launch.env is not None + assert Path(launch.config_args[1]) == Path(launch.env["CLAUDE_CONFIG_DIR"]) / ".claude.json" + + observed = subprocess.run( + [claude_bin, "mcp", "list"], + cwd=tmp_path, + env=launch.env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + server_names = sorted( + line.split(":", 1)[0].strip() + for line in observed.stdout.splitlines() + if ": " in line and not line.startswith("Checking MCP server health") + ) + help_text = subprocess.run( + [claude_bin, "--help"], + text=True, + capture_output=True, + check=True, + timeout=15, + ).stdout + + assert server_names == ["plane"] + assert "Only use MCP servers from --mcp-config" in help_text + assert "ignoring all other MCP configurations" in help_text + + +def test_opencode_isolated_environment_effective_mcp_server_list_is_exactly_plane(tmp_path: Path): + opencode_bin = shutil.which("opencode") + assert opencode_bin is not None, "OpenCode CLI is required to observe its effective MCP configuration" + + driver = OpencodeCliDriver(opencode_bin=opencode_bin, runner=lambda *_args, **_kwargs: None) + task_state = tmp_path / "task-state" + task_state.mkdir() + launch = driver.write_mcp_config( + task_state, + task_cwd=tmp_path, + server_command=["/usr/bin/true"], + child_env={"PATH": os.environ["PATH"]}, + ) + assert launch.env is not None + user_config = Path(launch.env["HOME"]) / ".config" / "opencode" / "opencode.json" + user_config.parent.mkdir(parents=True) + user_config.write_text( + json.dumps( + { + "mcp": { + "forbidden_global": { + "type": "local", + "command": ["/usr/bin/false"], + "enabled": True, + } + } + } + ), + encoding="utf-8", + ) + + # The isolated environment is the point of the test, and it is also why the readback can + # fail: opencode in a scrubbed HOME may block on setup it cannot complete. An unavailable + # CLI cannot answer the question, so skip — never pass — rather than leaving `pytest + # tests/evals` deterministically red on any machine where opencode is installed. + try: + observed = subprocess.run( + [opencode_bin, "debug", "config"], + cwd=launch.cwd, + env=launch.env, + text=True, + capture_output=True, + check=True, + timeout=15, + ) + except subprocess.TimeoutExpired: + pytest.skip("opencode did not answer 'debug config' in an isolated environment") + except subprocess.CalledProcessError as exc: + pytest.skip(f"opencode 'debug config' failed in an isolated environment: {exc.stderr or exc}") + effective_config = json.loads(observed.stdout) + + assert sorted((effective_config.get("mcp") or {}).keys()) == ["plane"] + + +@pytest.mark.parametrize( + ("Driver", "bin_key"), + [ + pytest.param(ClaudeCliDriver, "claude_bin", id="claude-config"), + pytest.param(CodexCliDriver, "codex_bin", id="codex-argv"), + pytest.param(AntigravityCliDriver, "agy_bin", id="antigravity-gemini-dir-config"), + pytest.param(OpencodeCliDriver, "opencode_bin", id="opencode-cwd-config"), + ], +) +def test_cli_agent_surfaces_never_contain_evidence_truth( + tmp_path: Path, + Driver: type[CliDriver], + bin_key: str, +): + sentinel = "hidden-target-fact-7b0a1f9c" + total_count = 918273 + grouped_counts = {"project-1": 564738, "project-2": 102938} + seen: dict[str, object] = {} + + def fake_run(cmd, **kwargs): + configs: list[dict] = [] + proxy_args: list[str] | None = None + if Driver is ClaudeCliDriver: + config_path = Path(cmd[cmd.index("--mcp-config") + 1]) + configs.append(json.loads(config_path.read_text())) + proxy_args = configs[0]["mcpServers"]["plane"]["args"] + elif Driver is CodexCliDriver: + codex_home = Path(kwargs["env"]["CODEX_HOME"]) + with (codex_home / "config.toml").open("rb") as stream: + config = tomllib.load(stream) + configs.append(config) + server = config["mcp_servers"]["plane"] + proxy_args = [server["command"], *server["args"]] + elif Driver is AntigravityCliDriver: + flag = next(a for a in cmd if a.startswith("--gemini_dir=")) + gemini_dir = Path(flag.split("=", 1)[1]) + for rel in ( + Path("config/mcp_config.json"), + Path("antigravity-cli/mcp_config.json"), + ): + configs.append(json.loads((gemini_dir / rel).read_text())) + proxy_args = configs[0]["mcpServers"]["plane"]["args"] + else: + config_path = Path(kwargs["cwd"]) / "opencode.json" + configs.append(json.loads(config_path.read_text())) + proxy_args = configs[0]["mcp"]["plane"]["command"] + + if Driver is ClaudeCliDriver: + assert "--strict-mcp-config" in cmd + assert set(configs[0]["mcpServers"]) == {"plane"} + assert all( + kwargs["env"].get(name) for name in ("HOME", "CLAUDE_CONFIG_DIR", "XDG_CONFIG_HOME", "XDG_DATA_HOME") + ) + elif Driver is CodexCliDriver: + assert set(configs[0]["mcp_servers"]) == {"plane"} + elif Driver is AntigravityCliDriver: + assert set(configs[0]["mcpServers"]) == {"plane"} + # HOME is intentionally the real one here — see the keychain note on the driver. + assert Path(gemini_dir).is_absolute() + else: + assert set(configs[0]["mcp"]) == {"plane"} + assert all(kwargs["env"].get(name) for name in ("HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME")) + + assert proxy_args is not None and "--evidence-file" in proxy_args + evidence_path = Path(proxy_args[proxy_args.index("--evidence-file") + 1]) + launch_cwd = Path(kwargs["cwd"]).resolve() + assert not evidence_path.resolve().is_relative_to(launch_cwd) + # Even a lazy MCP launcher leaves only non-invertible fingerprints for a + # shell-capable agent that follows the pathname before proxy startup. + evidence_config = evidence_path.read_text(encoding="utf-8") + seen["surface"] = json.dumps({"argv": cmd, "configs": configs, "evidence_file": evidence_config}) + out = ( + json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + } + ) + if Driver is ClaudeCliDriver + else "{}" + ) + return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="") + + kwargs = { + "runner": fake_run, + "python_bin": sys.executable, + "use_proxy": True, + bin_key: "fake-bin", + } + if Driver is CodexCliDriver: + kwargs["allow_live"] = True + driver = Driver(**kwargs) + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="test-model", + max_turns=1, + cwd=tmp_path, + evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1", *grouped_counts]}, + evidence_aggregates={ + TARGET_ENTITY_EVIDENCE: [ + {"kind": "total_count", "value": total_count}, + {"kind": "grouped_counts", "values": grouped_counts}, + ] + }, + ) + + surface = str(seen["surface"]) + assert sentinel not in surface + assert str(total_count) not in surface + assert all(str(count) not in surface for count in grouped_counts.values()) + assert EVIDENCE_SENTINELS_ENV not in surface + + +def test_antigravity_effective_config_exclusivity_is_documented_as_unverifiable(): + driver_doc = AntigravityCliDriver.__doc__ or "" + design = (REPO / "evals" / "DESIGN.md").read_text(encoding="utf-8") + + assert "has no MCP or effective-config introspection command" in driver_doc + assert "cannot be proven by real-binary readback" in driver_doc + assert "| Antigravity | **Unverifiable.**" in design + assert "neither the harness nor this design treats that as observed effective-config exclusivity" in design + assert 'The Antigravity "unverifiable" regression test is documentation coverage' in design + + +def test_claude_effective_config_claim_scopes_observation_and_vendor_contract(): + driver_doc = ClaudeCliDriver.__doc__ or "" + design = (REPO / "evals" / "DESIGN.md").read_text(encoding="utf-8") + + assert "not a behavioral probe of the evaluated ``claude -p`` invocation" in driver_doc + assert "Readback-supported, not behaviorally proven for the evaluated invocation" in design + assert "rests on the CLI's documented strict-config contract" in design + + +def test_use_proxy_false_call_source_not_proxy(tmp_path: Path): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"x"}', stderr="") + + for Driver in (AntigravityCliDriver, OpencodeCliDriver): + d = Driver(runner=fake_run, use_proxy=False, python_bin=sys.executable) + run = d.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + assert run.call_source != "proxy" + + +def test_ensure_proxy_pythonpath_injects_repo(): + env = ensure_proxy_pythonpath({}) + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + # Idempotent + env2 = ensure_proxy_pythonpath(env) + assert env2["PYTHONPATH"].count(str(REPO)) == 1 + + +def _timeout_harvests_sidecar_calls(tmp_path): + side_calls = [ + { + "tool": "pre_timeout", + "args": {"a": 1}, + "is_error": False, + "result_chars": 3, + "duration_ms": 1, + "seq": 1, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 1, + "tool_request_count": 1, + "child_killed": False, + "evidence_trace_available": True, + }, + ] + + def fake_run(cmd, **kwargs): + # Plant a complete sidecar next to the mcp config (temp dir still alive). + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + # Sidecar path is in the same temp dir as mcp.json for Claude. + # Find sidecar from proxy args in mcp config. + mcp = json.loads(cfg.read_text()) + assert EVIDENCE_SENTINELS_ENV not in mcp["mcpServers"]["plane"]["env"] + args = mcp["mcpServers"]["plane"]["args"] + assert "--evidence-file" in args + log_idx = args.index("--log") + 1 + side = Path(args[log_idx]) + side.write_text("\n".join(json.dumps(r) for r in side_calls) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + evidence_sentinels={TARGET_ENTITY_EVIDENCE: ["hidden-target-fact-7b0a1f9c"]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "pre_timeout" + assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.evidence_trace_available is True + + +def test_cli_verifier_compares_observed_aggregate_values_after_agent_run(tmp_path): + rows = [ + { + "tool": "count_work_items", + "args": {"project_id": "project-1"}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 1, + "observed_sentinels": [], + "observed_aggregates": [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 3}], + }, + { + "tool": "count_work_items", + "args": {"project_id": "project-1"}, + "is_error": False, + "result_chars": 17, + "duration_ms": 1, + "seq": 2, + "observed_sentinels": [], + "observed_aggregates": [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 4}], + }, + { + "row_type": "proxy_meta", + "relayed_lines": 2, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "non_tool_pending_left": 0, + "last_seq": 2, + "tool_request_count": 2, + "child_killed": False, + "evidence_trace_available": True, + }, + ] + + def fake_run(cmd, **kwargs): + del kwargs + config_path = Path(cmd[cmd.index("--mcp-config") + 1]) + config = json.loads(config_path.read_text(encoding="utf-8")) + proxy_args = config["mcpServers"]["plane"]["args"] + sidecar = Path(proxy_args[proxy_args.index("--log") + 1]) + sidecar.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1"]}, + evidence_aggregates={TARGET_ENTITY_EVIDENCE: [{"kind": "total_count", "value": 4}]}, + ) + + assert run.evidence_trace_available is True + assert run.calls[0]["observed_sentinels"] == [] + assert run.calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + + +def _timeout_harvest_waits_for_delayed_meta(tmp_path): + import threading + import time as time_mod + + call_row = { + "tool": "late_meta_tool", + "args": {"n": 1}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + } + meta_row = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + "pumps_alive": False, + } + seen: dict = {"waited": False} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + # Call row first — no meta yet (simulates proxy still finalizing). + side.write_text(json.dumps(call_row) + "\n", encoding="utf-8") + + def write_meta_later() -> None: + time_mod.sleep(0.45) + with side.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta_row) + "\n") + seen["waited"] = True + + threading.Thread(target=write_meta_later, daemon=True).start() + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + t0 = time_mod.monotonic() + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + elapsed = time_mod.monotonic() - t0 + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "late_meta_tool" + assert seen["waited"] is True + # Must have waited for the delayed meta (~0.45s), not returned instantly. + assert elapsed >= 0.4 + assert "proxy_meta_wait_timeout" not in run.notes + + +def _timeout_incomplete_sidecar_cannot_supply_response_evidence(tmp_path): + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + mcp = json.loads(cfg.read_text()) + args = mcp["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + call = { + "tool": "evidence_call", + "args": {}, + "is_error": False, + "result_chars": 5, + "duration_ms": 1, + "seq": 1, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + meta = { + "row_type": "proxy_meta", + "pending_left": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + "evidence_trace_available": True, + } + side.write_text( + "\n".join((json.dumps(call), "{corrupted mid-stream row", json.dumps(meta))) + "\n", + encoding="utf-8", + ) + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + evidence_sentinels={TARGET_ENTITY_EVIDENCE: ["hidden-target-fact-7b0a1f9c"]}, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + + assert run.call_source == "proxy" + assert run.calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert run.evidence_trace_available is False + assert "proxy_sidecar_incomplete:skipped_rows=1" in run.notes + assert "proxy_response_evidence_unavailable" in run.notes + + +@pytest.mark.parametrize( + "case", + case_params( + _timeout_harvests_sidecar_calls, + _timeout_harvest_waits_for_delayed_meta, + _timeout_incomplete_sidecar_cannot_supply_response_evidence, + ), +) +def test_timeout_behaviours(case, tmp_path): + case(tmp_path) + + +def test_wait_for_proxy_meta_unit(tmp_path: Path): + side = tmp_path / "s.jsonl" + side.write_text("", encoding="utf-8") + assert wait_for_proxy_meta(side, max_wait_s=0.15, poll_s=0.05) is False + side.write_text(json.dumps({"row_type": "proxy_meta", "pending_left": 0}) + "\n", encoding="utf-8") + assert wait_for_proxy_meta(side, max_wait_s=1.0, poll_s=0.05) is True + + +def test_normal_completion_waits_for_delayed_proxy_meta(tmp_path: Path): + """The shared normal path must not harvest the call row before final metadata.""" + import threading + + early_notes: list[str] = [] + meta_written = threading.Event() + writer_threads: list[threading.Thread] = [] + + def fake_run(cmd, **kwargs): + del kwargs + config_path = Path(cmd[cmd.index("--mcp-config") + 1]) + config = json.loads(config_path.read_text(encoding="utf-8")) + proxy_args = config["mcpServers"]["plane"]["args"] + sidecar = Path(proxy_args[proxy_args.index("--log") + 1]) + call = { + "tool": "delayed_meta_tool", + "args": {"n": 1}, + "is_error": False, + "result_chars": 2, + "duration_ms": 1, + "seq": 1, + } + sidecar.write_text(json.dumps(call) + "\n", encoding="utf-8") + proxy_pid_path(sidecar).write_text(str(os.getpid()), encoding="ascii") + + # This is the historical harvest: the call exists, but finalization has + # not happened, so accepting it now manufactures recorder loss. + early = apply_proxy_sidecar([], [], sidecar, early_notes, max_wait_s=0) + assert early.trace_integrity is False + assert "proxy_sidecar_incomplete:no_meta=1" in early_notes + + def write_meta_later() -> None: + time.sleep(0.08) + meta = { + "row_type": "proxy_meta", + "pending_left": 0, + "non_tool_pending_left": 0, + "unmatched_responses": 0, + "unparsed_lines": 0, + "recorder_errors": 0, + "pumps_alive": False, + "last_seq": 1, + "tool_request_count": 1, + } + with sidecar.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(meta) + "\n") + meta_written.set() + + writer = threading.Thread(target=write_meta_later, daemon=True) + writer_threads.append(writer) + writer.start() + output = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "delayed-meta-session", + "num_turns": 1, + } + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(output), stderr="") + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + for writer in writer_threads: + writer.join(timeout=1.0) + + assert meta_written.is_set() + assert run.trace_integrity is True + assert run.trace_integrity_reason is None + assert run.call_source == "proxy" + assert [call["tool"] for call in run.calls] == ["delayed_meta_tool"] + assert not any("no_meta" in note for note in run.notes) + + +def test_proxy_meta_wait_timeout_is_fatal_and_diagnosable(tmp_path: Path): + sidecar = tmp_path / "never-finalized.jsonl" + sidecar.write_text( + json.dumps({"tool": "unfinished", "args": {}, "seq": 1}) + "\n", + encoding="utf-8", + ) + proxy_pid_path(sidecar).write_text(str(os.getpid()), encoding="ascii") + notes: list[str] = [] + + result = apply_proxy_sidecar([], [], sidecar, notes, poll_s=0.005, max_wait_s=0.03) + + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert "proxy_meta_wait_timeout:proxy_alive=1" in notes + assert "proxy_sidecar_incomplete:no_meta=1" in notes + + +def test_proxy_exit_before_meta_is_fatal_and_diagnosable(tmp_path: Path): + sidecar = tmp_path / "exited-before-meta.jsonl" + sidecar.write_text( + json.dumps({"tool": "unfinished", "args": {}, "seq": 1}) + "\n", + encoding="utf-8", + ) + exited = subprocess.Popen([sys.executable, "-c", "pass"]) + exited.wait(timeout=2.0) + proxy_pid_path(sidecar).write_text(str(exited.pid), encoding="ascii") + notes: list[str] = [] + started = time.monotonic() + + result = apply_proxy_sidecar([], [], sidecar, notes, poll_s=0.01, max_wait_s=1.0) + + assert time.monotonic() - started < 0.2 + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert "proxy_meta_missing_after_proxy_exit" in notes + assert "proxy_sidecar_incomplete:no_meta=1" in notes + + +def test_proxy_meta_fast_path_does_not_sleep(tmp_path: Path, monkeypatch): + sidecar = tmp_path / "already-finalized.jsonl" + meta = { + "row_type": "proxy_meta", + "pending_left": 0, + "last_seq": 0, + "tool_request_count": 0, + } + sidecar.write_text(json.dumps(meta) + "\n", encoding="utf-8") + monkeypatch.setattr("evals.drivers.cli.sidecar.time.sleep", lambda _seconds: pytest.fail("fast path slept")) + notes: list[str] = [] + started = time.monotonic() + + result = apply_proxy_sidecar([], [], sidecar, notes, max_wait_s=1.0) + + assert time.monotonic() - started < 0.1 + assert result.trace_integrity is True + assert not any("proxy_meta_wait" in note for note in notes) + + +def test_harvest_proxy_after_cli_timeout_incomplete_note(tmp_path: Path): + """If meta never arrives, harvest still returns with incomplete note.""" + side = tmp_path / "s.jsonl" + side.write_text( + json.dumps({"tool": "only", "args": {}, "seq": 1, "is_error": False, "result_chars": 0}) + "\n", + encoding="utf-8", + ) + notes: list[str] = [] + calls, _client, src = harvest_proxy_after_cli_timeout([], [], side, notes, max_wait_s=0.25) + assert "proxy_meta_wait_timeout:proxy_state=unknown" in notes + assert len(calls) == 1 + assert src == "proxy" + assert any("incomplete" in n for n in notes) + + +def test_codex_isolated_home_routes_approvals_through_automatic_review(tmp_path: Path): + """An isolated home must permit unattended MCP calls, or the battery measures nothing. + + ``codex exec`` is non-interactive: an MCP call that raises an approval request is cancelled + by Codex itself with ``user cancelled MCP tool call``, and the agent then answers without + touching the surface. A live run recorded zero calls on all four tasks while still emitting + confident answers, and 562 passing tests said nothing about it. + """ + codex_home = tmp_path / "codex-home" + prepare_codex_home( + codex_home, + command="/usr/bin/true", + args=["-m", "plane_mcp", "stdio"], + env={"PLANE_API_KEY": "k"}, + real_codex_home=tmp_path / "absent", + ) + config = (codex_home / "config.toml").read_text(encoding="utf-8") + assert 'approvals_reviewer = "auto_review"' in config + + +def test_codex_nonzero_exit_is_not_scored_as_a_finished_attempt(tmp_path: Path): + """A failed codex process is infrastructure, not a model failure. + + Codex defaulted to ``end_turn`` whatever its exit code, so an authentication or network + failure that still emitted a partial ``agent_message`` was verified and counted in the + success rate. opencode and antigravity already map a nonzero exit to ``error``, so this + also skewed every driver comparison against them. + """ + driver = CodexCliDriver(codex_bin="/usr/bin/true", runner=lambda *_a, **_k: None, allow_live=True) + stdout = json.dumps({"type": "agent_message", "message": "partial"}) + "\n" + notes: list[str] = [] + launch = None + + completed = driver.parse_output( + subprocess.CompletedProcess(["codex"], 1, stdout=stdout, stderr="auth failed"), + launch=launch, + task_cwd=tmp_path, + max_turns=10, + notes=notes, + ) + assert completed.stopped_reason == "error" + assert any(note == "codex_exit=1" for note in notes) + + clean = driver.parse_output( + subprocess.CompletedProcess(["codex"], 0, stdout=stdout, stderr=""), + launch=launch, + task_cwd=tmp_path, + max_turns=10, + notes=[], + ) + assert clean.stopped_reason == "end_turn" + + +def _claude_command(**kw): + driver = ClaudeCliDriver(**kw) + launch = CliLaunch(cwd=Path("/tmp"), config_args=["--mcp-config", "/tmp/cfg.json"]) + return driver.build_command("do the task", model="haiku", max_turns=8, system=None, launch=launch) + + +def test_the_agent_gets_no_builtin_tools_by_default(): + """An eval of a tool surface must not hand the agent a shell to route around it. + + Measured: with Bash available, a model that could not work the surface out read the + repo it was standing in, harvested the API key from its own environment and called + Plane's REST API directly — the task verified as passed with zero MCP calls. + """ + command = _claude_command() + assert "--tools=" in command, command + # The `=` form matters: --tools is variadic and the bare form eats the prompt. + assert not any(part == "--tools" for part in command), command + assert command[-1] == "do the task", command[-1] + + +def test_builtin_tools_can_be_restored_or_named(): + assert "--tools=" not in _claude_command(builtin_tools=None) + assert "--tools=Bash,Read" in _claude_command(builtin_tools="Bash,Read") diff --git a/tests/evals/drivers/test_vendors.py b/tests/evals/drivers/test_vendors.py new file mode 100644 index 0000000..ac3e625 --- /dev/null +++ b/tests/evals/drivers/test_vendors.py @@ -0,0 +1,1113 @@ +"""Offline eval tests for vendors.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from evals.core.tool_names import ( + split_plane_and_client_calls, +) +from evals.drivers import KNOWN_DRIVERS, get_driver +from evals.drivers.api.driver import ApiDriver +from evals.drivers.cli.antigravity import ( + AntigravityCliDriver, + prepare_antigravity_gemini_dir, + write_antigravity_mcp_config, +) +from evals.drivers.cli.claude import ( + ClaudeCliDriver, + normalize_claude_usage, + parse_claude_json_result, + parse_claude_transcript_calls, + write_claude_mcp_config, +) +from evals.drivers.cli.codex import CodexCliDriver, parse_codex_jsonl_events +from evals.drivers.cli.opencode import OpencodeCliDriver, write_opencode_mcp_config +from tests.evals.conftest import case_params + +CLAUDE_JSON_RESULT = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "The work item is in Todo.", + "session_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "num_turns": 3, + "total_cost_usd": 0.291, + "stop_reason": "end_turn", + "usage": { + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "iterations": [ + { + "input_tokens": 2, + "output_tokens": 8, + "cache_read_input_tokens": 57985, + "cache_creation_input_tokens": 754, + "type": "message", + } + ], + "speed": "standard", + }, + "modelUsage": { + "claude-sonnet-4-20250514": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.291, + "contextWindow": 200000, + } + }, +} + +CLAUDE_JSON_WITH_CALLS = { + **CLAUDE_JSON_RESULT, + "tool_calls": [ + { + "name": "ToolSearch", + "input": {"query": "work items", "max_results": 5}, + }, + { + "name": "mcp__plane__find_work_items", + "input": {"project": "EVAL deadbeef", "limit": 10}, + }, + { + "name": "mcp__plane__get_work_item", + "input": {"project_id": "p1", "work_item_id": "w1"}, + }, + ], +} + + +def _transcript_lines(*, include_tool_search: bool = False) -> str: + content_blocks: list[dict] = [] + if include_tool_search: + content_blocks.append( + { + "type": "tool_use", + "id": "toolu_0", + "name": "ToolSearch", + "input": {"query": "select:find_work_items", "max_results": 1}, + } + ) + content_blocks.append( + { + "type": "tool_use", + "id": "toolu_1", + "name": "mcp__plane__list_work_items", + "input": {"project_id": "proj-1", "per_page": 25}, + } + ) + rows = [ + { + "type": "assistant", + "sessionId": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "message": { + "role": "assistant", + "content": content_blocks, + "usage": {"input_tokens": 100, "output_tokens": 20}, + }, + }, + { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "[]"}], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_2", + "name": "mcp__plane__get_work_item", + "input": {"project_id": "proj-1", "work_item_id": "wi-1"}, + } + ], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "Done."}], + "stop_reason": "end_turn", + }, + }, + ] + return "\n".join(json.dumps(r) for r in rows) + "\n" + + +CODEX_JSONL = "\n".join( + [ + json.dumps( + { + "type": "session_meta", + "payload": {"id": "sess-codex-1", "cwd": "/tmp", "cli_version": "0.0-test"}, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "mcp__plane__find_work_items", + "arguments": json.dumps({"project": "EVAL x", "limit": 5}), + "call_id": "call_1", + }, + } + ), + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec_command", + "arguments": json.dumps({"cmd": "echo hi"}), + "call_id": "call_2", + }, + } + ), + json.dumps( + { + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "total_token_usage": { + "input_tokens": 5000, + "output_tokens": 200, + "cached_input_tokens": 1000, + "cache_write_input_tokens": 50, + "total_tokens": 5200, + } + }, + }, + } + ), + json.dumps( + { + "type": "event_msg", + "payload": {"type": "agent_message", "message": "All set."}, + } + ), + json.dumps({"type": "event_msg", "payload": {"type": "task_complete"}}), + ] +) + +CODEX_V0147_JSONL = "\n".join( + [ + json.dumps( + { + "type": "thread.started", + "thread_id": "019ff6af-69df-7022-b353-322ffe1ececb", + } + ), + json.dumps({"type": "turn.started"}), + json.dumps( + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "PING"}, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 16050, + "cached_input_tokens": 15104, + "cache_write_input_tokens": 0, + "output_tokens": 5, + "reasoning_output_tokens": 0, + }, + } + ), + ] +) + +REPO = Path(__file__).resolve().parents[3] + + +def test_normalize_claude_usage_real_shape(): + """F2: uncached input_tokens=10 must not be treated as run total.""" + raw, total = normalize_claude_usage(CLAUDE_JSON_RESULT) + assert raw is not None + assert raw["input_tokens"] == 10 # uncached-only + assert total is not None + assert total["input_tokens"] == 10 + assert total["cache_read_input_tokens"] == 250433 + assert total["cache_creation_input_tokens"] == 33838 + assert total["output_tokens"] == 865 + assert total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert total["total_cost_usd"] == 0.291 + assert total["source"] == "modelUsage" + + +def _parse_claude_json_result_usage_and_cost(_tmp_path): + out = parse_claude_json_result(CLAUDE_JSON_RESULT) + assert out["final_text"] == "The work item is in Todo." + assert out["session_id"] == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + assert out["num_turns"] == 3 + assert out["usage"]["input_tokens"] == 10 + assert out["usage"]["total_cost_usd"] == 0.291 + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["calls"] == [] + assert out["stopped_reason"] == "end_turn" + + +def _parse_claude_json_with_embedded_calls_splits_toolsearch(_tmp_path): + out = parse_claude_json_result(CLAUDE_JSON_WITH_CALLS) + assert [c["tool"] for c in out["calls"]] == ["find_work_items", "get_work_item"] + assert all(c["origin"] == "plane" for c in out["calls"]) + assert [c["tool"] for c in out["client_tool_calls"]] == ["ToolSearch"] + assert out["calls"][0]["args"]["limit"] == 10 + + +def _parse_claude_json_preserves_error_subtype(_tmp_path): + out = parse_claude_json_result( + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "x", + "session_id": "s", + "num_turns": 1, + } + ) + assert out["stopped_reason"] == "error_during_execution" + + +def _parse_claude_transcript_calls(tmp_path): + p = tmp_path / "sess.jsonl" + p.write_text(_transcript_lines(include_tool_search=True), encoding="utf-8") + tagged = parse_claude_transcript_calls(p) + plane, client = split_plane_and_client_calls(tagged) + assert [c["tool"] for c in plane] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in client] == ["ToolSearch"] + assert plane[0]["args"]["project_id"] == "proj-1" + + +def _parse_codex_jsonl_events(_tmp_path): + out = parse_codex_jsonl_events(CODEX_JSONL) + assert out["session_id"] == "sess-codex-1" + assert out["final_text"] == "All set." + assert out["usage"]["input_tokens"] == 5000 + assert out["usage"]["cache_read_input_tokens"] == 1000 + # plane only in calls; exec_command is client machinery + tools = [c["tool"] for c in out["calls"]] + assert tools == ["find_work_items"] + assert [c["tool"] for c in out["client_tool_calls"]] == ["exec_command"] + assert out["stopped_reason"] == "end_turn" + + +def _parse_codex_jsonl_events_v0147_schema(_tmp_path): + out = parse_codex_jsonl_events(CODEX_V0147_JSONL) + assert out["session_id"] == "019ff6af-69df-7022-b353-322ffe1ececb" + assert out["final_text"] == "PING" + assert out["usage"]["input_tokens"] == 16050 + assert out["usage"]["cache_read_input_tokens"] == 15104 + assert out["usage"]["cache_creation_input_tokens"] == 0 + assert out["usage"]["output_tokens"] == 5 + + +def _parse_codex_jsonl_events_mixed_old_and_new_schema(_tmp_path): + mixed = "\n".join( + [ + json.dumps({"type": "thread.started", "thread_id": "thread-new-1"}), + json.dumps( + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "Hello from new"}, + } + ), + # Legacy call row still harvested + json.dumps( + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "mcp__plane__list_work_items", + "arguments": json.dumps({"project_id": "p"}), + }, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 2, + }, + } + ), + ] + ) + out = parse_codex_jsonl_events(mixed) + assert out["session_id"] == "thread-new-1" + assert "Hello from new" in out["final_text"] + assert [c["tool"] for c in out["calls"]] == ["list_work_items"] + assert out["usage"]["input_tokens"] == 10 + + +@pytest.mark.parametrize( + "case", + case_params( + _parse_claude_json_result_usage_and_cost, + _parse_claude_json_with_embedded_calls_splits_toolsearch, + _parse_claude_json_preserves_error_subtype, + _parse_claude_transcript_calls, + _parse_codex_jsonl_events, + _parse_codex_jsonl_events_v0147_schema, + _parse_codex_jsonl_events_mixed_old_and_new_schema, + ), +) +def test_parse_behaviours(case, tmp_path): + case(tmp_path) + + +def _find_codex_rollout_exact_match_and_unmatched(tmp_path, monkeypatch): + from evals.drivers.cli import codex as codex_mod + + sessions = tmp_path / ".codex" / "sessions" / "2026" / "04" / "01" + sessions.mkdir(parents=True) + tid = "019ff6af-69df-7022-b353-322ffe1ececb" + # Unrelated newer session (must never be returned when looking for tid) + other = sessions / "rollout-2026-04-01T12-00-00-other-session-zzzz.jsonl" + other.write_text( + json.dumps({"type": "thread.started", "thread_id": "other-session-zzzz"}) + "\n", + encoding="utf-8", + ) + # Exact match via filename suffix + match = sessions / f"rollout-2026-04-01T12-00-01-{tid}.jsonl" + match.write_text( + json.dumps({"type": "thread.started", "thread_id": tid}) + "\n", + encoding="utf-8", + ) + + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + found = codex_mod.find_codex_rollout(tid) + assert found is not None + assert tid in found.name + # Must not return the other concurrent session + assert "other-session" not in found.name + + assert codex_mod.find_codex_rollout("does-not-exist-anywhere") is None + assert codex_mod.find_codex_rollout(None) is None + + +def _find_codex_rollout_session_meta_id(tmp_path, monkeypatch): + from evals.drivers.cli import codex as codex_mod + + sessions = tmp_path / ".codex" / "sessions" + sessions.mkdir(parents=True) + p = sessions / "rollout-meta-only.jsonl" + p.write_text( + json.dumps({"type": "session_meta", "payload": {"id": "sess-meta-42"}}) + "\n", + encoding="utf-8", + ) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + found = codex_mod.find_codex_rollout("sess-meta-42") + assert found is not None + assert found.name == "rollout-meta-only.jsonl" + + +@pytest.mark.parametrize( + "case", + case_params(_find_codex_rollout_exact_match_and_unmatched, _find_codex_rollout_session_meta_id), +) +def test_find_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def _codex_driver_notes_rollout_unmatched_when_no_file(tmp_path, monkeypatch): + (tmp_path / ".codex" / "sessions").mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_V0147_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run, use_proxy=False) + run = driver.run_task( + "ping", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + # Final text still from stdout (new schema) — never from a wrong rollout + assert run.final_text == "PING" + # Unmatched note only when looking for enrichment; with final_text present + # need_rollout is false for final_text — still may note if no calls. + # v0147 fixture has no tool calls → need_rollout True → unmatched note. + assert "codex_rollout_unmatched" in run.notes + + +def _codex_driver_parses_fake_stdout_no_live(_tmp_path, _monkeypatch): + def fake_run(cmd, **kwargs): + assert cmd[0] == "codex" + assert "exec" in cmd + assert "--json" in cmd + return subprocess.CompletedProcess(cmd, 0, stdout=CODEX_JSONL, stderr="") + + driver = CodexCliDriver(runner=fake_run) # fake runner → no allow_live needed + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="gpt-test", + max_turns=5, + cwd=Path("/tmp"), + ) + assert run.experimental is True + assert run.call_source == "stream" + assert run.calls[0]["tool"] == "find_work_items" + assert [c["tool"] for c in run.client_tool_calls] == ["exec_command"] + assert run.usage is not None + assert run.usage["input_tokens"] == 5000 + assert run.final_text == "All set." + + +def _codex_driver_refuses_live_by_default(_tmp_path, _monkeypatch): + driver = CodexCliDriver() # real subprocess.run + with pytest.raises(RuntimeError, match="refuses live"): + driver.run_task("x", mcp_env={}, model=None, max_turns=1) + + +@pytest.mark.parametrize( + "case", + case_params( + _codex_driver_notes_rollout_unmatched_when_no_file, + _codex_driver_parses_fake_stdout_no_live, + _codex_driver_refuses_live_by_default, + ), +) +def test_codex_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def test_max_turns_detection_from_num_turns(): + """When num_turns >= max_turns, driver reports hit_max_turns / max_turns stop.""" + payload = {**CLAUDE_JSON_RESULT, "num_turns": 15, "tool_calls": []} + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + + driver = ClaudeCliDriver(runner=fake_run) + # Avoid looking for a real transcript for empty calls + run = driver.run_task( + "do the thing", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=15, + cwd=Path("/tmp"), + ) + assert run.hit_max_turns is True + assert run.stopped_reason == "max_turns" + assert run.usage_scope == "run" + assert run.usage is not None + assert run.usage["input_tokens"] == 10 # uncached-only from real shape + assert run.usage_total is not None + assert run.usage_total["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + + +def _claude_driver_falls_back_to_transcript(tmp_path, _monkeypatch): + session_id = "bbbbbbbb-cccc-dddd-eeee-ffffffffffff" + payload = { + **CLAUDE_JSON_RESULT, + "session_id": session_id, + "tool_calls": [], # force transcript path + "result": "from-json", + } + + def fake_run(cmd, **kwargs): + config_dir = Path(kwargs["env"]["CLAUDE_CONFIG_DIR"]) + munged = str(tmp_path.resolve()).replace("/", "-") + project_dir = config_dir / "projects" / munged + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / f"{session_id}.jsonl").write_text( + _transcript_lines(include_tool_search=True), + encoding="utf-8", + ) + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "prompt", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=10, + cwd=tmp_path, + ) + assert run.call_source == "transcript" + assert [c["tool"] for c in run.calls] == ["list_work_items", "get_work_item"] + assert [c["tool"] for c in run.client_tool_calls] == ["ToolSearch"] + assert run.final_text == "from-json" + + +def test_claude_transcript_raw_ref_survives_run_task(tmp_path): + session_id = "cccccccc-dddd-eeee-ffff-000000000001" + payload = { + **CLAUDE_JSON_RESULT, + "session_id": session_id, + "tool_calls": [], + "result": "from-json", + } + + def fake_run(cmd, **kwargs): + config_dir = Path(kwargs["env"]["CLAUDE_CONFIG_DIR"]) + munged = str(tmp_path.resolve()).replace("/", "-") + project_dir = config_dir / "projects" / munged + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / f"{session_id}.jsonl").write_text( + _transcript_lines(include_tool_search=True), + encoding="utf-8", + ) + return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps(payload), stderr="") + + run = ClaudeCliDriver(runner=fake_run, use_proxy=False).run_task( + "prompt", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=10, + cwd=tmp_path, + ) + + assert run.raw_ref is not None + transcript = Path(run.raw_ref) + assert transcript.is_file() + assert [call["tool"] for call in parse_claude_transcript_calls(transcript)] == [ + "ToolSearch", + "list_work_items", + "get_work_item", + ] + + +def test_claude_transcript_lookup_is_launch_local_and_reentrant(tmp_path): + session_id = "same-session-id" + task_cwd = tmp_path / "task-cwd" + task_cwd.mkdir() + driver = ClaudeCliDriver(runner=lambda *_args, **_kwargs: None, use_proxy=False) + + def launch_with_transcript(name: str): + launch = driver.write_mcp_config( + tmp_path / f"state-{name}", + task_cwd=task_cwd, + server_command=["/usr/bin/true"], + child_env={}, + ) + launch.artifact_dir = tmp_path / f"artifacts-{name}" + config_dir = Path((launch.env or {})["CLAUDE_CONFIG_DIR"]) + munged = str(task_cwd.resolve()).replace("/", "-") + transcript_dir = config_dir / "projects" / munged + transcript_dir.mkdir(parents=True) + transcript_dir.joinpath(f"{session_id}.jsonl").write_text( + json.dumps( + { + "message": { + "content": [ + { + "type": "tool_use", + "name": f"mcp__plane__tool_{name}", + "input": {"surface": name}, + } + ] + } + } + ) + + "\n", + encoding="utf-8", + ) + return launch + + launch_a = launch_with_transcript("a") + launch_b = launch_with_transcript("b") + payload = json.dumps({**CLAUDE_JSON_RESULT, "session_id": session_id, "tool_calls": []}) + proc = subprocess.CompletedProcess([], 0, stdout=payload, stderr="") + + output_a = driver.parse_output(proc, launch=launch_a, task_cwd=task_cwd, max_turns=10, notes=[]) + output_b = driver.parse_output(proc, launch=launch_b, task_cwd=task_cwd, max_turns=10, notes=[]) + + assert [call["tool"] for call in output_a.calls] == ["tool_a"] + assert [call["tool"] for call in output_b.calls] == ["tool_b"] + assert output_a.raw_ref is not None and Path(output_a.raw_ref).is_relative_to(launch_a.artifact_dir) + assert output_b.raw_ref is not None and Path(output_b.raw_ref).is_relative_to(launch_b.artifact_dir) + + +def _claude_driver_writes_mcp_config_and_cmd_flags(tmp_path, _monkeypatch): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + seen["cwd"] = kwargs.get("cwd") + seen["env"] = kwargs.get("env") + # Return minimal valid JSON + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, python_bin="/venv/bin/python") + driver.run_task( + "hello", + mcp_env={ + "PLANE_API_KEY": "key", + "PLANE_WORKSPACE_SLUG": "slug", + "PLANE_BASE_URL": "https://api.example", + "CUSTOM_SETTING": "enabled", + "PATH": "/usr/bin", + }, + model="sonnet", + max_turns=7, + cwd=tmp_path, + system="sys", + ) + cmd = seen["cmd"] + assert cmd[0] == "claude" + assert "-p" in cmd + assert "--output-format" in cmd and "json" in cmd + assert "--mcp-config" in cmd + assert "--max-turns" in cmd and "7" in cmd + assert "--model" in cmd and "sonnet" in cmd + assert "--permission-mode" in cmd and "bypassPermissions" in cmd + assert "--strict-mcp-config" in cmd + assert seen["env"]["HOME"] != str(Path.home()) + assert seen["env"]["CLAUDE_CONFIG_DIR"] + assert all(seen["env"][name] for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME")) + # mcp-config path is a temp file cleaned after run — re-check via write helper + cfg = tmp_path / "mcp.json" + write_claude_mcp_config( + cfg, + command="/venv/bin/python", + args=["-m", "plane_mcp", "stdio"], + env={"PLANE_API_KEY": "key"}, + ) + data = json.loads(cfg.read_text()) + assert "mcpServers" in data + assert data["mcpServers"]["plane"]["args"] == ["-m", "plane_mcp", "stdio"] + + +def _claude_driver_server_command_override(tmp_path, _monkeypatch): + seen: dict[str, Any] = {} + + def fake_run(cmd, **kwargs): + # Capture the mcp.json content while it still exists (temp dir). + cfg_path = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp_cfg"] = json.loads(cfg_path.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps({**CLAUDE_JSON_RESULT, "tool_calls": [], "num_turns": 1}), + stderr="", + ) + + driver = ClaudeCliDriver( + runner=fake_run, + server_command=["/elsewhere/.venv/bin/plane-mcp-server", "stdio", "--mode", "candidate"], + ) + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "s", "PLANE_FOREIGN_MODE": "candidate"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp_cfg"]["mcpServers"]["plane"] + # Default use_proxy=True: command is the proxy; real server follows "--". + assert server["args"][:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + dash = server["args"].index("--") + assert server["args"][dash + 1 :] == [ + "/elsewhere/.venv/bin/plane-mcp-server", + "stdio", + "--mode", + "candidate", + ] + # Explicit foreign selection variables pass through to the child. + assert server["env"]["PLANE_FOREIGN_MODE"] == "candidate" + + +def _claude_driver_timeout_returns_agent_run_not_raise(_tmp_path, _monkeypatch): + def fake_run(cmd, **kwargs): + raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout") or 120) + + driver = ClaudeCliDriver(runner=fake_run) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=2, + cwd=Path("/tmp"), + ) + assert run.stopped_reason == "timeout" + assert run.calls == [] + assert any("timeout after" in n for n in run.notes) + + +def _claude_driver_json_parse_failure_raises_for_infra_cli(_tmp_path, _monkeypatch): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout="not-json", stderr="boom") + + driver = ClaudeCliDriver(runner=fake_run) + with pytest.raises(RuntimeError, match="claude cli failed"): + driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=Path("/tmp"), + ) + + +def _claude_driver_uses_proxy_in_mcp_config(tmp_path, _monkeypatch): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + # Leave empty sidecar (proxy not really run under fake runner). + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "done", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin="/venv/bin/python") + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=3, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["command"] == "/venv/bin/python" + assert server["args"][0:3] == ["-m", "evals.proxy", "--log"] + assert "--" in server["args"] + assert "plane_mcp" in server["args"] + assert "proxy_sidecar_empty" in run.notes + + +def _claude_driver_proxy_disabled_no_wrap(tmp_path, _monkeypatch): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["mcp"] = json.loads(cfg.read_text()) + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + driver = ClaudeCliDriver(runner=fake_run, use_proxy=False, python_bin="/venv/bin/python") + driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + server = seen["mcp"]["mcpServers"]["plane"] + assert server["args"] == ["-m", "plane_mcp", "stdio"] + + +def _claude_mcp_env_has_pythonpath_when_proxied(tmp_path, _monkeypatch): + seen: dict = {} + + def fake_run(cmd, **kwargs): + cfg = Path(cmd[cmd.index("--mcp-config") + 1]) + seen["env"] = json.loads(cfg.read_text())["mcpServers"]["plane"]["env"] + return subprocess.CompletedProcess( + cmd, + 0, + stdout=json.dumps( + { + "type": "result", + "subtype": "success", + "is_error": False, + "result": "ok", + "session_id": "s", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ), + stderr="", + ) + + ClaudeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable).run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="sonnet", + max_turns=1, + cwd=tmp_path, + ) + assert str(REPO) in seen["env"].get("PYTHONPATH", "") + + +_CLAUDE_CASES = case_params( + _claude_driver_falls_back_to_transcript, + _claude_driver_writes_mcp_config_and_cmd_flags, + _claude_driver_server_command_override, + _claude_driver_timeout_returns_agent_run_not_raise, + _claude_driver_json_parse_failure_raises_for_infra_cli, + _claude_driver_uses_proxy_in_mcp_config, + _claude_driver_proxy_disabled_no_wrap, + _claude_mcp_env_has_pythonpath_when_proxied, +) + + +@pytest.mark.parametrize("case", _CLAUDE_CASES) +def test_claude_behaviours(case, tmp_path, monkeypatch): + case(tmp_path, monkeypatch) + + +def _known_drivers(): + assert KNOWN_DRIVERS == {"api", "claude-cli", "codex-cli", "antigravity-cli", "opencode-cli"} + + +def _known_drivers_and_get_driver(): + assert "antigravity-cli" in KNOWN_DRIVERS + assert "opencode-cli" in KNOWN_DRIVERS + assert isinstance(get_driver("antigravity-cli"), AntigravityCliDriver) + assert isinstance(get_driver("opencode-cli"), OpencodeCliDriver) + + +@pytest.mark.parametrize( + "case", + case_params(_known_drivers, _known_drivers_and_get_driver), +) +def test_known_drivers_behaviours(case): + case() + + +def test_get_driver_api(): + assert isinstance(get_driver("api"), ApiDriver) + assert isinstance(get_driver("claude-cli"), ClaudeCliDriver) + assert isinstance(get_driver("codex-cli"), CodexCliDriver) + + +def _antigravity_driver_isolates_via_gemini_dir_not_home(tmp_path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + seen["env"] = kwargs.get("env") or {} + flag = next((a for a in cmd if a.startswith("--gemini_dir=")), None) + seen["gemini_dir"] = flag + if flag: + cfg = Path(flag.split("=", 1)[1]) / "config" / "mcp_config.json" + seen["mcp_cfg"] = json.loads(cfg.read_text()) if cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout='{"result":"hi"}', stderr="") + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "do it", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws", "PATH": "/bin"}, + model="gemini-2.5", + max_turns=5, + cwd=tmp_path, + ) + assert seen["cmd"][0] == "agy" + assert "--output-format=json" in seen["cmd"] + assert "--dangerously-skip-permissions" in seen["cmd"] + assert "--model=gemini-2.5" in seen["cmd"] + # The prompt must be the VALUE of --print: agy parses with Go's flag package, so a + # bare "-p" followed by other flags eats the next flag as its prompt and drops the + # real one, taking --dangerously-skip-permissions down with it. + assert seen["cmd"][-1] == "--print=do it" + assert not any(a in ("-p", "--print") for a in seen["cmd"]) + assert "no_turn_cap" in run.notes + # The flag carries an absolute path and precedes the -p subcommand flags. + assert seen["gemini_dir"] is not None + assert Path(seen["gemini_dir"].split("=", 1)[1]).is_absolute() + assert seen["cmd"].index(seen["gemini_dir"]) == 1 + # HOME must stay the real one: agy reads its OAuth token from the macOS login + # keychain, which Security resolves under $HOME/Library/Keychains. Overriding it + # made the keychain unfindable and every run failed unauthenticated. + assert seen["env"].get("HOME") == os.environ.get("HOME") + assert seen.get("mcp_cfg") is not None + assert "mcpServers" in seen["mcp_cfg"] + assert "evals.proxy" in " ".join(seen["mcp_cfg"]["mcpServers"]["plane"]["args"]) + + +def _antigravity_timeout_still_harvests_proxy_rows(tmp_path): + call_row = { + "tool": "g_tool", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + } + meta = { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + } + + def fake_run(cmd, **kwargs): + # Plant the sidecar the way a real run would have, from whichever of the two + # dual-written configs is present, then time out. The rows must still be + # harvested: a timed-out agy has usually already made its calls. + flag = next(a for a in cmd if a.startswith("--gemini_dir=")) + gemini_dir = Path(flag.split("=", 1)[1]) + for rel in ( + gemini_dir / "config" / "mcp_config.json", + gemini_dir / "antigravity-cli" / "mcp_config.json", + ): + if rel.is_file(): + args = json.loads(rel.read_text())["mcpServers"]["plane"]["args"] + side = Path(args[args.index("--log") + 1]) + side.write_text("\n".join(json.dumps(r) for r in (call_row, meta)) + "\n", encoding="utf-8") + break + raise subprocess.TimeoutExpired(cmd=cmd, timeout=1) + + driver = AntigravityCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hi", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model=None, + max_turns=1, + cwd=tmp_path, + ) + assert run.stopped_reason == "timeout" + assert run.call_source == "proxy" + assert len(run.calls) == 1 + assert run.calls[0]["tool"] == "g_tool" + + +@pytest.mark.parametrize( + "case", + case_params( + _antigravity_driver_isolates_via_gemini_dir_not_home, + _antigravity_timeout_still_harvests_proxy_rows, + ), +) +def test_antigravity_behaviours(case, tmp_path): + case(tmp_path) + + +def _write_antigravity_mcp_config_shape(tmp_path): + p = tmp_path / "mcp_config.json" + write_antigravity_mcp_config(p, command="python", args=["-m", "x"], env={"A": "1"}) + data = json.loads(p.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + assert data["mcpServers"]["plane"]["env"]["A"] == "1" + + +def _write_opencode_mcp_config_shape(tmp_path): + p = tmp_path / "opencode.json" + write_opencode_mcp_config(p, command=["py", "-m", "plane_mcp", "stdio"], env={"K": "V"}) + data = json.loads(p.read_text()) + assert data["mcp"]["plane"]["command"][0] == "py" + assert data["mcp"]["plane"]["environment"]["K"] == "V" + + +@pytest.mark.parametrize( + "case", + case_params(_write_antigravity_mcp_config_shape, _write_opencode_mcp_config_shape), +) +def test_write_behaviours(case, tmp_path): + case(tmp_path) + + +def test_opencode_driver_writes_project_config(tmp_path: Path): + seen: dict = {} + + def fake_run(cmd, **kwargs): + seen["cmd"] = cmd + cwd = kwargs.get("cwd") + seen["cwd"] = cwd + cfg = Path(cwd) / "opencode.json" if cwd else None + seen["opencode_cfg"] = json.loads(cfg.read_text()) if cfg and cfg.is_file() else None + return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") + + driver = OpencodeCliDriver(runner=fake_run, use_proxy=True, python_bin=sys.executable) + run = driver.run_task( + "hello", + mcp_env={"PLANE_API_KEY": "k", "PLANE_WORKSPACE_SLUG": "ws"}, + model="openai/gpt-test", + max_turns=4, + cwd=tmp_path, + ) + assert seen["cmd"][0] == "opencode" + assert "run" in seen["cmd"] + assert "--format" in seen["cmd"] and "json" in seen["cmd"] + assert "-m" in seen["cmd"] and "openai/gpt-test" in seen["cmd"] + assert "no_turn_cap" in run.notes + data = seen["opencode_cfg"] + assert data is not None + assert data["mcp"]["plane"]["type"] == "local" + assert "evals.proxy" in " ".join(data["mcp"]["plane"]["command"]) + + +def test_prepare_antigravity_gemini_dir_dual_writes_and_leaves_home_alone(tmp_path: Path): + real_home = tmp_path / "real" + cli = real_home / ".gemini" / "antigravity-cli" + cli.mkdir(parents=True) + (cli / "antigravity-oauth-token").write_text("secret", encoding="utf-8") + (cli / "mcp_config.json").write_text('{"mcpServers": {"other": {}}}', encoding="utf-8") + before = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} + + gemini_dir = tmp_path / "isolated" / "gemini" + prepare_antigravity_gemini_dir( + gemini_dir, + command="python", + args=["-m", "evals.proxy", "--log", "s", "--", "x"], + env={"PLANE_API_KEY": "k"}, + ) + p1 = gemini_dir / "config" / "mcp_config.json" + p2 = gemini_dir / "antigravity-cli" / "mcp_config.json" + assert p1.is_file() and p2.is_file() + for path in (p1, p2): + data = json.loads(path.read_text()) + assert data["mcpServers"]["plane"]["command"] == "python" + # Only our server — the user's own entries are not carried over. + assert list(data["mcpServers"]) == ["plane"] + # Nothing is copied out of the real home and nothing written into it: auth comes + # from the login keychain, which stays reachable because HOME is never moved. + assert not (gemini_dir / "antigravity-cli" / "antigravity-oauth-token").exists() + after = {p.relative_to(real_home): p.read_bytes() for p in real_home.rglob("*") if p.is_file()} + assert after == before diff --git a/tests/evals/report/__init__.py b/tests/evals/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/report/test_compare.py b/tests/evals/report/test_compare.py new file mode 100644 index 0000000..e939823 --- /dev/null +++ b/tests/evals/report/test_compare.py @@ -0,0 +1,262 @@ +"""Offline eval tests for paired comparisons.""" + +from __future__ import annotations + +import math +from pathlib import Path + +import pytest + +from evals.report import ( + ab_compare, + paired_bootstrap_mean_ci, + paired_permutation_pvalue, + print_ab_report, +) + + +def test_paired_permutation_retains_ties_and_uses_delta_magnitudes(): + assert paired_permutation_pvalue([]) is None + assert paired_permutation_pvalue([0.0, 0.0]) == 1.0 + + # A sign test sees 6 positive versus 10 negative deltas and cannot detect + # the coherent large-magnitude shift. The paired randomization distribution + # uses those magnitudes while all 30 zero-delta task pairs remain in n=46. + deltas = [10.0] * 6 + [-0.1] * 10 + [0.0] * 30 + permutation_p = paired_permutation_pvalue(deltas) + old_sign_p = 2 * sum(math.comb(16, index) for index in range(7)) / (2**16) + assert old_sign_p > 0.05 + assert permutation_p == pytest.approx(0.03125) + + +def test_paired_bootstrap_small_sample_is_task_paired_and_wide(): + deltas = [1.0, 1.0, -1.0, 0.0, 0.0] + + lower, upper = paired_bootstrap_mean_ci(deltas) + + assert lower is not None and upper is not None + assert lower < 0.0 < upper + assert upper - lower >= 1.0 + + +def test_ab_compare_behaviours(capsys): + rows_a = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 5, "calls": [], "trace_integrity": True}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 3, "calls": [], "trace_integrity": True}, + {"task_id": "R3", "rep": 0, "success": False, "num_calls": 9, "calls": [], "trace_integrity": True}, + ] + rows_b = [ + {"task_id": "R1", "rep": 0, "success": True, "num_calls": 2, "calls": [], "trace_integrity": True}, + {"task_id": "R2", "rep": 0, "success": True, "num_calls": 4, "calls": [], "trace_integrity": True}, + {"task_id": "R3", "rep": 0, "success": True, "num_calls": 1, "calls": [], "trace_integrity": True}, + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["n_paired"] == 2 # R3 has no successful A call count + deltas = {pair["task_id"]: pair["delta"] for pair in comparison["paired_tasks"]} + assert deltas == {"R1": -3.0, "R2": 1.0} + assert comparison["mean_delta"] == pytest.approx(-1.0) + assert comparison["median_delta"] == pytest.approx(-1.0) + assert comparison["call_permutation_p"] is not None + assert comparison["call_zero_deltas"] == 0 + assert comparison["n_paired_success"] == 3 + assert comparison["paired_success_delta"] == pytest.approx(1 / 3) + assert comparison["success_a"]["k"] == 2 and comparison["success_a"]["n"] == 3 + assert comparison["success_b"]["k"] == 3 and comparison["success_b"]["n"] == 3 + + print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) + output = capsys.readouterr().out + assert "paired-bootstrap95" in output + assert "paired permutation p-value" in output + assert "zero-delta ties retained" in output + assert "sign-test" not in output + assert "noise floor" not in output + + +def test_ab_report_distinguishes_identical_success_and_calls_by_errored_calls(capsys): + rows_a = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 4, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + } + ] + rows_b = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 4, + "errored_calls": 2, + "calls": [], + "trace_integrity": True, + } + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["success_a"]["k"] == comparison["success_b"]["k"] == 1 + assert comparison["mean_delta"] == 0.0 + assert comparison["mean_errored_call_delta"] == 2.0 + assert comparison["mean_errored_call_rate_delta"] == pytest.approx(0.5) + assert comparison["paired_schema_friction"] == [ + { + "task_id": "R1", + "errored_calls_a": 0.0, + "errored_calls_b": 2.0, + "errored_call_delta": 2.0, + "errored_call_rate_a": 0.0, + "errored_call_rate_b": 0.5, + "errored_call_rate_delta": 0.5, + "raw_errored_calls_a": 0, + "raw_total_calls_a": 4, + "raw_errored_calls_b": 2, + "raw_total_calls_b": 4, + } + ] + + print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) + output = capsys.readouterr().out + assert "mean errored-call delta (B−A): +2.0" in output + assert "mean errored-call-rate delta (B−A): +50.0 percentage points" in output + assert "R1: A=0/4 (0.0%), median=0.0; B=2/4 (50.0%), median=2.0" in output + assert "is_error is the MCP-level error flag" in output + + +def test_ab_errored_call_rate_delta_averages_paired_tasks_instead_of_pooling_calls(): + rows_a = [ + { + "task_id": "R1", + "success": True, + "num_calls": 1, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R2", + "success": True, + "num_calls": 9, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + ] + rows_b = [ + { + "task_id": "R1", + "success": True, + "num_calls": 1, + "errored_calls": 1, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R2", + "success": True, + "num_calls": 9, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["mean_errored_call_delta"] == 0.5 + assert comparison["mean_errored_call_rate_delta"] == 0.5 + assert comparison["mean_errored_call_rate_delta"] != pytest.approx(1 / 10) + assert comparison["n_paired_errored_call_rates"] == 2 + assert comparison["errored_call_rate_delta_ci"] == pytest.approx((0.0375, 0.9625)) + + +def test_ab_compare_multi_rep_uses_median_successful_call_counts(): + rows_a = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 1, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 1, + "success": False, + "num_calls": 9, + "errored_calls": 99, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 2, + "success": True, + "num_calls": 5, + "errored_calls": 2, + "calls": [], + "trace_integrity": True, + }, + ] + rows_b = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "num_calls": 2, + "errored_calls": 0, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "num_calls": 4, + "errored_calls": 2, + "calls": [], + "trace_integrity": True, + }, + { + "task_id": "R1", + "rep": 2, + "success": True, + "num_calls": 6, + "errored_calls": 4, + "calls": [], + "trace_integrity": True, + }, + ] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["multi_rep"] is True + assert comparison["paired_tasks"] == [{"task_id": "R1", "calls_a": 3.0, "calls_b": 4.0, "delta": 1.0}] + assert comparison["mean_errored_call_delta"] == 1.0 + assert comparison["mean_errored_call_rate_delta"] == pytest.approx(1 / 6) + + +def test_ab_compare_excludes_trace_invalid_call_counts(): + rows_a = [ + { + "task_id": "R1", + "success": True, + "trace_integrity": False, + "trace_integrity_reason": "result_pair_mismatch", + "num_calls": 99, + "calls": [], + } + ] + rows_b = [{"task_id": "R1", "success": True, "trace_integrity": True, "num_calls": 1, "calls": []}] + + comparison = ab_compare(rows_a, rows_b) + + assert comparison["n_paired"] == 0 + assert comparison["paired_tasks"] == [] diff --git a/tests/evals/report/test_economics.py b/tests/evals/report/test_economics.py new file mode 100644 index 0000000..d026a68 --- /dev/null +++ b/tests/evals/report/test_economics.py @@ -0,0 +1,93 @@ +"""Cost, input volume, result volume and latency must reach the A/B view. + +These are all recorded already. Result tokens even reach ``--table``. They were +absent from the two-file comparison, which is the view a two-arm question is asked +in -- so the arm that made 32% fewer calls while burning 3.1x the input tokens read +as the efficient one for two days. +""" + +from __future__ import annotations + +from evals.core.pricing import PRICED, UNMEASURED +from evals.report import summarize +from evals.report.compare import ab_compare, print_ab_report + +OPENAI_USAGE = { + "input_tokens": 97813, + "output_tokens": 1121, + "cache_read_input_tokens": 83460, + "cache_creation_input_tokens": 0, + "source": "iterations", + "cache_semantics": "inclusive", +} +CODEX_USAGE = { + "input_tokens": 249983, + "output_tokens": 682, + "cache_read_input_tokens": 224768, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 474751, + "source": "codex_token_count", +} + + +def row(task_id, *, usage, model, calls=2, wall=10.0, latency=500.0, result_tokens=900): + return { + "task_id": task_id, + "success": True, + "trace_integrity": True, + "model": model, + "usage_total": usage, + "wall_time_s": wall, + "num_calls": calls, + "calls": [{"tool": "workitem", "duration_ms": latency, "result_tokens": result_tokens} for _ in range(calls)], + } + + +def test_summary_carries_arm_cost_and_normalised_input(): + economics = summarize([row("R1", usage=OPENAI_USAGE, model="gpt-5.6-luna")]).economics + assert economics.cost_outcome == PRICED + assert economics.cost_usd is not None and economics.cost_usd > 0 + assert economics.total_input_tokens == 97813 + assert economics.total_wall_time_s == 10.0 + assert economics.med_call_latency_ms == 500.0 + + +def test_an_arm_with_no_usage_reports_unmeasured_not_zero(): + """antigravity records usage on no row at all; $0.00 would read as free.""" + economics = summarize([row("R1", usage=None, model="gemini-3.6-flash-low")]).economics + assert economics.cost_outcome == UNMEASURED + assert economics.cost_usd is None + assert economics.unmeasured_rows == 1 + assert economics.cost_text == UNMEASURED + + +def test_ab_block_reports_the_metrics_that_invert_the_call_verdict(capsys): + """B makes fewer calls and costs far more -- both facts must be visible together.""" + rows_a = [row("R1", usage=OPENAI_USAGE, model="gpt-5.6-luna", calls=7, result_tokens=900)] + rows_b = [row("R1", usage=CODEX_USAGE, model="gpt-5.6-luna", calls=4, result_tokens=2500)] + comparison = ab_compare(rows_a, rows_b) + + assert comparison["total_input_a"] == 97813 + assert comparison["total_input_b"] == 474751 + assert comparison["cost_a"] is not None and comparison["cost_b"] is not None + assert comparison["cost_b"] > comparison["cost_a"] + + print_ab_report(comparison, "A.jsonl", "B.jsonl") + out = capsys.readouterr().out + for expected in ("input tokens", "cost", "result tokens", "wall time", "call latency"): + assert expected in out, f"{expected!r} missing from the A/B block" + # The call delta says B is better; the cost delta must be right there beside it. + assert "median call delta" in out + + +def test_unmeasured_arm_prints_a_word_not_a_zero(capsys): + rows_a = [row("R1", usage=OPENAI_USAGE, model="gpt-5.6-luna")] + rows_b = [row("R1", usage=None, model="gemini-3.6-flash-low")] + print_ab_report(ab_compare(rows_a, rows_b), "A.jsonl", "B.jsonl") + out = capsys.readouterr().out + b_lines = [line for line in out.splitlines() if line.startswith(" B economics:")] + assert b_lines, "arm B has no economics line" + assert "cost=unmeasured" in b_lines[0] + assert "input tokens=unmeasured" in b_lines[0] + # The unknown must never be rendered as a figure of any size, zero included. + assert "$" not in b_lines[0] diff --git a/tests/evals/report/test_economics_review.py b/tests/evals/report/test_economics_review.py new file mode 100644 index 0000000..eca524d --- /dev/null +++ b/tests/evals/report/test_economics_review.py @@ -0,0 +1,132 @@ +"""Regressions from the adversarial review of the pricing and economics change. + +Eight findings, all reproduced before being fixed. The three most serious all had +the same shape: something unknown or unverified presenting as a confident number, +which is the exact failure this code was written to prevent. +""" + +from __future__ import annotations + +from evals.core.pricing import PRICED, UNMEASURED, UNPRICED, price_usage, resolve_model_id +from evals.core.token_accounting import EXCLUSIVE, INCLUSIVE, normalize_usage +from evals.report.economics import measure_economics + +CLAUDE_USAGE = { + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 9.0, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + "source": "modelUsage", +} + + +def row(**overrides): + base = { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "model": "haiku", + "num_calls": 1, + "calls": [], + "usage_total": dict(CLAUDE_USAGE), + } + base.update(overrides) + return base + + +def test_f1_usage_present_but_carrying_no_tokens_is_unmeasured(): + """The api driver always writes a usage_total, even when every turn returned None. + + A dict with a source and no counts is not a measurement, and pricing it at + $0.00 is the precise "unknown reads as free" bug being designed against. + """ + empty = {"source": "iterations", "cache_semantics": INCLUSIVE} + assert normalize_usage(empty, model="gpt-5.6-luna") is None + assert price_usage(empty, model="gpt-5.6-luna").outcome == UNMEASURED + + +def test_f2_a_charged_row_whose_verifier_crashed_still_counts(): + """The model ran and the tokens were billed; a later crash does not refund them.""" + measurement = measure_economics([row(error="verifier crashed")]) + assert measurement.priced_rows == 1 + assert measurement.cost_usd is not None + + +def test_f3_drift_compares_the_table_against_the_vendor_not_the_vendor_against_itself(): + """The staleness detector was summing billed_usd, which already prefers vendor cost. + + Reported drift was therefore always $0.000 -- the one check the design leaned on + was structurally incapable of firing. + """ + measurement = measure_economics([row()]) + assert measurement.vendor_cost_usd == 9.0 + assert measurement.computed_cost_usd is not None + # The real table price for this row is cents, so drift against a $9 vendor + # figure must be large and negative. + assert measurement.computed_cost_usd < 1.0 + assert measurement.cost_drift_usd is not None and measurement.cost_drift_usd < -8.0 + + +def test_f4_a_partial_input_total_says_it_is_partial(): + priced = row() + blind = row(task_id="R2", usage_total={"input_tokens": 5, "cache_read_input_tokens": 4}, model="mystery") + measurement = measure_economics([priced, blind]) + assert measurement.missing_input_rows == 1 + assert "1 row" in measurement.input_text + + +def test_f5_a_declaration_that_contradicts_an_explicit_total_is_refused(): + """Declared semantics may interpret, but they may not override arithmetic.""" + contradictory = { + "input_tokens": 100, + "cache_read_input_tokens": 90, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 190, + "cache_semantics": INCLUSIVE, + } + assert normalize_usage(contradictory) is None + # Agreement still resolves normally. + consistent = dict(contradictory, cache_semantics=EXCLUSIVE) + accounting = normalize_usage(consistent) + assert accounting is not None and accounting.total_input == 190 + + +def test_f6_multi_model_usage_is_unpriced_rather_than_billed_at_one_rate(): + """Haiku and Opus tokens summed and priced at whichever alias the row carried.""" + usage = dict(CLAUDE_USAGE, modelUsage={"claude-haiku-4-5-20251001": {}, "claude-opus-5": {}}) + assert resolve_model_id(usage, model="claude-haiku-4-5") is None + assert price_usage(usage, model="claude-haiku-4-5").outcome == UNPRICED + + +def test_f6b_an_authoritative_vendor_cost_survives_an_unpriced_table_lookup(): + """A real billed figure must not vanish because the table lacks the model.""" + usage = dict(CLAUDE_USAGE, modelUsage={"a": {}, "b": {}}) + measurement = measure_economics([row(usage_total=usage, model="mystery")]) + assert measurement.unpriced_rows == 1 + assert measurement.vendor_cost_usd == 9.0 + assert measurement.cost_usd == 9.0 + + +def test_f7_a_small_real_cost_does_not_render_as_zero(): + tiny = {"input_tokens": 100, "output_tokens": 1, "cache_read_input_tokens": 0, "source": "iterations"} + measurement = measure_economics([row(usage_total=tiny, model="gpt-5.6-luna")]) + assert measurement.cost_outcome == PRICED + assert "$0.000 " not in measurement.cost_text + assert measurement.cost_text.startswith("$") or measurement.cost_text.startswith("<$") + + +def test_f8_a_backend_that_declares_nothing_is_not_recorded_as_exclusive(): + """getattr(..., False) turned an undeclared backend into a confident claim.""" + from evals.drivers.api.driver import cache_semantics_for + + class Undeclared: + pass + + class Inclusive: + input_tokens_include_cache = True + + assert cache_semantics_for(Undeclared()) is None + assert cache_semantics_for(Inclusive()) == INCLUSIVE diff --git a/tests/evals/report/test_identity.py b/tests/evals/report/test_identity.py new file mode 100644 index 0000000..26393f4 --- /dev/null +++ b/tests/evals/report/test_identity.py @@ -0,0 +1,491 @@ +"""Persisted run-identity guards for every report path.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from evals import report as report_mod +from evals.report import identity + + +def _row(task_id: str = "R1", **overrides: Any) -> dict[str, Any]: + row = { + "task_id": task_id, + "rep": 0, + "label": "candidate", + "battery": "samebattery1", + "resolved_model": "configured-model", + "provider": "anthropic", + "driver": "api", + "server": "local", + "model": "realized-model", + "requested_model": "standard", + "requested_tier": "standard", + "tool_manifest_fingerprint": "manifest-a", + "success": True, + "num_calls": 1, + "calls": [], + } + row.update(overrides) + return row + + +def _meta(**overrides: Any) -> dict[str, Any]: + row = { + "row_type": "meta", + "run_id": "run-1", + "battery": "samebattery1", + "resolved_model": "configured-model", + "provider": "anthropic", + "driver": "api", + "server": "local", + "model": "configured-model", + } + row.update(overrides) + return row + + +def _write(path: Path, *rows: dict[str, Any]) -> None: + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def _assert_refused(rc: int, capsys, detail: str) -> None: + assert rc == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "comparability cannot be established from the persisted identity" in captured.err + assert detail in captured.err + assert "aggregate success" not in captured.err + assert "A/B compare" not in captured.err + + +def test_single_summary_refuses_rows_mixing_batteries(tmp_path, capsys): + path = tmp_path / "mixed.jsonl" + _write(path, _row("R1", battery="battery-a"), _row("R2", battery="battery-b")) + + _assert_refused(report_mod.main([str(path)]), capsys, "rows disagree on battery") + + +def test_ab_report_refuses_battery_mismatch(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(battery="battery-a")) + _write(path_b, _row(battery="battery-b")) + + _assert_refused(report_mod.main([str(path_a), str(path_b)]), capsys, "battery differs across files") + + +def test_multi_surface_table_refuses_battery_mismatch(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(battery="battery-a")) + _write(path_b, _row(battery="battery-b")) + + rc = report_mod.main(["--table", str(path_a), str(path_b)]) + + _assert_refused(rc, capsys, "battery differs across files") + + +def test_vary_battery_is_a_usage_error(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "battery", str(path)]) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "--vary battery is not allowed" in captured.err + assert "measurement universe cannot be waived" in captured.err + + +def test_vary_all_is_a_usage_error(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "all", str(path)]) == 2 + assert "--vary has no 'all'" in capsys.readouterr().err + + +def test_vary_requires_each_dimension_to_be_named(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "provider,", str(path)]) == 2 + assert "--vary requires a dimension name" in capsys.readouterr().err + + +def test_requested_tier_cannot_be_declared_as_an_identity_dimension(tmp_path, capsys): + path = tmp_path / "a.jsonl" + _write(path, _row()) + + assert report_mod.main(["--vary", "requested_tier", str(path)]) == 2 + assert "unknown --vary dimension(s): requested_tier" in capsys.readouterr().err + + +def test_vary_resolved_model_prints_treatment_and_reports_normally(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(resolved_model="model-a", model="realized-a")) + _write(path_b, _row(resolved_model="model-b", model="realized-b")) + + rc = report_mod.main(["--vary", "resolved_model", str(path_a), str(path_b)]) + + assert rc == 0 + captured = capsys.readouterr() + assert "Treatment: resolved_model" in captured.out + assert "Realized model evidence:" in captured.out + assert "A/B compare:" in captured.out + assert captured.err == "" + + +def test_two_varied_dimensions_print_end_to_end_attribution_label(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(resolved_model="model-a", provider="anthropic")) + _write(path_b, _row(resolved_model="model-b", provider="openai")) + + rc = report_mod.main(["--vary", "provider,resolved_model", str(path_a), str(path_b)]) + + assert rc == 0 + output = capsys.readouterr().out + assert ( + "Treatment: resolved_model, provider — end-to-end comparison; effect not attributable to any single dimension" + ) in output + + +def test_header_row_disagreement_is_refused_before_latest_wins_dedupe(tmp_path, capsys): + path = tmp_path / "resume.jsonl" + _write( + path, + _meta(), + _row(resolved_model="conflicting-model"), + _row(resolved_model="configured-model"), + ) + + _assert_refused(report_mod.main([str(path)]), capsys, "raw rows on resolved_model") + + +def test_single_header_row_identity_disagreement_is_refused(tmp_path, capsys): + path = tmp_path / "integrity.jsonl" + _write(path, _meta(), _row(provider="openai")) + + _assert_refused(report_mod.main([str(path)]), capsys, "raw rows on provider") + + +def test_conflicting_meta_headers_are_refused(tmp_path, capsys): + path = tmp_path / "headers.jsonl" + _write(path, _meta(run_id="run-1"), _meta(run_id="run-2", driver="codex-cli")) + + _assert_refused(report_mod.main([str(path)]), capsys, "conflicting meta headers for driver") + + +def test_missing_identity_value_is_not_a_wildcard(tmp_path, capsys): + path_a = tmp_path / "legacy.jsonl" + path_b = tmp_path / "identified.jsonl" + _write(path_a, _row(battery="")) + _write(path_b, _row(battery="samebattery1")) + + _assert_refused(report_mod.main([str(path_a), str(path_b)]), capsys, "") + + +def test_requested_tier_difference_is_not_an_identity_mismatch(tmp_path, capsys): + path_a = tmp_path / "tier.jsonl" + path_b = tmp_path / "model-id.jsonl" + _write(path_a, _row(requested_model="standard", requested_tier="standard")) + _write(path_b, _row(requested_model="configured-model", requested_tier=None)) + + assert report_mod.main([str(path_a), str(path_b)]) == 0 + captured = capsys.readouterr() + assert "A/B compare:" in captured.out + assert captured.err == "" + + +def test_meta_configured_model_is_not_compared_to_row_realized_model(tmp_path, capsys): + path = tmp_path / "api.jsonl" + _write(path, _meta(model="configured-model"), _row(model="provider-reported-model")) + + assert report_mod.main([str(path)]) == 0 + captured = capsys.readouterr() + assert "Realized model evidence:" in captured.out + assert "provider-reported-model" in captured.out + assert captured.err == "" + + +def test_resume_rows_may_have_different_run_ids(tmp_path, capsys): + path = tmp_path / "resume.jsonl" + _write(path, _meta(run_id="original"), _row("R1", run_id="original"), _row("R2", run_id="resumed")) + + assert report_mod.main([str(path)]) == 0 + assert capsys.readouterr().err == "" + + +def test_unacknowledged_provider_mismatch_is_refused(tmp_path, capsys): + path_a = tmp_path / "a.jsonl" + path_b = tmp_path / "b.jsonl" + _write(path_a, _row(provider="anthropic")) + _write(path_b, _row(provider="openai")) + + _assert_refused(report_mod.main([str(path_a), str(path_b)]), capsys, "provider differs across files") + + +def test_realized_model_change_within_run_is_flagged_without_refusal(tmp_path, capsys): + path = tmp_path / "changed-model.jsonl" + _write(path, _row("R1", model="reported-a"), _row("R2", model="reported-b")) + + assert report_mod.main([str(path)]) == 0 + captured = capsys.readouterr() + assert "WARNING: realized model changed within" in captured.out + assert "reported-a,reported-b" in captured.out + assert captured.err == "" + + +def test_vary_driver_header_limits_claim_to_end_to_end_driver_question(tmp_path, capsys): + path_a = tmp_path / "api.jsonl" + path_b = tmp_path / "cli.jsonl" + _write(path_a, _row(driver="api")) + _write(path_b, _row(driver="codex-cli")) + + assert report_mod.main(["--vary", "driver", str(path_a), str(path_b)]) == 0 + output = capsys.readouterr().out + assert "end-to-end driver question; cannot support a surface-only claim" in output + + +def test_server_can_be_declared_as_a_treatment(tmp_path, capsys): + path_a = tmp_path / "local.jsonl" + path_b = tmp_path / "external.jsonl" + _write(path_a, _row(server="local")) + _write(path_b, _row(server="external")) + + assert report_mod.main(["--vary", "server", str(path_a), str(path_b)]) == 0 + assert "Treatment: server" in capsys.readouterr().out + + +def test_markdown_table_separates_identity_header_from_table(tmp_path, capsys): + path = tmp_path / "model.jsonl" + _write(path, _row()) + + assert report_mod.main(["--table", "--markdown", str(path)]) == 0 + output = capsys.readouterr().out + assert "Realized model evidence:" in output + assert "\n\n| task |" in output + + +def test_report_rejects_manifest_variation_within_one_result_file(tmp_path, capsys): + path = tmp_path / "manifest-changed.jsonl" + _write( + path, + _row("R1", tool_manifest_fingerprint="manifest-a"), + _row("R2", tool_manifest_fingerprint="manifest-b"), + ) + _assert_refused( + report_mod.main([str(path)]), + capsys, + "rows disagree on tool_manifest_fingerprint", + ) + + +def test_report_rejects_mixed_present_and_missing_manifests_within_one_file(tmp_path, capsys): + path = tmp_path / "partially-identified.jsonl" + _write( + path, + _row("R1", tool_manifest_fingerprint="manifest-a"), + _row("R2", tool_manifest_fingerprint=None), + ) + + _assert_refused( + report_mod.main([str(path)]), + capsys, + "", + ) + + +def test_report_identifies_but_does_not_refuse_different_tool_manifests(tmp_path, capsys): + path_a = tmp_path / "surface-a.jsonl" + path_b = tmp_path / "surface-b.jsonl" + _write(path_a, _row(tool_manifest_fingerprint="manifest-a")) + _write(path_b, _row(tool_manifest_fingerprint="manifest-b")) + + assert report_mod.main([str(path_a), str(path_b)]) == 0 + captured = capsys.readouterr() + assert "Tool manifest evidence:" in captured.out + assert "manifest-a" in captured.out + assert "manifest-b" in captured.out + assert captured.err == "" + + +def test_missing_manifest_observation_is_not_fatal(tmp_path, capsys): + path = tmp_path / "no-manifest.jsonl" + _write(path, _row(tool_manifest_fingerprint=None)) + + assert report_mod.main([str(path)]) == 0 + output = capsys.readouterr().out + assert "Summary:" in output + assert "TOOL MANIFEST ABSENT" not in output + + +def test_ab_and_table_refuse_when_any_tool_manifest_is_absent(tmp_path, capsys): + path_a = tmp_path / "surface-a.jsonl" + path_b = tmp_path / "surface-b.jsonl" + _write(path_a, _row(tool_manifest_fingerprint="manifest-a")) + _write(path_b, _row(tool_manifest_fingerprint=None)) + + _assert_refused( + report_mod.main([str(path_a), str(path_b)]), + capsys, + "tool_manifest_fingerprint is missing for comparison input(s)", + ) + + _assert_refused( + report_mod.main(["--table", str(path_a), str(path_b)]), + capsys, + "every compared surface must be identified", + ) + + _write(path_a, _row(tool_manifest_fingerprint=None)) + _assert_refused( + report_mod.main([str(path_a), str(path_b)]), + capsys, + "every compared surface must be identified", + ) + + +def test_exact_run_keys_name_missing_and_duplicate_rows_before_latest_wins(tmp_path, capsys): + path = tmp_path / "wrong-keys.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R2"], expected_reps=1), + _row("R1"), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 1 + output = capsys.readouterr().out + assert "RUN INCOMPLETE:" in output + assert "missing keys=[R2[rep=0]]" in output + assert "unexpected keys=[R1[rep=0]]" in output + + +def test_exact_run_keys_reject_duplicate_excess_even_when_every_expected_key_exists(tmp_path, capsys): + path = tmp_path / "duplicate-excess.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R2"], expected_reps=1), + _row("R1"), + _row("R2"), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 1 + output = capsys.readouterr().out + assert "RUN INCOMPLETE: 2/2 rows completed" in output + assert "unexpected keys=[R1[rep=0]]" in output + + +def test_exact_run_keys_accept_append_only_retry_history_when_only_last_row_is_terminal(tmp_path, capsys): + path = tmp_path / "retry-history.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R2"], expected_reps=1), + _row("R1", success=False, error="timeout", error_class="infra_cli"), + _row("R2"), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 0 + output = capsys.readouterr().out + assert "RUN COMPLETE: 2/2 rows completed" in output + + +def test_exact_run_keys_accept_declared_task_subset_with_all_repetitions(tmp_path, capsys): + path = tmp_path / "subset.jsonl" + _write( + path, + _meta(expected_rows=4, expected_task_ids=["R1", "W1"], expected_reps=2), + _row("R1", rep=0), + _row("R1", rep=1), + _row("W1", rep=0), + _row("W1", rep=1), + ) + + assert report_mod.main([str(path)]) == 0 + assert "RUN COMPLETE: 4/4 rows completed" in capsys.readouterr().out + + +def test_malformed_exact_run_expectation_is_refused_instead_of_treated_as_legacy(tmp_path, capsys): + path = tmp_path / "malformed-expectation.jsonl" + _write( + path, + _meta(expected_rows=2, expected_task_ids=["R1", "R1"], expected_reps=1), + _row("R1"), + ) + + assert report_mod.main([str(path)]) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "invalid run expectation" in captured.err + assert "expected_task_ids contains duplicates" in captured.err + + +def test_infra_error_rows_without_a_manifest_do_not_refuse_the_comparison(tmp_path: Path): + """A row that died in seeding never ran an agent, so it has no manifest to carry. + + The first live A/B was refused because six infra_seed rows (L2 and L5, identical on both + surfaces) had no tool_manifest_fingerprint — rows every statistic already excludes. Demanding + identity from a row that never reached the surface refuses sound comparisons. + """ + path = tmp_path / "rows.jsonl" + surface = { + "task_id": "R1", + "rep": 0, + "label": "local", + "battery": "b1", + "server": "local", + "driver": "codex-cli", + "provider": "openai", + "resolved_model": "m", + "tool_manifest_fingerprint": "fp-a", + "success": True, + } + seed_failure = { + "task_id": "L5", + "rep": 0, + "label": "local", + "battery": "b1", + "server": "local", + "driver": "codex-cli", + "provider": "openai", + "resolved_model": "m", + "error_class": "infra_seed", + "error": "boom", + } + path.write_text( + json.dumps(surface) + "\n" + json.dumps(seed_failure) + "\n", + encoding="utf-8", + ) + report = identity.validate_persisted_identity([path]) + assert report.files[0].values[identity.TOOL_MANIFEST_FIELD] == "fp-a" + + +def test_expected_skip_rows_without_a_manifest_do_not_refuse_the_comparison(tmp_path: Path): + """A plan-gated skip ends before the agent starts, so it carries no manifest either. + + Every report command exited 2 on any file mixing one such skip with evaluated rows, even + though summary semantics count an expected skip as a complete row. + """ + path = tmp_path / "rows.jsonl" + common = { + "rep": 0, + "label": "local", + "battery": "b1", + "server": "local", + "driver": "codex-cli", + "provider": "openai", + "resolved_model": "m", + } + surface = {**common, "task_id": "R1", "tool_manifest_fingerprint": "fp-a", "success": True} + plan_gated = {**common, "task_id": "L4", "skipped": "env:plan-gated:customers"} + path.write_text(json.dumps(surface) + "\n" + json.dumps(plan_gated) + "\n", encoding="utf-8") + + report = identity.validate_persisted_identity([path]) + assert report.files[0].values[identity.TOOL_MANIFEST_FIELD] == "fp-a" diff --git a/tests/evals/report/test_load.py b/tests/evals/report/test_load.py new file mode 100644 index 0000000..fa9cc07 --- /dev/null +++ b/tests/evals/report/test_load.py @@ -0,0 +1,155 @@ +"""Offline eval tests for load.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.report import ( + dedupe_rows_latest, + is_infra_error_row, + load_rows, + summarize, +) +from tests.evals.conftest import case_params + + +def test_is_infra_error_row_covers_infrastructure_prefix(): + assert is_infra_error_row({"error_class": "infra_cli"}) is True + assert is_infra_error_row({"error_class": "task"}) is False + + +def _load_rows_dedupe_latest_wins(tmp_path, _capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True, "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False, "num_calls": 9}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p) # default dedupe=latest + assert len(loaded) == 1 + assert loaded[0].num_calls == 9 + assert loaded[0].success is False + + +def _load_rows_no_dedupe_warns_on_duplicate_keys(tmp_path, capsys): + p = tmp_path / "dup.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "success": True}, + {"task_id": "R1", "rep": 0, "label": "local", "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + loaded = load_rows(p, dedupe="none") + assert len(loaded) == 2 + err = capsys.readouterr().err + assert "duplicate" in err + assert "R1" in err + + +def _load_rows_skips_meta_and_surfaces_missing_task_id(tmp_path, capsys): + p = tmp_path / "r.jsonl" + lines = [ + json.dumps( + { + "row_type": "meta", + "run_id": "abc", + "label": "candidate", + "battery": "deadbeef0001", + "model": "sonnet", + "driver": "claude-cli", + "git_sha": "x", + "ts": "t", + } + ), + json.dumps({"label": "candidate", "rep": 0, "success": True}), # no task_id + json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True, "num_calls": 2}), + ] + p.write_text("\n".join(lines) + "\n", encoding="utf-8") + rows = load_rows(p) + assert len(rows) == 2 + assert rows[0].error_class == "harness_report_load" + assert rows[1].task_id == "R1" + assert "recording result without task_id as a harness error" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "case", + case_params( + _load_rows_dedupe_latest_wins, + _load_rows_no_dedupe_warns_on_duplicate_keys, + _load_rows_skips_meta_and_surfaces_missing_task_id, + ), +) +def test_load_behaviours(case, tmp_path, capsys): + case(tmp_path, capsys) + + +def test_schema_v0_rows_parse_with_unknown_trace_integrity(): + """Synthetic schema-0 rows keep data but do not claim verified traces.""" + fixture = Path(__file__).parents[2] / "fixtures" / "evals_schema_v0_rows.jsonl" + rows = load_rows(fixture) + + assert [row.schema_version for row in rows] == [0, 0] + by_task = {row.task_id: row for row in rows} + release_row = by_task["L3"] + assert release_row.final_text == "" + assert release_row.result_tokens_estimated is None + assert release_row.calls[0].result_tokens is None + assert release_row.calls[0].action == "create" + + count_row = by_task["R2"] + assert count_row.final_text.endswith("\n4") + assert count_row.result_tokens_estimated is True + assert [call.result_tokens for call in count_row.calls] == [315, 64] + assert release_row.trace_integrity is None + assert count_row.trace_integrity is None + + summary = summarize(rows) + assert summary.tasks["L3"].success == "1/1" + assert summary.tasks["L3"].med_calls is None + assert summary.tasks["L3"].result_tokens_mode == "unavailable" + assert summary.tasks["R2"].success == "1/1" + assert summary.tasks["R2"].med_calls is None + assert summary.tasks["R2"].result_tokens_mode == "unavailable" + + +def test_dedupe_rows_latest_pure(): + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 1}, + {"task_id": "R1", "rep": 0, "label": "local", "num_calls": 5}, + {"task_id": "R2", "rep": 0, "label": "local", "num_calls": 3}, + ] + out = dedupe_rows_latest(rows) + assert len(out) == 2 + by_id = {r.task_id: r for r in out} + assert by_id["R1"].num_calls == 5 + assert by_id["R2"].num_calls == 3 + + +def test_malformed_rows_surface_as_completeness_errors(tmp_path, capsys): + path = tmp_path / "malformed.jsonl" + path.write_text( + "\n".join( + [ + json.dumps({"task_id": "R1", "success": True, "calls": []}), + "{not-json", + json.dumps(["not", "an", "object"]), + ] + ) + + "\n", + encoding="utf-8", + ) + + rows = load_rows(path) + summary = summarize(rows, expected_rows=3) + + assert len(rows) == 3 + assert summary.aggregate_n == 1 + assert summary.harness_errors == 2 + assert summary.complete is False + assert {row.error_class for row in rows if row.error} == {"harness_report_load"} + warnings = capsys.readouterr().err + assert "recording invalid JSON as a harness error" in warnings + assert "recording non-object JSON as a harness error" in warnings diff --git a/tests/evals/report/test_lookup_reuse.py b/tests/evals/report/test_lookup_reuse.py new file mode 100644 index 0000000..f2e6bc1 --- /dev/null +++ b/tests/evals/report/test_lookup_reuse.py @@ -0,0 +1,88 @@ +"""Re-hunting an entity whose id is already in hand is a surface property. + +The sharpest finding of the 2026-08-24 pair was workitem.search 111 vs 26 against +workitem.retrieve 4 vs 20: one arm carried resolved ids across turns and the other +re-searched for them. A surface with stickier identifiers would close that gap with +no change to either agent. +""" + +from __future__ import annotations + +from evals.report.lookup_reuse import measure_lookup_reuse + +WORKITEM_ID = "0cf779b6-9e18-4209-9e09-2264b889be42" + + +def call(tool, action, **args): + import json + + return {"tool": tool, "action": action, "args_json": json.dumps(args)} + + +def row(*calls, task_id="R1", rep=0): + return { + "task_id": task_id, + "rep": rep, + "success": True, + "trace_integrity": True, + "num_calls": len(calls), + "calls": list(calls), + } + + +def test_searching_for_something_already_in_hand_is_flagged(): + measurement = measure_lookup_reuse( + [row(call("workitem", "retrieve", workitem_id=WORKITEM_ID), call("workitem", "search", query="thing"))] + ) + assert measurement.total == 1 + assert measurement.by_resource["workitem"] == 1 + + +def test_searching_before_any_id_is_known_is_not_flagged(): + """The first lookup is how the id is obtained; charging for it would be wrong.""" + measurement = measure_lookup_reuse( + [row(call("workitem", "search", query="thing"), call("workitem", "retrieve", workitem_id=WORKITEM_ID))] + ) + assert measurement.total == 0 + + +def test_a_search_on_a_different_resource_is_not_flagged(): + measurement = measure_lookup_reuse( + [row(call("workitem", "retrieve", workitem_id=WORKITEM_ID), call("cycle", "list", project_id="p1"))] + ) + assert measurement.total == 0 + + +def test_ids_do_not_carry_across_rows(): + """Each repetition is a fresh conversation; nothing is in hand at its start.""" + first = row(call("workitem", "retrieve", workitem_id=WORKITEM_ID), task_id="R1", rep=0) + second = row(call("workitem", "search", query="thing"), task_id="R1", rep=1) + assert measure_lookup_reuse([first, second]).total == 0 + + +def test_a_run_without_recorded_arguments_reports_that_it_cannot_tell(): + """Zero must never be reported when the question could not be asked.""" + blind = { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [{"tool": "workitem", "action": "search"}, {"tool": "workitem", "action": "retrieve"}], + } + measurement = measure_lookup_reuse([blind]) + assert measurement.rows_without_args == 1 + assert measurement.measurable is False + assert "not measured" in measurement.statement() + + +def test_repeat_searches_after_the_id_is_known_each_count(): + measurement = measure_lookup_reuse( + [ + row( + call("workitem", "retrieve", workitem_id=WORKITEM_ID), + call("workitem", "search", query="a"), + call("workitem", "search", query="b"), + ) + ] + ) + assert measurement.total == 2 diff --git a/tests/evals/report/test_off_surface.py b/tests/evals/report/test_off_surface.py new file mode 100644 index 0000000..bc58950 --- /dev/null +++ b/tests/evals/report/test_off_surface.py @@ -0,0 +1,154 @@ +"""Offline tests for off-surface trace-signature indicators.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import RESULT_SCHEMA_VERSION +from evals.report import ( + ANSWER_WITHOUT_PROVENANCE, + IMPLAUSIBLY_FEW_CALLS, + WRITE_WITHOUT_WRITE_CALL, + ZERO_CALL_SUCCESS, + ab_compare, + measure_off_surface, + print_ab_report, + print_table, + summarize, +) + + +def _row( + task_id: str, + *, + rep: int = 0, + success: bool = True, + num_calls: int = 1, + calls: list[dict[str, Any]] | None = None, + evidence_trace_available: bool = False, + verify_note: str = "", +) -> dict[str, Any]: + return { + "schema_version": RESULT_SCHEMA_VERSION, + "task_id": task_id, + "rep": rep, + "success": success, + "num_calls": num_calls, + "calls": list(calls or []), + "trace_integrity": True, + "evidence_trace_available": evidence_trace_available, + "verify_note": verify_note, + } + + +def test_zero_call_success_flags_synthetic_bypass_and_clean_row(): + bypass = _row("R1", rep=0, success=True, num_calls=0, calls=[]) + clean = _row("R1", rep=1, success=True, calls=[{"tool": "list_work_items"}]) + trace_invalid = {**_row("R1", rep=2, success=True, num_calls=0, calls=[]), "trace_integrity": False} + + measurement = measure_off_surface([bypass, clean, trace_invalid]) + + assert measurement.addresses(ZERO_CALL_SUCCESS) == ("R1[rep=0]",) + assert all(row.address != "R1[rep=1]" for row in measurement.rows) + assert all(row.address != "R1[rep=2]" for row in measurement.rows) + + # Indicators are measurements, not another pass/fail or completeness policy. + summary = summarize([bypass], expected_rows=1) + assert summary.aggregate_k == summary.aggregate_n == 1 + assert summary.complete is True + assert summary.off_surface.addresses(ZERO_CALL_SUCCESS) == ("R1[rep=0]",) + + +def test_write_without_write_call_uses_catalog_tags(): + suspicious_write = _row("W1", rep=0, calls=[{"tool": "list_labels"}]) + clean_write = _row("W1", rep=1, calls=[{"tool": "manage_work_item_label"}]) + clean_setup = _row("S1", rep=0, calls=[{"tool": "create_work_item_property"}]) + future_setup = _row("FUTURE", rep=0, calls=[{"tool": "list_work_items"}]) + + measurement = measure_off_surface( + [suspicious_write, clean_write, clean_setup, future_setup], + task_catalog={ + "W1": {"tags": {"write"}}, + "S1": {"tags": {"setup"}}, + "FUTURE": {"tags": {"setup"}}, + }, + ) + + assert measurement.addresses(WRITE_WITHOUT_WRITE_CALL) == ("W1[rep=0]", "FUTURE[rep=0]") + + +def test_answer_without_provenance_counts_correct_answer(): + missing = _row( + "R1", + rep=0, + success=False, # Existing provenance enforcement already fails this row. + calls=[{"tool": "retrieve_work_item", "observed_sentinels": []}], + evidence_trace_available=True, + verify_note="answer_correct=true (seed state reported); provenance=missing", + ) + clean = _row( + "R1", + rep=1, + calls=[ + { + "tool": "retrieve_work_item", + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + evidence_trace_available=True, + verify_note="answer_correct=true (seed state reported); provenance=observed", + ) + + measurement = measure_off_surface([missing, clean]) + + assert measurement.addresses(ANSWER_WITHOUT_PROVENANCE) == ("R1[rep=0]",) + + +def test_implausibly_few_calls_uses_observed_outer_fence(): + rows = [ + _row( + "R2", + rep=rep, + num_calls=count, + calls=[{"tool": "list_work_items"}] * count, + ) + for rep, count in enumerate([10, 10, 10, 10, 1]) + ] + + measurement = measure_off_surface(rows) + + assert measurement.addresses(IMPLAUSIBLY_FEW_CALLS) == ("R2[rep=4]",) + assert measure_off_surface(rows[:4]).addresses(IMPLAUSIBLY_FEW_CALLS) == () + benign_dispersion = [ + _row("R2", rep=rep, num_calls=count, calls=[{"tool": "list_work_items"}] * count) + for rep, count in enumerate([3, 3, 3, 3, 2]) + ] + assert measure_off_surface(benign_dispersion).addresses(IMPLAUSIBLY_FEW_CALLS) == () + + +def test_reports_print_explicit_zero_addresses_rule_and_limitation(capsys): + clean = _row("R1", calls=[{"tool": "list_work_items"}]) + print_table(summarize([clean]), "single") + single_output = capsys.readouterr().out + + assert "EXECUTION COVERAGE:" in single_output + assert "off-surface indicators: 0" in single_output + assert "zero-call success: 0" in single_output + assert "calls < Q1 - 3×IQR and calls ≤ half the task median" in single_output + assert "cannot detect an agent" in single_output + assert "RUN COMPLETE:" in single_output + + bypass = _row("W1", rep=3, num_calls=0, calls=[]) + # Mutation intent is a fact about the run, so the file carries it. Without the header a + # hand-built row has no tags and the write indicator is correctly silent. + write_meta = {"row_type": "meta", "task_metadata": {"W1": {"tags": ["write"]}}} + comparison = ab_compare([write_meta, clean], [write_meta, bypass]) + print_ab_report(comparison, Path("a.jsonl"), Path("b.jsonl")) + ab_output = capsys.readouterr().out + + assert "A off-surface indicators: 0" in ab_output + assert "B off-surface indicators: 1 flagged rows" in ab_output + assert "B zero-call success: 1 [W1[rep=3]]" in ab_output + assert "B write without a write call: 1 [W1[rep=3]]" in ab_output diff --git a/tests/evals/report/test_power.py b/tests/evals/report/test_power.py new file mode 100644 index 0000000..387030e --- /dev/null +++ b/tests/evals/report/test_power.py @@ -0,0 +1,67 @@ +"""At low rep counts the per-task verdicts support nothing; the report must say so. + +At 2 reps every mixed task is `1/2 UNSTABLE` with a 95% interval of [0.09, 0.91], +which is compatible with almost any true rate -- while the paired aggregate over 35 +tasks is well powered. That asymmetry is easy to misread, and was misread. +""" + +from __future__ import annotations + +from evals.report import summarize +from evals.report.power import UNDERPOWERED_REPS, power_statement + + +def rows_at(reps: int, tasks: int = 3) -> list[dict]: + return [ + { + "task_id": f"T{task}", + "rep": rep, + "success": rep % 2 == 0, + "trace_integrity": True, + "num_calls": 1, + "calls": [], + } + for task in range(tasks) + for rep in range(reps) + ] + + +def test_the_guardrail_appears_at_two_reps(): + statement = power_statement(summarize(rows_at(2))) + assert statement is not None + assert "per-task" in statement + assert "aggregate" in statement + + +def test_the_guardrail_is_absent_at_five_reps(): + assert power_statement(summarize(rows_at(UNDERPOWERED_REPS))) is None + + +def test_one_deep_task_does_not_silence_the_caveat_for_the_shallow_ones(): + """Keying off the best-covered task left a mixed run's shallow tasks uncaveated. + + That is the opposite of the intended failure, so the line is scoped to the tasks + that are actually shallow and names how many they are. + """ + rows = rows_at(2, tasks=2) + [ + {"task_id": "T9", "rep": rep, "success": True, "trace_integrity": True, "num_calls": 1, "calls": []} + for rep in range(UNDERPOWERED_REPS) + ] + statement = power_statement(summarize(rows)) + assert statement is not None + assert "2 of 3" in statement + + +def test_the_guardrail_names_the_rep_count_it_saw(): + assert "fewest 2" in (power_statement(summarize(rows_at(2))) or "") + + +def test_no_evaluated_rows_produces_no_claim(): + assert power_statement(summarize([])) is None + + +def test_it_reaches_the_printed_report(capsys): + import evals.report as report_mod + + report_mod.print_table(summarize(rows_at(2)), "Summary: low-power.jsonl") + assert "per-task" in capsys.readouterr().out diff --git a/tests/evals/report/test_review_round2.py b/tests/evals/report/test_review_round2.py new file mode 100644 index 0000000..4d427dc --- /dev/null +++ b/tests/evals/report/test_review_round2.py @@ -0,0 +1,222 @@ +"""Regressions from the second adversarial review. + +Ten findings. The two that mattered most were a metric measuring something other +than what it claimed, and a field documented as "ids and short strings" that had no +size bound at all. +""" + +from __future__ import annotations + +import json + +from evals.core.failure_kind import ABANDONED, MISSING_WRITE, PARTIAL_WRITE, UNPROVEN, WRONG_VALUE, classify_failure +from evals.core.pricing import price_usage +from evals.core.results import AgentRun, Usage, agent_run_to_task_result +from evals.report import summarize +from evals.report.economics import measure_economics +from evals.report.lookup_reuse import measure_lookup_reuse +from evals.report.power import power_statement + + +def call(tool, action, **args): + return {"tool": tool, "action": action, "args_json": json.dumps(args)} + + +def task_row(*calls, **overrides): + row = { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "num_calls": len(calls), + "calls": list(calls), + } + row.update(overrides) + return row + + +# --- F1: the lookup rule must establish reuse of the same entity ------------------ + + +def test_f1_pagination_is_not_a_redundant_lookup(): + assert ( + measure_lookup_reuse( + [ + task_row( + call("workitem", "list", project_id="p", cursor="a"), + call("workitem", "list", project_id="p", cursor="b"), + ) + ] + ).total + == 0 + ) + + +def test_f1_a_list_after_a_create_is_not_a_redundant_lookup(): + assert ( + measure_lookup_reuse( + [task_row(call("workitem", "create", project_id="p", name="x"), call("workitem", "list", project_id="p"))] + ).total + == 0 + ) + + +def test_f1_a_scope_id_does_not_put_the_entity_in_hand(): + """project_id says which project to look in, not which work item is known.""" + assert ( + measure_lookup_reuse( + [task_row(call("workitem", "retrieve", project_id="p"), call("workitem", "search", query="z"))] + ).total + == 0 + ) + + +def test_f1_the_entitys_own_id_still_counts(): + assert ( + measure_lookup_reuse( + [task_row(call("workitem", "retrieve", workitem_id="wi-1"), call("workitem", "search", query="z"))] + ).total + == 1 + ) + + +# --- F2: recorded arguments need a size bound ------------------------------------ + + +def test_f2_a_huge_argument_value_is_bounded(): + run = AgentRun( + calls=[{"tool": "workitem", "args": {"action": "update", "description_html": "x" * 1_000_000}}], + final_text="", + usage=Usage(), + stopped_reason="end_turn", + call_source="api", + ) + args_json = agent_run_to_task_result(run).calls[0].args_json + assert args_json is not None + assert len(args_json) < 5_000, f"args_json was {len(args_json)} bytes" + parsed = json.loads(args_json) + # Structure and the short discriminating values survive; only the bulk is cut. + assert parsed["action"] == "update" + assert parsed["description_html"].endswith("…[truncated]") + + +def test_f2_short_arguments_are_untouched(): + args = {"action": "retrieve", "workitem_id": "wi-42"} + run = AgentRun( + calls=[{"tool": "workitem", "args": args}], + final_text="", + usage=Usage(), + stopped_reason="end_turn", + call_source="api", + ) + assert json.loads(agent_run_to_task_result(run).calls[0].args_json) == args + + +# --- F3: charged rows the runner later marked infrastructure --------------------- + + +def test_f3_an_infra_terminated_row_that_burned_tokens_is_still_charged(): + usage = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 0, "source": "iterations"} + rows = [task_row(model="gpt-5.6-luna", usage_total=usage, error="infra_cli timeout", error_class="infra_cli")] + measurement = measure_economics(rows) + assert measurement.priced_rows == 1 + assert measurement.cost_usd is not None + + +def test_f3_a_vendor_only_cost_is_not_thrown_away(): + """Tokens unmeasured, but the vendor still said what it charged.""" + cost = price_usage({"source": "modelUsage", "total_cost_usd": 1.25}, model="haiku") + assert cost.vendor_usd == 1.25 + assert cost.billed_usd == 1.25 + + +# --- F5/F6: classifier precedence ------------------------------------------------- + + +def test_f5_a_wrong_answer_quoting_the_true_marker_is_not_unproven(): + note = "answer_correct=false (values=[\"answer_correct=true\"]; want ['x'])" + assert classify_failure(note) == WRONG_VALUE + + +def test_f6_a_capped_run_that_also_wrote_the_wrong_value_is_not_a_non_defect(): + """A cap can coexist with a proven wrong write; abandoned would excuse it.""" + assert classify_failure("answer_correct=false (values=['a']; want ['b'])", hit_max_iterations=True) == WRONG_VALUE + + +def test_f6_a_capped_run_with_an_unfinished_state_is_still_abandoned(): + assert classify_failure("estimate points missing fib subset", hit_max_iterations=True) == ABANDONED + + +def test_f6_linked_counts_as_a_present_marker(): + assert classify_failure("request 'x' missing; R1 item ABC-1 linked") == PARTIAL_WRITE + + +def test_f6_absence_phrasings_from_real_verifiers_are_recognised(): + for note in ("no comments on target item", "no 120-minute work log", "2 module items not archived"): + assert classify_failure(note) == MISSING_WRITE, note + + +def test_unproven_still_works(): + assert classify_failure("answer_correct=true (...); provenance=missing (0 evidence-bearing)") == UNPROVEN + + +# --- F7: the power line must not be silenced by one well-covered task ------------- + + +def test_f7_underpowered_tasks_are_still_named_when_another_task_is_deep(): + rows = [task_row(task_id="T1", rep=rep, success=True, calls=[]) for rep in range(2)] + [ + task_row(task_id="T2", rep=rep, success=True, calls=[]) for rep in range(5) + ] + statement = power_statement(summarize(rows)) + assert statement is not None + assert "1 of 2" in statement + + +def test_f7_no_line_when_every_task_is_deep_enough(): + rows = [task_row(task_id="T1", rep=rep, success=True, calls=[]) for rep in range(5)] + assert power_statement(summarize(rows)) is None + + +# --- F8/F10: the drift cohort and its formatting --------------------------------- + + +def test_f8_drift_uses_only_rows_carrying_both_figures(): + both = task_row( + model="haiku", + usage_total={ + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 0.0753878, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + }, + ) + computed_only = task_row( + task_id="R2", + model="gpt-5.6-luna", + usage_total={"input_tokens": 900_000, "output_tokens": 5000, "source": "iterations"}, + ) + measurement = measure_economics([both, computed_only]) + assert measurement.drift_rows == 1 + # Without cohort alignment the second row's cost would inflate the drift. + assert abs(measurement.cost_drift_usd) < 0.01 + + +def test_f10_a_sub_cent_drift_line_does_not_read_as_all_zeroes(): + tiny = task_row( + model="haiku", + usage_total={ + "input_tokens": 100, + "output_tokens": 1, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 100, + "total_cost_usd": 0.00032, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + }, + ) + from evals.report.economics import economics_statement + + line = economics_statement(measure_economics([tiny])) + assert "$0.000;" not in line and "$0.000 " not in line diff --git a/tests/evals/report/test_schema_friction.py b/tests/evals/report/test_schema_friction.py new file mode 100644 index 0000000..3750060 --- /dev/null +++ b/tests/evals/report/test_schema_friction.py @@ -0,0 +1,90 @@ +"""Offline tests for success-conditioned MCP error measurements.""" + +from __future__ import annotations + +import pytest + +from evals.report import measure_schema_friction, schema_friction_statement + + +def _row(task_id: str, **overrides): + row = { + "task_id": task_id, + "rep": 0, + "success": True, + "trace_integrity": True, + "num_calls": 4, + "errored_calls": 1, + "calls": [], + } + row.update(overrides) + return row + + +def test_schema_friction_uses_exact_successful_call_delta_population(): + measurement = measure_schema_friction( + [ + _row("R1", errored_calls=2), + _row("R2", success=False, errored_calls=20), + _row("R3", trace_integrity=False, errored_calls=30), + _row("R4", error="harness failed", errored_calls=40), + _row("R5", error_class="infra_cli", errored_calls=50), + _row("R6", skipped="env:plan-gated:feature", errored_calls=60), + { + "row_type": "meta", + "expected_rows": 6, + "success": True, + "trace_integrity": True, + "num_calls": 4, + "errored_calls": 70, + }, + ] + ) + + assert list(measurement.tasks) == ["R1"] + assert measurement.tasks["R1"].errored_calls == 2 + assert measurement.tasks["R1"].total_calls == 4 + assert measurement.task_mean_errored_calls == 2.0 + assert measurement.task_mean_errored_call_rate == 0.5 + + +def test_schema_friction_rate_is_task_mean_instead_of_pooled_call_rate(): + measurement = measure_schema_friction( + [ + _row("R1", num_calls=1, errored_calls=1), + _row("R2", num_calls=9, errored_calls=0), + ] + ) + + assert measurement.task_mean_errored_calls == 0.5 + assert measurement.task_mean_errored_call_rate == 0.5 + assert measurement.task_mean_errored_call_rate != pytest.approx(1 / 10) + + +def test_schema_friction_absolute_count_is_per_task_median_across_repetitions(): + measurement = measure_schema_friction( + [ + _row("R1", rep=0, num_calls=10, errored_calls=0), + _row("R1", rep=1, num_calls=10, errored_calls=0), + _row("R1", rep=2, num_calls=10, errored_calls=9), + ] + ) + + assert measurement.tasks["R1"].median_errored_calls == 0.0 + assert measurement.tasks["R1"].errored_calls == 9 + assert measurement.tasks["R1"].errored_call_rate == pytest.approx(0.3) + + +def test_schema_friction_prints_zero_and_does_not_invent_zero_attempt_rate(): + statement = schema_friction_statement(measure_schema_friction([_row("R1", num_calls=0, errored_calls=0)])) + + assert "task-mean median errored calls=0.0 across 1 tasks" in statement + assert "task-mean errored-call rate=n/a across 0 tasks with calls" in statement + assert "errored-call tasks: 0/1 []" in statement + assert "is_error is the MCP-level error flag" in statement + assert "correct task outcome still contributes" in statement + assert "wrong tool successfully does not" in statement + + no_data = schema_friction_statement(measure_schema_friction([])) + assert "task-mean median errored calls=n/a across 0 tasks" in no_data + assert "errored-call tasks: 0/0 []" in no_data diff --git a/tests/evals/report/test_summary.py b/tests/evals/report/test_summary.py new file mode 100644 index 0000000..02fc016 --- /dev/null +++ b/tests/evals/report/test_summary.py @@ -0,0 +1,432 @@ +"""Offline eval tests for summary.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals import report as report_mod +from evals.core.results import RESULT_SCHEMA_VERSION +from evals.report import ( + completeness_statement, + execution_coverage_statement, + format_tool_distribution, + format_tool_variability, + is_infra_error_row, + load_rows, + summarize, + wilson_interval, +) +from tests.evals.conftest import case_params + + +def test_completeness_is_independent_from_success_rate(): + complete = summarize( + [ + {"task_id": "R1", "success": True, "calls": []}, + {"task_id": "R2", "success": False, "calls": []}, + {"task_id": "L4", "skipped": "env:plan-gated:customers", "calls": []}, + ], + expected_rows=3, + ) + assert complete.aggregate_k == 1 + assert complete.aggregate_n == 2 + assert complete.completed_rows == 3 + assert complete.expected_skips == 1 + assert complete.complete is True + assert completeness_statement(complete).startswith("RUN COMPLETE:") + assert execution_coverage_statement(complete) == ( + "EXECUTION COVERAGE: 2/3 rows evaluated (66.7%); skipped tasks=[L4 (env:plan-gated:customers)]" + ) + + missing_worker = summarize( + [{"task_id": "L2", "skipped": "env:no-activity-worker", "calls": []}], + expected_rows=1, + ) + assert missing_worker.aggregate_n == 0 + assert missing_worker.completed_rows == 1 + assert missing_worker.expected_skips == 1 + assert missing_worker.expected_skip_reasons == {"no-activity-worker": 1} + assert missing_worker.complete is True + assert completeness_statement(missing_worker).startswith("RUN COMPLETE:") + + verifier_crash = summarize( + [ + {"task_id": "R1", "success": True, "calls": []}, + {"task_id": "W6", "error": "RuntimeError: verifier broke", "error_class": "task", "calls": []}, + ], + expected_rows=2, + ) + assert verifier_crash.aggregate_k == 1 + assert verifier_crash.aggregate_n == 1 + assert verifier_crash.harness_errors == 1 + assert verifier_crash.complete is False + assert completeness_statement(verifier_crash).startswith("RUN INCOMPLETE:") + + collisions = summarize( + [ + {"task_id": "R1", "skipped": "env:fixture-collision:customers:Acme", "calls": []}, + {"task_id": "R2", "skipped": "env:fixture-collision:release_tags:eval-rc1", "calls": []}, + ], + expected_rows=2, + ) + assert collisions.completed_rows == 0 + assert collisions.unexpected_skips == 2 + assert collisions.unexpected_skip_reasons == {"fixture-collision": 2} + assert collisions.complete is False + + unknown = summarize( + [{"task_id": "L2", "skipped": "env:new-reason", "calls": []}], + expected_rows=1, + ) + assert unknown.unexpected_skips == 1 + assert unknown.unexpected_skip_reasons == {"env:new-reason": 1} + assert unknown.complete is False + + cleanup = summarize( + [{"task_id": "R1", "success": True, "cleanup_error": "RuntimeError: delete failed", "calls": []}], + expected_rows=1, + ) + assert cleanup.aggregate_k == cleanup.aggregate_n == 1 + assert cleanup.cleanup_errors == 1 + assert cleanup.complete is False + + +def test_result_pair_mismatch_preserves_outcome_but_excludes_trace_metrics(): + summary = summarize( + [ + { + "task_id": "R1", + "success": True, + "trace_integrity": False, + "trace_integrity_reason": "result_pair_mismatch", + "result_pair_mismatch": True, + "num_calls": 99, + "calls": [{"tool": "untrustworthy", "result_tokens": 200}], + } + ], + expected_rows=1, + ) + + assert summary.complete is False + assert summary.trace_invalid_rows == 1 + assert summary.aggregate_k == summary.aggregate_n == 1 + assert summary.tasks["R1"].med_calls is None + assert summary.tasks["R1"].tool_reps == 0 + assert summary.tasks["R1"].med_result_tokens is None + + +def test_missing_trace_integrity_is_unknown_and_current_schema_run_is_incomplete(): + row = { + "schema_version": RESULT_SCHEMA_VERSION, + "task_id": "R1", + "success": True, + "num_calls": 7, + "calls": [{"tool": "unverified", "result_tokens": 99}], + } + + summary = summarize([row], expected_rows=1) + + assert summary.complete is False + assert summary.trace_invalid_rows == 1 + assert summary.tasks["R1"].med_calls is None + assert summary.tasks["R1"].tool_reps == 0 + assert summary.tasks["R1"].med_result_tokens is None + + +def _summarize_excludes_infra_errors_from_success(): + rows = [ + { + "task_id": "R1", + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [], + "error": None, + }, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "HttpError: 409", + "error_class": "infra_seed", + }, + { + "task_id": "R1", + "success": False, + "num_calls": 0, + "calls": [], + "error": "timeout after 120s", + "error_class": "infra_cli", + }, + {"task_id": "R1", "success": False, "num_calls": 3, "calls": [], "error": None}, + ] + summary = summarize(rows) + assert summary.infra_errors == 2 + assert summary.tasks["R1"].n == 2 # only non-infra, non-error rows + assert summary.tasks["R1"].k == 1 + assert summary.tasks["R1"].success == "1/2" + assert summary.tasks["R1"].infra_err == 2 + assert summary.tasks["R1"].failed_tool_reps == 3 + assert "failed excluded=3" in format_tool_distribution(summary.tasks["R1"]) + assert is_infra_error_row(rows[1]) is True + assert is_infra_error_row(rows[0]) is False + + +def _summarize_aggregate_wilson_and_call_variance(): + rows = [ + {"task_id": "R1", "rep": 0, "success": True, "trace_integrity": True, "num_calls": 2, "calls": []}, + {"task_id": "R1", "rep": 1, "success": True, "trace_integrity": True, "num_calls": 4, "calls": []}, + {"task_id": "R1", "rep": 2, "success": False, "trace_integrity": True, "num_calls": 6, "calls": []}, + {"task_id": "R2", "rep": 0, "success": True, "trace_integrity": True, "num_calls": 1, "calls": []}, + ] + s = summarize(rows) + assert s.tasks["R1"].n == 3 + assert s.tasks["R1"].k == 2 + assert s.tasks["R1"].calls_min == 2.0 + assert s.tasks["R1"].calls_q1 == 2.5 + assert s.tasks["R1"].med_calls == 3.0 + assert s.tasks["R1"].calls_q3 == 3.5 + assert s.tasks["R1"].calls_max == 4.0 + assert s.tasks["R1"].unstable is True + assert s.tasks["R2"].unstable is False + assert s.aggregate_k == 3 + assert s.aggregate_n == 4 + assert s.multi_rep is True + assert s.unstable_task_ids == ["R1"] + assert s.unstable_tasks == 1 + assert 0.0 <= s.aggregate_wilson_lo <= s.aggregate_wilson_hi <= 1.0 + + +def _tool_distribution_uses_successful_repetitions(): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "trace_integrity": True, + "num_calls": 3, + "calls": [{"tool": "a"}, {"tool": "a"}, {"tool": "b"}], + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [{"tool": "a"}, {"tool": "c"}], + }, + { + "task_id": "R1", + "rep": 2, + "success": False, + "trace_integrity": True, + "num_calls": 1, + "calls": [{"tool": "failed_only"}], + }, + { + "task_id": "R2", + "rep": 0, + "success": True, + "trace_integrity": True, + "num_calls": 1, + "calls": [{"tool": "one_rep"}], + }, + { + "task_id": "R3", + "rep": 0, + "success": False, + "trace_integrity": True, + "num_calls": 1, + "calls": [{"tool": "failed_only"}], + }, + {"task_id": "R4", "rep": 0, "skipped": "unavailable", "calls": []}, + ] + + summary = summarize(rows) + r1 = summary.tasks["R1"] + assert r1.calls_min == 2.0 # failed one-call repetition is not an observed successful floor + assert r1.calls_q1 == 2.25 + assert r1.med_calls == 2.5 + assert r1.calls_q3 == 2.75 + assert r1.calls_max == 3.0 + assert r1.calls_min <= r1.calls_q1 <= r1.med_calls <= r1.calls_q3 <= r1.calls_max + assert r1.tool_reps == 2 + assert r1.failed_tool_reps == 1 + assert r1.tool_rep_frequency == {"a": 1.0, "b": 0.5, "c": 0.5} + assert r1.tool_call_counts == {"a": 3, "b": 1, "c": 1} + assert "failed_only" not in r1.tool_call_counts + assert r1.variable_tool_names == ["b", "c"] + assert summary.variable_tool_tasks == 1 + assert summary.total_tasks == 4 + assert format_tool_variability(summary) == "1/4 tasks" + r1_distribution = format_tool_distribution(r1) + assert "success-only n=2; failed excluded=1" in r1_distribution + assert "core:a(3c)" in r1_distribution + assert "variable:b=50%(1c),c=50%(1c)" in r1_distribution + + r2 = summary.tasks["R2"] + assert r2.tool_reps == 1 + assert r2.failed_tool_reps == 0 + assert r2.tool_rep_frequency == {} + assert r2.tool_call_counts == {"one_rep": 1} + assert format_tool_distribution(r2) == "success-only n=1; failed excluded=0; frequency=—" + + r3 = summary.tasks["R3"] + assert r3.tool_reps == 0 + assert r3.failed_tool_reps == 1 + assert r3.tool_rep_frequency == {} + assert r3.tool_call_counts == {} + assert format_tool_distribution(r3) == "success-only n=0; failed excluded=1; frequency=—" + + +@pytest.mark.parametrize( + "case", + case_params( + _summarize_excludes_infra_errors_from_success, + _summarize_aggregate_wilson_and_call_variance, + _tool_distribution_uses_successful_repetitions, + ), +) +def test_summarize_behaviours(case): + case() + + +def test_wilson_interval_bounds(): + lo, hi = wilson_interval(5, 10) + assert lo == pytest.approx(0.2366, abs=1e-4) + assert hi == pytest.approx(0.7634, abs=1e-4) + lo0, hi0 = wilson_interval(0, 10) + assert lo0 == 0.0 + assert hi0 == pytest.approx(0.27754, abs=1e-4) + assert wilson_interval(0, 0) == (0.0, 0.0) + + +def test_headline_interval_bootstraps_35_task_clusters_instead_of_175_repetitions(): + rows = [ + {"task_id": f"T{task_index:02d}", "rep": rep, "success": task_index < 17, "calls": []} + for task_index in range(35) + for rep in range(5) + ] + + summary = summarize(rows) + + assert (summary.aggregate_k, summary.aggregate_n) == (85, 175) + assert summary.aggregate_wilson_lo == pytest.approx(0.4127693534) + assert summary.aggregate_wilson_hi == pytest.approx(0.5592729455) + assert summary.task_mean_success == pytest.approx(17 / 35) + assert summary.task_cluster_lo == pytest.approx(0.3142857143) + assert summary.task_cluster_hi == pytest.approx(0.6571428571) + assert summary.task_cluster_hi - summary.task_cluster_lo > ( + summary.aggregate_wilson_hi - summary.aggregate_wilson_lo + ) + + +def test_multi_rep_synthetic_file_reports_wilson_and_instability_without_noise_claim(tmp_path: Path, capsys): + path = tmp_path / "multi.jsonl" + outcomes = { + "R1": [True, True, True], + "R2": [True, False, True], + "R3": [False, False, False], + } + rows = [ + { + "task_id": task_id, + "rep": rep, + "label": "local", + "success": success, + "num_calls": rep + 1, + "calls": [], + } + for task_id, task_outcomes in outcomes.items() + for rep, success in enumerate(task_outcomes) + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + loaded = load_rows(path) + summary = summarize(loaded) + + assert len(loaded) == 9 # distinct rep keys are not deduped away + assert summary.tasks["R1"].success == "3/3" + assert summary.tasks["R1"].unstable is False + assert summary.tasks["R2"].success == "2/3" + assert summary.tasks["R2"].wilson_lo == pytest.approx(0.2077, abs=1e-4) + assert summary.tasks["R2"].wilson_hi == pytest.approx(0.9385, abs=1e-4) + assert summary.tasks["R2"].unstable is True + assert summary.tasks["R3"].success == "0/3" + assert summary.tasks["R3"].unstable is False + assert summary.unstable_task_ids == ["R2"] + + report_mod.print_table(summary, "Summary: multi.jsonl") + output = capsys.readouterr().out + assert "unstable" in output + r2_line = next(line for line in output.splitlines() if line.startswith("R2")) + assert "2/3" in r2_line + assert "[0.21,0.94]" in r2_line + assert "YES" in r2_line + assert "noise floor" not in output + + +def test_single_rep_summary_renders_tool_distribution_unavailable(capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "label": "local", + "success": True, + "trace_integrity": True, + "num_calls": 2, + "calls": [], + } + ] + + report_mod.print_table(summarize(rows), "Summary: sample.jsonl") + + assert capsys.readouterr().out == ( + "Summary: sample.jsonl\n" + "task-cluster success: 100.0% across 1 tasks cluster-bootstrap95 [1.00,1.00]\n" + "pooled repetition success: 1/1 (100.0%) Wilson95 [0.21,1.00]\n" + "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" + "off-surface indicators: 0\n" + " zero-call success: 0\n" + " write without a write call: not evaluated — this file declares no task tags, " + "so mutation intent is unknown\n" + " answer without provenance: 0\n" + " implausibly few calls: 0; rule: among at least 5 successful trace-usable repetitions for the same " + "task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" + " limitation: detects off-surface work only when it leaves a trace signature; it cannot detect an agent " + "that performs the work off-surface and also makes convincing surface calls\n" + "schema friction (same successful, trace-intact rows as call deltas): task-mean median errored calls=0.0 " + "across 1 tasks; task-mean errored-call rate=0.0% across 1 tasks with calls\n" + " errored-call tasks: 0/1 []\n" + " by kind, of 2 calls: surface friction=0 (0.0%), navigation=0 (0.0%), " + "answered existence questions=0 (0.0%), other=0 (0.0%), unclassified=0 (0.0%)\n" + " surface friction is the number to act on: a well-formed call the API refused on " + "meaning \u2014 none in this run\n" + " limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; an error that is " + "the correct task outcome still contributes, while calling the wrong tool successfully does not\n" + " limitation: a first not_found is read as the answer to an existence question, since asking has no " + "cheaper form; only a repeat on the same tool and action is counted as friction. A surface that " + "misleads an agent into one wrong lookup is therefore not charged for it\n" + "POWER: 1 of 1 task(s) below 5 repetitions (fewest 1) \u2014 their per-task pass rates " + "and UNSTABLE flags are not verdicts at that depth; read the aggregate and paired " + "deltas for those. Raising --reps narrows a per-task interval but no fixed count " + "makes one conclusive.\n" + "failure kinds: no failed rows\n" + "redundant lookups: not measured \u2014 1 row(s) carry no recorded call arguments\n" + "economics: cost=unmeasured (prices as of 2026-08-24); input tokens=unmeasured; result tokens=0\n" + " wall time=0s; call latency n/a\n" + " limitation: cost is computed from a static price table; a model absent from it reports " + "unpriced, and a row whose driver recorded no usage at all reports unmeasured. Neither is $0\n" + "RUN COMPLETE: 1/1 rows completed\n" + "tool variability: —\n" + "task n success wilson95 success_calls_med success_calls_min success_calls_q1-q3 " + "err capped h_err i_err " + "med_rtok p95_rtok med_cum_in tool distribution\n" + "----------------------------------------------------------------------------------------------------------------------------------------------------------------------\n" + "R1 1 1/1 [0.21,1.00] 2.0 2.0 2.0-2.0 0 0 " + " 0 0 - - 0 success-only n=1; failed excluded=0; frequency=—\n" + ) diff --git a/tests/evals/report/test_table.py b/tests/evals/report/test_table.py new file mode 100644 index 0000000..bfeec17 --- /dev/null +++ b/tests/evals/report/test_table.py @@ -0,0 +1,414 @@ +"""Offline eval tests for table.""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from evals import report as report_mod +from evals.report import ( + build_multi_surface_table, + format_surface_cell, + render_multi_surface_table, + summarize, +) +from tests.evals.conftest import case_params + + +def _synth_row( + tid: str, + *, + rep: int = 0, + success: bool = True, + num_calls: int = 2, + server: str = "local", + skipped: str | None = None, + error: str | None = None, + error_class: str | None = None, + label: str = "local", + calls: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + return { + "task_id": tid, + "rep": rep, + "label": label, + "success": success, + "trace_integrity": True, + "num_calls": num_calls, + "server": server, + "skipped": skipped, + "error": error, + "error_class": error_class, + "calls": list(calls or []), + "tool_manifest_fingerprint": "manifest-a", + } + + +def test_print_table_shows_infra_errors(capsys): + summary = summarize( + [ + {"task_id": "R1", "success": True, "num_calls": 1, "calls": []}, + {"task_id": "R1", "error": "seed failed", "error_class": "infra_seed"}, + {"task_id": "R1", "error": "CLI failed", "error_class": "infra_cli"}, + ] + ) + report_mod.print_table(summary, "Summary: test") + out = capsys.readouterr().out + assert "infra errors: 2" in out + assert "i_err" in out + assert "R1" in out + # per-task infra_err value rendered next to h_err + assert " 2" in out # i_err column value + + +def test_report_separates_success_from_completeness_and_sets_exit_status(tmp_path, capsys): + collision = tmp_path / "collision.jsonl" + collision.write_text( + "\n".join( + [ + json.dumps({"row_type": "meta", "expected_rows": 1, "server": "local"}), + json.dumps(_synth_row("R1", skipped="env:fixture-collision:customers:Acme")), + ] + ) + + "\n", + encoding="utf-8", + ) + + assert report_mod.main([str(collision)]) == 1 + collision_output = capsys.readouterr().out + assert "pooled repetition success: 0/0" in collision_output + assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in collision_output + assert "R1 (env:fixture-collision:customers:Acme)" in collision_output + assert "RUN INCOMPLETE:" in collision_output + assert "unexpected skips=1 [fixture-collision=1]" in collision_output + + plan_gated = tmp_path / "plan-gated.jsonl" + plan_gated.write_text( + "\n".join( + [ + json.dumps({"row_type": "meta", "expected_rows": 1, "server": "local"}), + json.dumps(_synth_row("L4", skipped="env:plan-gated:customers")), + ] + ) + + "\n", + encoding="utf-8", + ) + + assert report_mod.main([str(plan_gated)]) == 0 + plan_output = capsys.readouterr().out + assert "pooled repetition success: 0/0" in plan_output + assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in plan_output + assert "L4 (env:plan-gated:customers)" in plan_output + assert "RUN COMPLETE:" in plan_output + assert "expected skips=1 [plan-gated=1]" in plan_output + + +def _report_marks_entirely_estimated_result_token_columns(_tmp_path, capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "trace_integrity": True, + "num_calls": 1, + "calls": [{"result_tokens": 12, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + } + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "estimated" + assert summary.tasks["R1"].result_tokens_mode == "estimated" + + report_mod.print_table(summary, "estimated") + output = capsys.readouterr().out + assert "entirely estimated" in output + assert "med_rtok~" in output + assert "~12" in output + + +def _report_marks_mixed_measured_and_estimated_columns(_tmp_path, capsys): + rows = [ + { + "task_id": "R1", + "rep": 0, + "success": True, + "trace_integrity": True, + "num_calls": 1, + "calls": [{"result_tokens": 8, "result_tokens_estimated": False}], + "result_tokens_estimated": False, + }, + { + "task_id": "R1", + "rep": 1, + "success": True, + "trace_integrity": True, + "num_calls": 1, + "calls": [{"result_tokens": 10, "result_tokens_estimated": True}], + "result_tokens_estimated": True, + }, + ] + summary = summarize(rows) + assert summary.result_tokens_mode == "mixed" + assert summary.tasks["R1"].result_tokens_mode == "mixed" + + report_mod.print_table(summary, "mixed") + output = capsys.readouterr().out + assert "mixed measured and estimated" in output + assert "med_rtok*" in output + + +def _report_main_table_cli(tmp_path, capsys): + f1 = tmp_path / "a.jsonl" + f2 = tmp_path / "b.jsonl" + f1.write_text( + json.dumps(_synth_row("R1", label="local", num_calls=2)) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps(_synth_row("R1", label="candidate", num_calls=1)) + "\n", + encoding="utf-8", + ) + rc = report_mod.main(["--table", str(f1), str(f2)]) + assert rc == 0 + out = capsys.readouterr().out + assert "local" in out and "candidate" in out + assert "R1" in out + + +def _report_main_table_refuses_when_battery_fingerprints_differ(tmp_path, capsys): + f1 = tmp_path / "old.jsonl" + f2 = tmp_path / "new.jsonl" + f1.write_text( + json.dumps({**_synth_row("R1", label="local"), "battery": "6425dcc64404"}) + "\n", + encoding="utf-8", + ) + f2.write_text( + json.dumps({**_synth_row("R1", label="candidate"), "battery": "newfinger001"}) + "\n", + encoding="utf-8", + ) + + rc = report_mod.main(["--table", str(f1), str(f2)]) + + assert rc == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert "comparability cannot be established from the persisted identity" in captured.err + assert "battery differs across files" in captured.err + + +def _report_main_markdown_flag(tmp_path, capsys): + f1 = tmp_path / "a.jsonl" + row = {**_synth_row("R1", label="candidate", num_calls=1), "tool_manifest_fingerprint": None} + f1.write_text(json.dumps(row) + "\n", encoding="utf-8") + rc = report_mod.main(["--table", "--markdown", str(f1)]) + assert rc == 0 + out = capsys.readouterr().out + assert out.startswith("WARNING: TOOL MANIFEST ABSENT") + assert "\n\n| task |" in out + assert "| R1 |" in out + assert "---" in out + + +def _report_main_no_dedupe_flag(tmp_path, capsys): + p = tmp_path / "d.jsonl" + rows = [ + _synth_row("R1", label="local", num_calls=1, success=True), + {**_synth_row("R1", label="local", num_calls=9, success=False)}, + ] + # Both rows have the same (task_id, rep, label), so latest-wins keeps one. + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + rc = report_mod.main(["--no-dedupe", str(p)]) + assert rc == 0 + # With no-dedupe, both rows enter summarize → n=2 for R1. + # (dedupe default would leave n=1.) + out = capsys.readouterr().out + assert "R1" in out + assert "2/2" in out or "1/2" in out # one success of two + + +@pytest.mark.parametrize( + "case", + case_params( + _report_marks_entirely_estimated_result_token_columns, + _report_marks_mixed_measured_and_estimated_columns, + _report_main_table_cli, + _report_main_table_refuses_when_battery_fingerprints_differ, + _report_main_markdown_flag, + _report_main_no_dedupe_flag, + ), +) +def test_report_behaviours(case, tmp_path, capsys): + case(tmp_path, capsys) + + +def test_format_surface_cell_variants(): + assert format_surface_cell(None) == "—" + assert format_surface_cell(_synth_row("R1", skipped="nope")) == "skip" + assert format_surface_cell(_synth_row("R1", error="boom")) == "ERR" + assert format_surface_cell(_synth_row("R1", error_class="infra_seed", error="x")) == "ERR" + assert format_surface_cell(_synth_row("R1", success=True, num_calls=3)) == "✅ 3c · tools —" + assert format_surface_cell(_synth_row("R1", success=False, num_calls=4)) == "❌ 4c · tools —" + assert format_surface_cell(_synth_row("R1", server="external", num_calls=5)) == "✅ 5c · tools —" + + +def _multi_surface_table_snapshot_with_external(): + local = [ + _synth_row("R1", label="local", num_calls=4), + _synth_row("R2", label="local", success=False, num_calls=2), + ] + candidate = [ + _synth_row("R1", label="candidate", num_calls=2), + _synth_row("R2", label="candidate", skipped="unsupported", num_calls=0), + ] + external = [ + _synth_row("R1", label="akhil", server="external", num_calls=3), + _synth_row("R2", label="akhil", server="external", num_calls=1, success=False), + _synth_row("R3", label="akhil", server="external", error="timeout", error_class="infra_cli"), + ] + table = build_multi_surface_table([("local", local), ("candidate", candidate), ("akhil", external)]) + assert table["columns"] == ["local", "candidate", "akhil"] + assert "R1" in table["task_ids"] and "R3" in table["task_ids"] + assert table["cells"]["R1"]["local"] == "✅ 4c · tools —" + assert table["cells"]["R1"]["candidate"] == "✅ 2c · tools —" + assert table["cells"]["R1"]["akhil"] == "✅ 3c · tools —" + assert table["cells"]["R2"]["candidate"] == "skip" + assert table["cells"]["R3"]["akhil"] == "ERR" + + text = render_multi_surface_table(table, markdown=False) + assert "local" in text and "candidate" in text and "akhil" in text + assert "✅ 3c · tools —" in text + assert "skip" in text + assert "ERR" in text + assert "infra 1" in text + + md = render_multi_surface_table(table, markdown=True) + assert md.startswith("| task |") + assert "| R1 |" in md + assert "---" in md + assert "**agg**" in md + + assert table["footer"]["akhil"]["tool_variability"] is None + assert table["footer"]["local"]["tool_variability"] is None + assert table["footer"]["akhil"]["infra_errors"] == 1 + + +def _multi_surface_table_aggregates_reps_and_flags_unstable(): + rows = [ + _synth_row("R1", rep=0, success=True, num_calls=2, label="local", calls=[{"tool": "a"}, {"tool": "b"}]), + _synth_row("R1", rep=1, success=True, num_calls=3, label="local", calls=[{"tool": "a"}]), + _synth_row("R1", rep=2, success=True, num_calls=2, label="local", calls=[{"tool": "a"}]), + _synth_row("R2", rep=0, success=True, num_calls=1, label="local", calls=[{"tool": "c"}]), + _synth_row("R2", rep=1, success=False, num_calls=4, label="local", calls=[{"tool": "failed_only"}]), + _synth_row("R2", rep=2, success=True, num_calls=2, label="local", calls=[{"tool": "c"}]), + ] + + table = build_multi_surface_table([("local", rows)]) + + assert table["multi_rep"] is True + assert table["cells"]["R1"]["local"] == ( + "✅ 3/3 [0.44,1.00] 2-3c · tools success-only n=3; failed excluded=0; core:a(3c); variable:b=33%(1c)" + ) + assert table["cells"]["R2"]["local"] == ( + "⚠ UNSTABLE 2/3 [0.21,0.94] 1-4c · tools success-only n=2; failed excluded=1; core:c(2c)" + ) + assert table["footer"]["local"]["success"] == 5 + assert table["footer"]["local"]["n"] == 6 + assert table["footer"]["local"]["tool_variability"] == 1 + rendered = render_multi_surface_table(table) + assert "tool variability 1/2 tasks" in rendered + assert "noise floor" not in rendered + + +@pytest.mark.parametrize( + "case", + case_params(_multi_surface_table_snapshot_with_external, _multi_surface_table_aggregates_reps_and_flags_unstable), +) +def test_multi_surface_behaviours(case): + case() + + +def test_single_rep_multi_surface_renders_tool_distribution_unavailable(): + # The prompt excerpt is read from the run's own metadata, never from the checkout, so a + # rendered table describes the prompt that actually ran. + meta = { + "row_type": "meta", + "task_metadata": {"R1": {"prompt": "In project {project}, what is the current state of the item?"}}, + } + rows = [meta, _synth_row("R1", label="local", success=True, num_calls=2)] + + rendered = render_multi_surface_table(build_multi_surface_table([("local", rows)])) + + assert rendered == ( + "task what local \n" + "-------------------------------------------------------\n" + "R1 In project P, what is the curren… ✅ 2c · tools —\n" + "-------------------------------------------------------\n" + "local success task-cluster 100.0% [1.00,1.00]; pooled 1/1 total calls 2 " + "tool variability — infra 0\n" + "local EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)\n" + "local off-surface indicators: 0\n" + "local zero-call success: 0\n" + "local write without a write call: 0\n" + "local answer without provenance: 0\n" + "local implausibly few calls: 0; rule: among at least 5 successful trace-usable repetitions for " + "the same task, calls < Q1 - 3×IQR and calls ≤ half the task median\n" + "local limitation: detects off-surface work only when it leaves a trace signature; it cannot detect " + "an agent that performs the work off-surface and also makes convincing surface calls\n" + "local schema friction (same successful, trace-intact rows as call deltas): task-mean median errored " + "calls=0.0 across 1 tasks; task-mean errored-call rate=0.0% across 1 tasks with calls\n" + "local errored-call tasks: 0/1 []\n" + "local by kind, of 2 calls: surface friction=0 (0.0%), navigation=0 (0.0%), answered " + "existence questions=0 (0.0%), other=0 (0.0%), unclassified=0 (0.0%)\n" + "local surface friction is the number to act on: a well-formed call the API refused on " + "meaning \u2014 none in this run\n" + "local limitation: is_error is the MCP-level error flag, so this counts tool-reported failures; an " + "error that is the correct task outcome still contributes, while calling the wrong tool successfully does not\n" + "local limitation: a first not_found is read as the answer to an existence question, since " + "asking has no cheaper form; only a repeat on the same tool and action is counted as friction. A " + "surface that misleads an agent into one wrong lookup is therefore not charged for it\n" + "local RUN COMPLETE: 1/1 rows completed\n" + ) + + +def test_multi_surface_table_reports_off_surface_indicators_per_column_in_plain_and_markdown(): + write_meta = {"row_type": "meta", "task_metadata": {"W1": {"tags": ["write"]}}} + clean = [_synth_row("R1", label="clean", num_calls=1, calls=[{"tool": "list_work_items"}])] + bypass = [write_meta, _synth_row("W1", label="bypass", num_calls=0, calls=[])] + table = build_multi_surface_table([("clean", clean), ("bypass", bypass)]) + + plain = render_multi_surface_table(table) + assert "clean off-surface indicators: 0" in plain + assert "bypass off-surface indicators: 1 flagged rows (2 indicator hits)" in plain + assert "bypass zero-call success: 1 [W1[rep=0]]" in plain + assert "bypass write without a write call: 1 [W1[rep=0]]" in plain + + markdown = render_multi_surface_table(table, markdown=True) + assert "| **off-surface indicators** | |" in markdown + assert "off-surface indicators: 0
" in markdown + assert "off-surface indicators: 1 flagged rows (2 indicator hits)
" in markdown + + +def test_multi_surface_table_reports_schema_friction_per_column_in_plain_and_markdown(): + clean = [_synth_row("R1", label="clean", num_calls=4, calls=[{"tool": "get_work_item"}])] + friction = [ + _synth_row( + "R1", + label="friction", + num_calls=4, + calls=[{"tool": "get_work_item", "is_error": True}], + ) + ] + table = build_multi_surface_table([("clean", clean), ("friction", friction)]) + + plain = render_multi_surface_table(table) + assert "clean schema friction" in plain + assert "clean errored-call tasks: 0/1 []" in plain + assert "friction errored-call tasks: 1/1 [R1=1/4 (25.0%)]" in plain + assert "friction limitation: is_error is the MCP-level error flag" in plain + + markdown = render_multi_surface_table(table, markdown=True) + assert "| **schema friction** | |" in markdown + assert "errored-call tasks: 0/1 []" in markdown + assert "errored-call tasks: 1/1 [R1=1/4 (25.0%)]" in markdown diff --git a/tests/evals/runner/__init__.py b/tests/evals/runner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/runner/test_canary.py b/tests/evals/runner/test_canary.py new file mode 100644 index 0000000..6e2b820 --- /dev/null +++ b/tests/evals/runner/test_canary.py @@ -0,0 +1,167 @@ +"""Offline eval tests for honest verifier-canary coverage.""" + +from __future__ import annotations + +import asyncio +import re +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +from plane.errors.errors import HttpError + +from evals.runner import canary as runner_canary +from evals.runner import run_canary +from evals.tasks.schema import verify_s2 +from evals.tasks.skip import TaskSkipped + + +def _task(task_id: str, verify: Any) -> dict[str, Any]: + return { + "id": task_id, + "prompt": "x {project}", + "needs": set(), + "verify": verify, + } + + +def _install_harness(monkeypatch, *, plane=None, seed=None, teardown=None): + monkeypatch.setattr(runner_canary, "make_plane_client", lambda: (plane or MagicMock(), "test-ws")) + monkeypatch.setattr( + runner_canary, + "seed", + seed or (lambda *args, **kwargs: kwargs["ctx"].update({"project_name": "P", "project_id": "1"})), + ) + monkeypatch.setattr(runner_canary, "teardown", teardown or (lambda *args, **kwargs: None)) + + +async def _reject(_plane, _ctx, run): + assert run["calls"] == [] + assert run["call_source"] == "canary" + return False, "zero-call probe rejected" + + +def test_canary_detects_a_verifier_that_accepts_empty_output(monkeypatch): + _install_harness(monkeypatch) + + async def always_ok(_plane, _ctx, _run): + return True, "false positive" + + rc = asyncio.run(run_canary([_task("GOOD", _reject), _task("BAD", always_ok)], label="local")) + assert rc == 1 + + +def test_canary_accepts_a_fully_verified_set(monkeypatch, capsys): + _install_harness(monkeypatch) + rc = asyncio.run(run_canary([_task("G1", _reject)], label="local")) + assert rc == 0 + output = capsys.readouterr().out + assert "verified=1/1 ids=['G1']" in output + assert "skipped=0 ids=[]" in output + assert "errored=0 ids=[]" in output + + +def test_canary_treats_missing_s2_estimate_as_a_rejected_probe_not_an_error(monkeypatch, capsys): + def missing_estimate(**kwargs): + raise HttpError("Estimate not found", 404, {}) + + plane = SimpleNamespace(estimates=SimpleNamespace(retrieve=missing_estimate)) + + def seed(*args, **kwargs): + kwargs["ctx"].update({"workspace_slug": "test-ws", "project_name": "P", "project_id": "1"}) + + _install_harness(monkeypatch, plane=plane, seed=seed) + rc = asyncio.run(run_canary([_task("S2", verify_s2)], label="local")) + + assert rc == 0 + output = capsys.readouterr().out + assert "verified=1/1 ids=['S2']" in output + assert "errored=0 ids=[]" in output + assert "requested Fibonacci scale was not created" in output + + +def test_canary_reports_partial_coverage_without_saying_all(monkeypatch, capsys): + def seed(*args, **kwargs): + if kwargs["task_id"] == "SKIP": + raise TaskSkipped("env:plan-gated:releases") + kwargs["ctx"].update({"project_name": "P", "project_id": "1"}) + + _install_harness(monkeypatch, seed=seed) + rc = asyncio.run(run_canary([_task("GOOD", _reject), _task("SKIP", _reject)], label="local")) + assert rc == 0 + captured = capsys.readouterr() + output = captured.out + captured.err + assert "verified=1/2 ids=['GOOD']" in output + assert "skipped=1 ids=['SKIP']" in output + assert "env:plan-gated:releases" in output + assert not re.search(r"\ball\b", output, flags=re.IGNORECASE) + + +def test_canary_strict_mode_fails_when_a_required_id_is_skipped(monkeypatch, capsys): + def seed(*args, **kwargs): + if kwargs["task_id"] == "SKIP": + raise TaskSkipped("env:plan-gated:releases") + kwargs["ctx"].update({"project_name": "P", "project_id": "1"}) + + _install_harness(monkeypatch, seed=seed) + rc = asyncio.run( + run_canary( + [_task("GOOD", _reject), _task("SKIP", _reject)], + label="local", + required_task_ids={"GOOD", "SKIP"}, + ) + ) + assert rc == 1 + assert "missing required ids=['SKIP']" in capsys.readouterr().err + + +def test_canary_teardown_error_affects_exit_and_error_report(monkeypatch, capsys): + def teardown(*args, **kwargs): + raise RuntimeError("cleanup failed") + + _install_harness(monkeypatch, teardown=teardown) + rc = asyncio.run(run_canary([_task("G1", _reject)], label="local")) + assert rc == 1 + captured = capsys.readouterr() + output = captured.out + captured.err + assert "errored=1 ids=['G1']" in output + assert "teardown RuntimeError: cleanup failed" in output + + +def test_canary_labels_attachment_storage_seed_failure_as_infrastructure(monkeypatch, capsys): + def seed(*args, **kwargs): + raise ConnectionError("localhost:9000 attachment storage unreachable") + + _install_harness(monkeypatch, seed=seed) + rc = asyncio.run(run_canary([_task("L5", _reject)], label="local")) + + assert rc == 1 + captured = capsys.readouterr() + output = captured.out + captured.err + assert "L5 canary ERROR[infra_seed]" in output + assert "infra_seed ConnectionError: localhost:9000 attachment storage unreachable" in output + assert "skipped=0 ids=[]" in output + + +def test_canary_catches_adversarial_canned_contract_output(monkeypatch, capsys): + _install_harness(monkeypatch) + + async def accepts_fabricated_count(_plane, _ctx, run): + if run["final_text"] == "": + return False, "empty rejected" + return run["final_text"] == "count: 0", "fabricated zero accepted" + + rc = asyncio.run(run_canary([_task("R2", accepts_fabricated_count)], label="local")) + assert rc == 1 + output = capsys.readouterr().out + assert "accepted canary probe" in output + assert "count: 0" in output + + +def test_canary_exits_nonzero_when_no_task_is_verified(monkeypatch): + _install_harness( + monkeypatch, + seed=lambda *args, **kwargs: (_ for _ in ()).throw(TaskSkipped("fixture unavailable")), + ) + rc = asyncio.run(run_canary([_task("SKIPME", _reject)], label="local")) + assert rc == 1 diff --git a/tests/evals/runner/test_live.py b/tests/evals/runner/test_live.py new file mode 100644 index 0000000..abdf30f --- /dev/null +++ b/tests/evals/runner/test_live.py @@ -0,0 +1,1317 @@ +"""Offline eval tests for live.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +import pytest +from plane.errors.errors import HttpError + +from evals import cli as run_mod +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import RESULT_SCHEMA_VERSION, AgentRun, TaskResult +from evals.drivers.cli.claude import ClaudeCliDriver +from evals.report import load_rows, summarize +from evals.runner import ( + is_infra_cli_stop_reason, + run_live, +) +from evals.runner import live as runner_live +from evals.runner.live import stdio_server_env +from evals.seed.identities import record_seeded_entity +from evals.seed.randomize import random_truth_token, record_randomized_truth +from evals.tasks.catalog import task_fingerprint +from evals.tasks.skip import TaskSkipped +from tests.evals.conftest import _data_rows, case_params + + +def _taxonomy_task( + task_id: str, + verify: Any, + *, + prompt: str = "do {project}", + needs: set[str] | None = None, +) -> dict[str, Any]: + return { + "id": task_id, + "prompt": prompt, + "tags": set(), + "needs": set(needs or set()), + "verify": verify, + } + + +def _stdio_env_still_works_for_cli_drivers(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_API_KEY", "k") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + monkeypatch.delenv("EVAL_PLANE_BASE_URL", raising=False) + env = stdio_server_env() + assert env["PLANE_API_KEY"] == "k" + assert "ANTHROPIC_API_KEY" not in env + + +def _stdio_server_env_does_not_leak_ambient_secrets(monkeypatch): + monkeypatch.setenv("SOME_SECRET", "x") + + environment = runner_live.stdio_server_env() + + assert "SOME_SECRET" not in environment + assert environment["PLANE_API_KEY"] == "test-key" + assert environment["PLANE_WORKSPACE_SLUG"] == "test-ws" + assert environment["PLANE_BASE_URL"] == "https://api.plane.so" + + +@pytest.mark.parametrize( + "case", + case_params(_stdio_env_still_works_for_cli_drivers, _stdio_server_env_does_not_leak_ambient_secrets), +) +def test_stdio_behaviours(monkeypatch, case): + case(monkeypatch) + + +def test_live_run_rejects_non_positive_reps(capsys): + assert run_mod.main(["--tasks", "R1", "--reps", "0"]) == 2 + assert "--reps must be at least 1" in capsys.readouterr().err + + +def _run_seed_failure_is_infra_seed(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + + fake_plane = MagicMock() + driver = MagicMock() + torn: list[dict[str, Any]] = [] + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + + def boom_seed(plane, run_id, needs, ctx, task_id=None): + ctx["project_name"] = "EVAL deadbeef" + raise HttpError("identifier already taken", 409) + + monkeypatch.setattr(runner_live, "seed", boom_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = { + "id": "T1", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + } + + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + resolved_model_id="sonnet", + ) + ) + assert rc == 1 + rows = _data_rows(out) + assert len(rows) == 1 + row = rows[0] + assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["error_class"] == "infra_seed" + assert row["success"] is False + assert row["verify_note"] == "" + assert "HttpError" in (row["error"] or "") + assert "identifier" in (row["error"] or "").lower() + assert row["battery"] # fingerprint written + assert row["task_fingerprint"] == task_fingerprint(task) + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "sonnet" + assert row["model"] == "sonnet" + meta = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) + assert meta["schema_version"] == RESULT_SCHEMA_VERSION + assert meta["requested_tier"] == "standard" + assert meta["resolved_model"] == "sonnet" + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL deadbeef"}] + + +def _run_missing_bug_type_uses_context_skip_reason(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + def seed_without_bug_type(plane, run_id, needs, ctx, task_id=None): + ctx.update( + { + "project_name": "EVAL no bug type", + "project_id": "p1", + "bug_type_skip_reason": "env:plan-gated:work-item-types", + } + ) + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr(runner_live, "seed", seed_without_bug_type) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "S1", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + needs={"bug_type"}, + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["skipped"] == "env:plan-gated:work-item-types" + assert row["verify_note"] == "env:plan-gated:work-item-types" + assert row["error"] is None + assert row["error_class"] is None + driver.run_task.assert_not_called() + assert torn == [ + { + "project_name": "EVAL no bug type", + "project_id": "p1", + "bug_type_skip_reason": "env:plan-gated:work-item-types", + } + ] + + +def _run_prompt_bind_failure_is_infra_seed(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + torn: list[dict[str, Any]] = [] + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL prompt", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "PROMPT", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + prompt="use {missing_seed_id} in {project}", + ) + rc = asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_seed" + assert row["verify_note"] == "" + assert row["error"].startswith("PromptBindError: missing prompt field {missing_seed_id}") + driver.run_task.assert_not_called() + assert torn == [{"project_name": "EVAL prompt", "project_id": "p1"}] + + +def _run_api_driver_exception_is_infra_api(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.side_effect = RuntimeError("provider unavailable") + torn: list[dict[str, Any]] = [] + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL api", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + task = _taxonomy_task( + "APIERR", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("verify must not run")), + ) + rc = asyncio.run( + run_live( + [task], + model_alias="standard", + reps=1, + label="local", + out_path=out, + driver_name="api", + resolved_model_id="provider-model-id", + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_api" + assert row["verify_note"] == "" + assert row["success"] is False + assert row["error"] == "RuntimeError: provider unavailable" + driver.run_task.assert_called_once() + assert torn == [{"project_name": "EVAL api", "project_id": "p1"}] + + +def _run_driver_exception_is_infra_cli(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + + def ok_seed(plane, run_id, needs, ctx, task_id=None): + ctx.update({"project_name": "EVAL deadbeef", "project_id": "p1"}) + + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + + class BoomDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + error = RuntimeError("claude cli failed: json_parse_failed") + error.trace_integrity = False + error.trace_integrity_reason = "recorder_loss" + error.tool_manifest_fingerprint = None + raise error + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: BoomDriver()) + + task = { + "id": "T2", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": lambda *a, **k: (False, "nope"), + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert "RuntimeError" in (row["error"] or "") + assert row["trace_integrity"] is False + assert row["trace_integrity_reason"] == "recorder_loss" + + +def _run_timeout_agent_is_infra_cli(tmp_path, monkeypatch, _capsys): + from evals.core.results import agent_run_to_harness_dict + + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + class TimeoutDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="timeout", + notes=["timeout after 900s"], + ) + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: TimeoutDriver()) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return True, "should not run" + + task = { + "id": "T3", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify, + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["error"] == "timeout after 900s" # from driver_notes, not recomputed + assert row["stop_reason"] == "timeout" + assert verify_calls == [] + d = agent_run_to_harness_dict(AgentRun(calls=[], final_text="", usage=None, stopped_reason="timeout")) + assert d["stop_reason"] == "timeout" + + +def _run_error_during_execution_is_infra_cli(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "result": "MCP server crashed", + "session_id": "sess-err", + "num_turns": 1, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="claude boom") + + monkeypatch.setattr( + runner_live, + "get_driver", + lambda name, **kw: ClaudeCliDriver(runner=fake_run, use_proxy=False), + ) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return False, "nope" + + task = { + "id": "T4", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_cli" + assert row["stop_reason"] == "error_during_execution" + assert verify_calls == [] + assert "claude_exit=1" in (row.get("driver_notes") or []) + + +def _run_error_max_turns_is_task_path(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + payload = { + "type": "result", + "subtype": "error_max_turns", + "is_error": True, + "result": "hit max turns", + "session_id": "sess-max", + "num_turns": 15, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 1, stdout=json.dumps(payload), stderr="") + + monkeypatch.setattr( + runner_live, + "get_driver", + lambda name, **kw: ClaudeCliDriver(runner=fake_run, use_proxy=False), + ) + + verify_calls: list[Any] = [] + + async def verify(*a, **k): + verify_calls.append(1) + return False, "agent exhausted turns" + + task = { + "id": "T5", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify, + } + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + ) + ) + assert rc == 0 + row = _data_rows(out)[0] + assert row["error_class"] is None + assert row["stop_reason"] == "error_max_turns" + assert row["success"] is False + assert verify_calls == [1] + + +def _run_verifier_skip_is_not_a_failure(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] + + async def skip_verify(plane, ctx, run): + raise TaskSkipped("env:verification-unavailable") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL skip", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYSKIP", skip_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["skipped"] == "env:verification-unavailable" + assert row["verify_note"] == "env:verification-unavailable" + assert row["success"] is False + assert row["error"] is None + assert row["error_class"] is None + assert torn == [{"project_name": "EVAL skip", "project_id": "p1"}] + + +def _run_verifier_exception_is_task_error(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + torn: list[dict[str, Any]] = [] + + async def broken_verify(plane, ctx, run): + raise ValueError("verifier broke") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL verify", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: torn.append(dict(ctx))) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("VERIFYERR", broken_verify)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["success"] is False + assert row["error_class"] == "task" + assert row["error"] == "ValueError: verifier broke" + assert row["verify_note"] == "" + assert row["skipped"] is None + assert torn == [{"project_name": "EVAL verify", "project_id": "p1"}] + + +def _run_external_server_records_observed_calls(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "search_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + async def verify_ok(plane, ctx, run): + return True, "external ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL external", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("EXTERNAL", verify_ok)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + server_cmd=["/bin/foreign", "stdio"], + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["server"] == "external" + assert row["num_calls"] == 1 + assert row["calls"][0]["tool"] == "search_work_items" + + +def _run_success_keeps_requested_and_resolved_models(tmp_path, monkeypatch, _capsys): + out = tmp_path / "rows.jsonl" + driver = MagicMock() + driver.run_task.return_value = AgentRun( + calls=[{"tool": "list_work_items", "args": {}}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + async def verify_ok(plane, ctx, run): + return True, "local ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "test-ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update({"project_name": "EVAL local", "project_id": "p1"}), + ) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("SUCCESS", verify_ok)], + model_alias="standard", + resolved_model_id="provider-model-id", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 0 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["requested_model"] == "standard" + assert row["requested_tier"] == "standard" + assert row["resolved_model"] == "provider-model-id" + assert row["server"] == "local" + + +def _run_multi_rep_uses_fresh_fixture_seed_and_teardown_per_rep(tmp_path, monkeypatch, _capsys): + out = tmp_path / "multi.jsonl" + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_ids: list[str] = [] + teardown_projects: list[str] = [] + + def fresh_seed(plane, run_id, needs, ctx, task_id=None): + seed_ids.append(run_id) + ctx.update({"project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) + + def record_teardown(plane, ctx): + teardown_projects.append(ctx["project_id"]) + + async def fake_agent(**kwargs): + return TaskResult(final_text="done", stop_reason="end_turn") + + monkeypatch.setattr(runner_live, "seed", fresh_seed) + monkeypatch.setattr(runner_live, "teardown", record_teardown) + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kwargs: object()) + monkeypatch.setattr(runner_live, "run_agent_task_via_driver", fake_agent) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + task = { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + } + + rc = asyncio.run( + run_live( + [task], + model_alias="sonnet", + reps=3, + label="local", + out_path=out, + driver_name="claude-cli", + ) + ) + + assert rc == 0 + assert len(seed_ids) == 3 + assert len(set(seed_ids)) == 3 + assert teardown_projects == seed_ids + rows = _data_rows(out) + assert {row["fixture_seed_id"] for row in rows} == set(seed_ids) + assert len({row["run_id"] for row in rows}) == 1 + rows = _data_rows(out) + assert [row["rep"] for row in rows] == [0, 1, 2] + assert all(row["success"] is True for row in rows) + + +def test_runner_passes_seeded_evidence_to_driver_and_retains_only_labels(monkeypatch): + sentinel = "hidden-target-fact-2f81a0cd" + captured: dict[str, Any] = {} + + class Driver: + def run_task(self, *args, **kwargs): + captured.update(kwargs) + return AgentRun( + calls=[ + { + "tool": "read_any_route", + "args": {}, + "is_error": False, + "result_chars": 42, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + final_text="count: 4", + usage=None, + stopped_reason="end_turn", + call_source="api", + evidence_trace_available=True, + ) + + monkeypatch.setenv("EVAL_PLANE_API_KEY", "key") + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + task = {"id": "R2", "prompt": "In {project}, count.", "tags": {"read"}, "needs": {"items"}} + context = { + "project_name": "EVAL deadbeef", + "evidence_sentinels": {TARGET_ENTITY_EVIDENCE: [sentinel]}, + "evidence_targets": {TARGET_ENTITY_EVIDENCE: ["project-1"]}, + } + + row = asyncio.run( + runner_live.run_agent_task_via_driver( + driver=Driver(), + model_id="model", + task=task, + ctx=context, + workspace_slug="ws", + ) + ) + + assert captured["evidence_sentinels"] == context["evidence_sentinels"] + assert captured["evidence_targets"] == context["evidence_targets"] + assert row.evidence_trace_available is True + assert row.calls[0].observed_sentinels == [TARGET_ENTITY_EVIDENCE] + assert sentinel not in json.dumps(row.to_row()) + + +def test_run_live_headline_uses_task_cluster_interval(tmp_path, monkeypatch, capsys): + async def passes(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + async def fake_drive(**kwargs): + del kwargs + return TaskResult(final_text="done", num_calls=1, trace_integrity=True) + + monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + + rc = asyncio.run( + run_live( + [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)], + model_alias="standard", + reps=1, + label="local", + out_path=tmp_path / "cluster.jsonl", + ) + ) + + assert rc == 0 + output = capsys.readouterr().out + assert "success: 100.0% across 2 tasks (cluster-bootstrap95 [1.00,1.00]; pooled repetitions 2/2)" in output + assert "success: 2/2 (100.0%)" not in output + + +def _run_passes_server_cmd_to_non_claude(tmp_path, monkeypatch, _capsys): + from evals.runner import live as run_mod + + captured: dict = {} + + def fake_get_driver(name, **kwargs): + captured["name"] = name + captured["kwargs"] = kwargs + + class Dummy: + def run_task(self, *a, **k): + return AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + call_source="json", + ) + + return Dummy() + + monkeypatch.setattr(run_mod, "get_driver", fake_get_driver) + monkeypatch.setattr(run_mod, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(run_mod, "seed", lambda *a, **k: k["ctx"].update({"project_name": "P", "project_id": "1"})) + monkeypatch.setattr(run_mod, "teardown", lambda *a, **k: None) + + import asyncio + + async def _verify(*a, **k): + return False, "n" + + task = { + "id": "T", + "prompt": "x {project}", + "needs": set(), + "verify": _verify, + } + rc = asyncio.run( + run_mod.run_live( + [task], + model_alias="sonnet", + reps=1, + label="local", + out_path=tmp_path / "o.jsonl", + driver_name="opencode-cli", + server_cmd=["/bin/foreign", "stdio"], + ) + ) + assert rc == 0 + assert captured["name"] == "opencode-cli" + assert captured["kwargs"].get("server_command") == ["/bin/foreign", "stdio"] + + +def _run_reports_progress_per_repetition(tmp_path, monkeypatch, capsys): + out = tmp_path / "out.jsonl" + + async def passes(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", lambda *a, **k: k["ctx"].update({"project_name": "EVAL x"})) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + async def fake_drive(**kwargs): + return TaskResult(final_text="done", num_calls=2) + + monkeypatch.setattr(runner_live, "_drive_agent", fake_drive) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + + tasks = [_taxonomy_task("R1", passes), _taxonomy_task("R2", passes)] + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + assert rc == 0 + + printed = capsys.readouterr().out + # Position out of total, before the task runs. + assert "[ 1/2] R1 rep=0 running" in printed + assert "[ 2/2] R2 rep=0 running" in printed + # A running tally after each, and one closing summary. + assert "1/2 done · 1 pass · 0 fail · 0 skip" in printed + assert "finished 2/2 in " in printed + assert "2 pass, 0 fail, 0 skip" in printed + assert "schema friction (same successful, trace-intact rows as call deltas)" in printed + assert "task-mean median errored calls=0.0 across 2 tasks" in printed + assert "errored-call tasks: 0/2 []" in printed + assert "is_error is the MCP-level error flag" in printed + + +_RUN_CASES = case_params( + _run_seed_failure_is_infra_seed, + _run_missing_bug_type_uses_context_skip_reason, + _run_prompt_bind_failure_is_infra_seed, + _run_api_driver_exception_is_infra_api, + _run_driver_exception_is_infra_cli, + _run_timeout_agent_is_infra_cli, + _run_error_during_execution_is_infra_cli, + _run_error_max_turns_is_task_path, + _run_verifier_skip_is_not_a_failure, + _run_verifier_exception_is_task_error, + _run_external_server_records_observed_calls, + _run_success_keeps_requested_and_resolved_models, + _run_multi_rep_uses_fresh_fixture_seed_and_teardown_per_rep, + _run_passes_server_cmd_to_non_claude, + _run_reports_progress_per_repetition, +) + + +@pytest.mark.parametrize("case", _RUN_CASES) +def test_run_behaviours(case, tmp_path, monkeypatch, capsys): + case(tmp_path, monkeypatch, capsys) + + +def test_is_infra_cli_stop_reason_matrix(): + assert is_infra_cli_stop_reason("timeout") is True + assert is_infra_cli_stop_reason("error_during_execution") is True + assert is_infra_cli_stop_reason("error") is True + assert is_infra_cli_stop_reason("error_max_turns") is False + assert is_infra_cli_stop_reason("end_turn") is False + assert is_infra_cli_stop_reason("max_turns") is False + + +def test_task_skipped_from_seed_records_a_skip_row(tmp_path: Path, monkeypatch): + """A fixture that cannot be seeded records a skip — no agent, no crash. + + Genuine skips (an absent activity worker, a plan-gated feature) reach the + row through TaskSkipped, so the driver must never run and the row must not + count as a failure. + """ + out = tmp_path / "out.jsonl" + driven: list[str] = [] + torn: list[Any] = [] + driver = MagicMock() + + def skip_seed(*_args: Any, **_kwargs: Any) -> None: + raise TaskSkipped("env:no-activity-worker") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", skip_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: torn.append(1)) + monkeypatch.setattr( + runner_live, + "get_driver", + lambda *a, **k: driven.append("ran") or driver, + ) + + tasks = [ + { + "id": "L2", + "prompt": "x {project}", + "needs": {"activity_feed"}, + "verify": None, # never reached + }, + ] + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == 0 + assert driven == ["ran"] # driver is constructed once per run, never invoked + row = _data_rows(out)[0] + assert row["task_id"] == "L2" + assert row["skipped"] == "env:no-activity-worker" + assert row["error"] is None + assert row["error_class"] is None + assert row["label"] == "local" + assert torn == [1] # teardown still runs + driver.run_task.assert_not_called() + + # `skipped` is the discriminator, not `success` — a skip must leave the + # success denominator empty rather than counting as a failed task. + summary = summarize(load_rows(out)) + assert "L2" not in summary.tasks + assert summary.aggregate_n == 0 + assert summary.expected_skips == 1 + assert summary.unexpected_skips == 0 + assert summary.complete is True + + +def test_attachment_storage_connection_failure_is_infra_seed(tmp_path, monkeypatch): + out = tmp_path / "out.jsonl" + driver = MagicMock() + + def storage_unreachable(*_args, **_kwargs): + raise ConnectionError("localhost:9000 attachment storage unreachable") + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", storage_unreachable) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: driver) + + rc = asyncio.run( + run_live([_taxonomy_task("L5", None)], model_alias="standard", reps=1, label="local", out_path=out) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["task_id"] == "L5" + assert row["error_class"] == "infra_seed" + assert row["error"] == "ConnectionError: localhost:9000 attachment storage unreachable" + assert row["skipped"] is None + driver.run_task.assert_not_called() + + +def test_activity_read_connection_failure_is_infra_seed_and_incomplete(tmp_path, monkeypatch, capsys): + from evals.seed.work_items import CHECKOUT_TIMEOUT_TITLE, require_activities + + out = tmp_path / "out.jsonl" + driver = MagicMock() + + def activity_backend_unreachable(**kwargs): + raise ConnectionError("activity backend unreachable") + + plane = SimpleNamespace(work_items=SimpleNamespace(activities=SimpleNamespace(list=activity_backend_unreachable))) + + def seed_l2(plane, run_id, needs, ctx, task_id=None): + ctx.update( + { + "workspace_slug": "ws", + "project_id": "project-1", + "items": {CHECKOUT_TIMEOUT_TITLE: "work-item-1"}, + } + ) + require_activities(plane, "ws", ctx) + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (plane, "ws")) + monkeypatch.setattr(runner_live, "seed", seed_l2) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("L2", None, needs={"activity_feed"})], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["error_class"] == "infra_seed" + assert row["error"] == "ConnectionError: activity backend unreachable" + assert row["skipped"] is None + assert "RUN INCOMPLETE:" in capsys.readouterr().out + driver.run_task.assert_not_called() + + +@pytest.mark.parametrize( + ("reason", "task_id", "needs", "expected_rc", "verdict"), + [ + ("env:plan-gated:customers", "L4", {"customer"}, 0, "RUN COMPLETE:"), + ("env:no-activity-worker", "L2", {"activity_feed"}, 0, "RUN COMPLETE:"), + ("env:plan-gated:customerz", "L4", {"customer"}, 1, "RUN INCOMPLETE:"), + ("env:fixture-collision:customers:Acme Corp", "C1", set(), 1, "RUN INCOMPLETE:"), + ("env:new-skip-reason", "R1", set(), 1, "RUN INCOMPLETE:"), + ], +) +def test_run_live_completeness_skip_taxonomy( + tmp_path, monkeypatch, capsys, reason, task_id, needs, expected_rc, verdict +): + out = tmp_path / "out.jsonl" + + def skip_seed(*_args, **_kwargs): + raise TaskSkipped(reason) + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", skip_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: MagicMock()) + tasks = [_taxonomy_task(task_id, None, needs=needs)] + + rc = asyncio.run(run_live(tasks, model_alias="standard", reps=1, label="local", out_path=out)) + + assert rc == expected_rc + output = capsys.readouterr().out + assert "success: 0/0" in output + assert "EXECUTION COVERAGE: 0/1 rows evaluated (0.0%)" in output + assert f"{task_id} ({reason})" in output + assert verdict in output + if reason.startswith("env:fixture-collision:"): + assert "unexpected skips=1 [fixture-collision=1]" in output + + +def test_persisted_seed_artifacts_never_contain_run_random_truth_or_entity_ids(tmp_path, monkeypatch): + out = tmp_path / "forensics.jsonl" + namespace = "C2.release" + seeded_values: dict[str, str] = {} + + def seed_forensics(_plane, run_id, needs, ctx, task_id=None): + del needs, task_id + ctx["run_id"] = run_id + sentinel = random_truth_token(ctx, namespace) + seeded_values.update(run_id=run_id, sentinel=sentinel) + ctx.update( + { + "project_name": "EVAL forensic", + "project_id": "project-1", + "evidence_sentinels": {TARGET_ENTITY_EVIDENCE: [sentinel]}, + "evidence_targets": {TARGET_ENTITY_EVIDENCE: ["item-1"]}, + } + ) + record_seeded_entity(ctx, "work_item", "item-1") + record_seeded_entity(ctx, "project", "project-1") + record_randomized_truth( + ctx, + namespace, + { + "intended_name": f"1.6.10-eval.{sentinel[:8]}", + "intended_changelog": f"ticket EVAL-{sentinel}", + }, + ) + + class InspectingDriver: + def run_task(self, *_args, **kwargs): + assert _data_rows(out) == [] + assert kwargs["evidence_sentinels"] == {TARGET_ENTITY_EVIDENCE: [seeded_values["sentinel"]]} + assert kwargs["artifact_dir"] == tmp_path / "forensics.artifacts" / "api" + return AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") + + async def verify_ok(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", seed_forensics) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: InspectingDriver()) + task = { + "id": "R1", + "prompt": "do {project}", + "tags": {"read"}, + "needs": {"items"}, + "verify": verify_ok, + } + + assert asyncio.run(run_live([task], model_alias="standard", reps=1, label="local", out_path=out)) == 0 + row = _data_rows(out)[0] + assert row["fixture_seed_id"] == seeded_values["run_id"] + assert row["run_id"] != row["fixture_seed_id"] + assert row["seeded_entity_kinds"] == ["project", "work_item"] + assert row["randomized_seed_namespaces"] == [namespace] + persisted = json.dumps(row, sort_keys=True) + assert seeded_values["sentinel"] not in persisted + assert "project-1" not in persisted + assert "item-1" not in persisted + assert "seeded_entity_ids" not in row + assert "randomized_seed_choices" not in row + assert "evidence_sentinels" not in row + assert "evidence_targets" not in row + + +def test_same_run_repetitions_produce_different_random_truth_sentinels(tmp_path, monkeypatch): + out = tmp_path / "per-repetition-truth.jsonl" + namespace = "R7.states" + seeded: list[tuple[str, str]] = [] + + def seed_with_sentinel(_plane, run_id, needs, ctx, task_id=None): + del needs, task_id + ctx.update({"run_id": run_id, "project_name": f"EVAL {run_id[:8]}", "project_id": run_id}) + sentinel = random_truth_token(ctx, namespace) + seeded.append((run_id, sentinel)) + ctx["evidence_sentinels"] = {TARGET_ENTITY_EVIDENCE: [sentinel]} + ctx["evidence_targets"] = {TARGET_ENTITY_EVIDENCE: [run_id]} + + class OkDriver: + def run_task(self, *_args, **_kwargs): + return AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") + + async def verify_ok(_plane, _ctx, _run): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (object(), "ws")) + monkeypatch.setattr(runner_live, "seed", seed_with_sentinel) + monkeypatch.setattr(runner_live, "teardown", lambda *args, **kwargs: None) + monkeypatch.setattr(runner_live, "get_driver", lambda *args, **kwargs: OkDriver()) + task = { + "id": "R7", + "prompt": "do {project}", + "tags": {"read"}, + "needs": {"items"}, + "verify": verify_ok, + } + + assert asyncio.run(run_live([task], model_alias="standard", reps=2, label="local", out_path=out)) == 0 + rows = _data_rows(out) + assert len({row["run_id"] for row in rows}) == 1 + assert [row["fixture_seed_id"] for row in rows] == [seed_id for seed_id, _ in seeded] + assert seeded[0][0] != seeded[1][0] + assert seeded[0][1] != seeded[1][1] + + +def test_run_live_cleanup_failure_is_incomplete_without_changing_success(tmp_path, monkeypatch, capsys): + out = tmp_path / "out.jsonl" + delete_calls: list[tuple[str, str]] = [] + + def fail_delete(kind: str, object_id: str) -> None: + delete_calls.append((kind, object_id)) + raise RuntimeError(f"delete failed for {kind} {object_id}") + + class _Page: + results: list[Any] = [] + next_page_results = False + next_cursor = None + + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page(), + delete=lambda **kw: fail_delete("customer", kw["customer_id"]), + properties=SimpleNamespace(list=lambda **kw: _Page(), delete=lambda **kw: None), + ), + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page(), + delete=lambda **kw: fail_delete("release_tag", kw["tag_id"]), + ) + ), + ) + driver = MagicMock() + driver.run_task.return_value = AgentRun(calls=[], final_text="done", usage=None, stopped_reason="end_turn") + + async def verify_ok(*_args, **_kwargs): + return True, "ok" + + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (plane, "ws")) + monkeypatch.setattr( + runner_live, + "seed", + lambda *a, **k: k["ctx"].update( + { + "workspace_slug": "ws", + "project_name": "EVAL cleanup", + "project_id": None, + "workspace_objects": [ + {"kind": "customer", "id": "customer-1"}, + {"kind": "release_tag", "id": "tag-1"}, + ], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + } + ), + ) + monkeypatch.setattr(runner_live, "get_driver", lambda *a, **k: driver) + + rc = asyncio.run( + run_live( + [_taxonomy_task("R1", verify_ok)], + model_alias="standard", + reps=1, + label="local", + out_path=out, + ) + ) + + assert rc == 1 + row = _data_rows(out)[0] + assert row["success"] is True + assert row["cleanup_error"].startswith("TeardownError: 2 cleanup operation(s) failed:") + assert delete_calls == [("customer", "customer-1"), ("release_tag", "tag-1")] + output = capsys.readouterr().out + assert "success: 100.0% across 1 tasks (cluster-bootstrap95 [1.00,1.00]; pooled repetitions 1/1)" in output + assert "EXECUTION COVERAGE: 1/1 rows evaluated (100.0%)" in output + assert "RUN INCOMPLETE:" in output + assert "cleanup errors=1" in output + + +# --------------------------------------------------------------------------- +# Progress reporting +# --------------------------------------------------------------------------- + + +def test_elapsed_formats_minutes_then_hours(monkeypatch): + from evals.runner.live import _elapsed + + clock = {"now": 1000.0} + monkeypatch.setattr(runner_live.time, "monotonic", lambda: clock["now"]) + clock["now"] = 1000.0 + 9 + assert _elapsed(1000.0) == "00:09" + clock["now"] = 1000.0 + 75 + assert _elapsed(1000.0) == "01:15" + clock["now"] = 1000.0 + 3671 + assert _elapsed(1000.0) == "1:01:11" + + +@pytest.mark.parametrize( + ("tags", "calls", "expect_pass", "why"), + [ + pytest.param({"write"}, [], False, "mutation with no tool call", id="write-with-zero-calls"), + pytest.param({"write"}, [{"tool": "cycle"}], True, "mutation through the surface", id="write-with-calls"), + # An errored call is not use of the surface. + pytest.param( + {"write"}, [{"tool": "cycle", "is_error": True}], False, "only failed calls", id="write-all-errored" + ), + # Non-write tasks are gated by answer_with_provenance instead; do not double-charge them. + pytest.param({"schema"}, [], True, "not a write task", id="non-write-untouched"), + ], +) +def test_a_write_task_must_change_plane_through_the_surface(monkeypatch, tmp_path, tags, calls, expect_pass, why): + """An off-surface mutation verifies as correct state; it must not score as a pass. + + Measured: an agent read the repo it was standing in, took the API key from its own + environment and mutated Plane over REST. The verifier read the state back, found it + correct, and the row passed with zero tool calls recorded. + """ + from evals.runner import live as runner_live + + out = tmp_path / "out.jsonl" + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (MagicMock(), "test-ws")) + + def ok_seed(plane, run_id, needs, ctx, task_id=None): + ctx.update({"project_name": "EVAL surface", "project_id": "p1"}) + + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda plane, ctx: None) + + class SurfacelessDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=list(calls), + final_text="done", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + trace_integrity=True, + ) + + async def verify(*args, **kwargs): + return True, "state is correct" + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: SurfacelessDriver()) + task = {"id": "T9", "prompt": "mutate {project}", "tags": tags, "needs": set(), "verify": verify} + asyncio.run(run_live([task], model_alias="haiku", reps=1, label="l", out_path=out, driver_name="claude-cli")) + row = [json.loads(line) for line in out.read_text().splitlines() if json.loads(line).get("row_type") != "meta"][0] + assert row["success"] is expect_pass, f"{why}: {row.get('verify_note') or row.get('error')}" + if not expect_pass: + assert "surface=missing" in row["verify_note"], row["verify_note"] diff --git a/tests/evals/runner/test_resume.py b/tests/evals/runner/test_resume.py new file mode 100644 index 0000000..5513c4e --- /dev/null +++ b/tests/evals/runner/test_resume.py @@ -0,0 +1,429 @@ +"""Offline eval tests for resume.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from evals.core.results import AgentRun +from evals.report import ( + is_meta_row, +) +from evals.runner import ( + is_meta_or_non_task_row, + load_resume_skip_keys, + make_run_meta_row, + maybe_write_run_meta, + run_live, + should_skip_resume_row, +) +from evals.runner import live as runner_live +from tests.evals.conftest import _data_rows, case_params + + +def _should_skip_resume_row_completed_success(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": True}) is True + + +def _should_skip_resume_row_verify_fail_without_error(): + assert should_skip_resume_row({"error": None, "error_class": None, "success": False}) is True + + +def _should_skip_resume_row_infra_seed_retries(): + assert should_skip_resume_row({"error": "HttpError: 409", "error_class": "infra_seed"}) is False + + +def _should_skip_resume_row_infra_cli_retries(): + assert should_skip_resume_row({"error": "timeout after 120s", "error_class": "infra_cli"}) is False + + +def _should_skip_resume_row_non_null_error_retries(): + assert should_skip_resume_row({"error": "TypeError: x", "error_class": "task"}) is False + assert should_skip_resume_row({"error": "boom", "error_class": None}) is False + + +def _should_skip_resume_row_only_plan_gated_skip_is_terminal(): + assert should_skip_resume_row({"task_id": "L4", "skipped": "env:plan-gated:customers"}) is True + assert should_skip_resume_row({"task_id": "L2", "skipped": "env:no-activity-worker"}) is True + assert should_skip_resume_row({"task_id": "W1", "skipped": "env:plan-gated:customers"}) is False + assert should_skip_resume_row({"skipped": "env:no-activity-worker (worker disabled)"}) is False + assert should_skip_resume_row({"skipped": "env:plan-gated:customerz"}) is False + assert should_skip_resume_row({"skipped": "env:fixture-collision:customers:Acme Corp"}) is False + assert should_skip_resume_row({"skipped": "env:unknown"}) is False + assert should_skip_resume_row({"cleanup_error": "TeardownError: delete failed"}) is False + + +@pytest.mark.parametrize( + "case", + case_params( + _should_skip_resume_row_completed_success, + _should_skip_resume_row_verify_fail_without_error, + _should_skip_resume_row_infra_seed_retries, + _should_skip_resume_row_infra_cli_retries, + _should_skip_resume_row_non_null_error_retries, + _should_skip_resume_row_only_plan_gated_skip_is_terminal, + ), +) +def test_should_skip_behaviours(case): + case() + + +def _load_resume_skip_keys_summary(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None}, + {"task_id": "R1", "rep": 1, "label": "local", "error": "x", "error_class": "infra_seed"}, + {"task_id": "W1", "rep": 0, "label": "local", "error": None, "success": False}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local"), ("W1", 0, "local")} + assert n_skip == 2 + assert n_retry == 1 + + +def _load_resume_skip_keys_n_retry_ignores_later_success(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + rows = [ + {"task_id": "R1", "rep": 0, "label": "local", "error": "boom", "error_class": "infra_cli"}, + {"task_id": "R1", "rep": 0, "label": "local", "error": None, "error_class": None, "success": True}, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + assert n_retry == 0 + + +def _load_resume_skip_keys_label_mismatch(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + p.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "other", "error": None}) + "\n") + with pytest.raises(SystemExit, match="label"): + load_resume_skip_keys(p, label="local") + + +def _load_resume_skip_keys_battery_model_driver_mismatch(tmp_path, _capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "battery": "aaaaaaaaaaaa", + "model": "sonnet", + "driver": "claude-cli", + "error": None, + } + ) + + "\n", + encoding="utf-8", + ) + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="local", battery="bbbbbbbbbbbb") + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="haiku") + with pytest.raises(SystemExit, match="driver"): + load_resume_skip_keys(p, label="local", battery="aaaaaaaaaaaa", model="sonnet", driver="unknown") + # Missing keys on older rows: pass (back-compat) + p2 = tmp_path / "old.jsonl" + p2.write_text(json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + "\n") + skip, _, _ = load_resume_skip_keys(p2, label="local", battery="anything", model="sonnet", driver="claude-cli") + assert ("R1", 0, "local") in skip + + +def _load_resume_skip_keys_truncated_json(tmp_path, capsys): + p = tmp_path / "out.jsonl" + p.write_text( + json.dumps({"task_id": "R1", "rep": 0, "label": "local", "error": None}) + + "\n" + + '{"task_id": "W1", "rep": 0, "label": "local", "error":\n', # truncated + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys(p, label="local") + assert skip == {("R1", 0, "local")} + assert n_skip == 1 + err = capsys.readouterr().err + assert "invalid JSON" in err + + +def _load_resume_skip_keys_missing_file(tmp_path, _capsys): + skip, n_skip, n_retry = load_resume_skip_keys(tmp_path / "missing.jsonl", label="local") + assert skip == set() and n_skip == 0 and n_retry == 0 + + +@pytest.mark.parametrize( + "case", + case_params( + _load_resume_skip_keys_summary, + _load_resume_skip_keys_n_retry_ignores_later_success, + _load_resume_skip_keys_label_mismatch, + _load_resume_skip_keys_battery_model_driver_mismatch, + _load_resume_skip_keys_truncated_json, + _load_resume_skip_keys_missing_file, + ), +) +def test_load_behaviours(case, tmp_path, capsys): + case(tmp_path, capsys) + + +def _resume_identity_uses_resolved_model_not_tier_label(tmp_path): + p = tmp_path / "tiered.jsonl" + p.write_text( + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "local", + "model": "provider-reported-id", + "requested_model": "standard", + "requested_tier": "standard", + "resolved_model": "old-standard-id", + "error": None, + } + ) + + "\n", + encoding="utf-8", + ) + + skip, _, _ = load_resume_skip_keys(p, label="local", model="old-standard-id") + assert skip == {("R1", 0, "local")} + with pytest.raises(SystemExit, match="model"): + load_resume_skip_keys(p, label="local", model="new-standard-id") + + +def _resume_skips_meta_and_mismatch_checks_it(tmp_path): + p = tmp_path / "out.jsonl" + p.write_text( + "\n".join( + [ + json.dumps( + { + "row_type": "meta", + "label": "candidate", + "battery": "bbbbbbbbbbbb", + "model": "sonnet", + "driver": "claude-cli", + } + ), + json.dumps( + { + "task_id": "R1", + "rep": 0, + "label": "candidate", + "error": None, + "error_class": None, + "success": True, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + skip, n_skip, n_retry = load_resume_skip_keys( + p, label="candidate", battery="bbbbbbbbbbbb", model="sonnet", driver="claude-cli" + ) + assert skip == {("R1", 0, "candidate")} + assert n_skip == 1 and n_retry == 0 + + with pytest.raises(SystemExit, match="battery"): + load_resume_skip_keys(p, label="candidate", battery="aaaaaaaaaaaa", model="sonnet", driver="claude-cli") + + +@pytest.mark.parametrize( + "case", + case_params(_resume_identity_uses_resolved_model_not_tier_label, _resume_skips_meta_and_mismatch_checks_it), +) +def test_resume_behaviours(case, tmp_path): + case(tmp_path) + + +def test_run_live_resume_retries_infra_and_unexpected_skips_but_not_plan_gates(tmp_path: Path, monkeypatch): + out = tmp_path / "resume.jsonl" + # Pre-write a completed result, infra error, expected skip, and unexpected skip. + # Battery is computed from the task list below — seed the file after we know it, + # or write rows without battery (back-compat) and only check skip/retry behavior. + prior = [ + { + "task_id": "R1", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "error": None, + "error_class": None, + "success": True, + }, + { + "task_id": "R2", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "error": "HttpError: 409", + "error_class": "infra_seed", + "success": False, + }, + { + "task_id": "L4", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "skipped": "env:plan-gated:customers", + }, + { + "task_id": "C2", + "rep": 0, + "label": "local", + "driver": "claude-cli", + "model": "sonnet", + "skipped": "env:fixture-collision:release_tags:eval-rc1", + }, + ] + out.write_text("\n".join(json.dumps(r) for r in prior) + "\n", encoding="utf-8") + original_bytes = out.read_bytes() + original_write_text = Path.write_text + + def reject_results_rewrite(path, *args, **kwargs): + if path == out: + raise AssertionError("resume must not rewrite its append-only results file") + return original_write_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", reject_results_rewrite) + + fake_plane = MagicMock() + monkeypatch.setattr(runner_live, "make_plane_client", lambda: (fake_plane, "test-ws")) + seed_calls: list[str] = [] + + def ok_seed(plane, run_id, needs, ctx, task_id=None): + # Infer task from empty ctx; runner sets project for verify path. + ctx.update({"project_name": "EVAL resume", "project_id": "p1"}) + seed_calls.append(str(task_id)) + + monkeypatch.setattr(runner_live, "seed", ok_seed) + monkeypatch.setattr(runner_live, "teardown", lambda *a, **k: None) + + class OkDriver: + name = "claude-cli" + + def run_task(self, *args, **kwargs): + return AgentRun( + calls=[{"tool": "list_work_items", "args": {}, "origin": "plane"}], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + + monkeypatch.setattr(runner_live, "get_driver", lambda name, **kw: OkDriver()) + + async def verify_ok(plane, ctx, run): + return True, "ok" + + tasks = [ + { + "id": "R1", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + }, + { + "id": "R2", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + }, + { + "id": "L4", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + }, + { + "id": "C2", + "prompt": "do {project}", + "tags": set(), + "needs": set(), + "verify": verify_ok, + }, + ] + + rc = asyncio.run( + run_live( + tasks, + model_alias="sonnet", + reps=1, + label="local", + out_path=out, + driver_name="claude-cli", + resume=True, + ) + ) + assert rc == 0 + # R1 completed and L4 was plan-gated. R2 infra and C2 collision are retried. + assert seed_calls == ["R2", "C2"] + assert out.read_bytes().startswith(original_bytes) + data = _data_rows(out) + # Resume is append-only: both retryable failures remain before their replacements. + assert len(data) == 6 + new_r2, new_c2 = data[-2:] + assert new_r2["task_id"] == "R2" + assert new_r2["success"] is True + assert new_r2["error_class"] is None + assert new_r2["final_text"] == "done" + assert new_c2["task_id"] == "C2" + assert new_c2["success"] is True + + +def test_make_run_meta_row_and_write_once(tmp_path: Path): + path = tmp_path / "out.jsonl" + meta = make_run_meta_row( + run_id="rid", + label="candidate", + server="local", + battery="abcd1234ef00", + model="sonnet", + driver="claude-cli", + git_sha="deadbeef", + expected_task_ids=["R1", "W1"], + expected_reps=3, + ts="2026-01-01T00:00:00+00:00", + ) + assert meta["row_type"] == "meta" + assert meta["expected_task_ids"] == ["R1", "W1"] + assert meta["expected_reps"] == 3 + assert is_meta_row(meta) + assert is_meta_or_non_task_row(meta) + assert maybe_write_run_meta(path, meta) is True + # Append a data row — a truncating rewrite on the second call would destroy it. + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({"task_id": "R1", "rep": 0, "label": "candidate", "success": True}) + "\n") + assert maybe_write_run_meta(path, meta) is False # file non-empty + lines = path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["row_type"] == "meta" + assert json.loads(lines[1])["task_id"] == "R1" + + +def test_make_run_meta_row_rejects_count_disagreement_with_exact_keys(): + with pytest.raises(ValueError, match="exact expectation=4"): + make_run_meta_row( + run_id="rid", + label="candidate", + server="local", + battery="abcd1234ef00", + model="sonnet", + driver="api", + git_sha="deadbeef", + expected_rows=3, + expected_task_ids=["R1", "W1"], + expected_reps=2, + ) diff --git a/tests/evals/seed/__init__.py b/tests/evals/seed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/seed/test_gate_tolerance.py b/tests/evals/seed/test_gate_tolerance.py new file mode 100644 index 0000000..3e0ec2a --- /dev/null +++ b/tests/evals/seed/test_gate_tolerance.py @@ -0,0 +1,219 @@ +"""Seeding a plan-gated capability skips the task instead of failing the run. + +`DESIGN.md` states a plan gate is not rewritten as an agent task failure. Before this, +only the work item type seeder honoured it; a gate while seeding a release or customer +raised, became `infra_seed`, and killed the task-rep. That is what made a flag server +answering everything "on" a hard prerequisite. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.seed import seed_customer, seed_release +from evals.tasks.skip import TaskSkipped + +PLAN_REFUSAL = HttpError("Payment required", 402, {"error": "Payment required", "error_code": 1999}) +RBAC_REFUSAL = HttpError("Forbidden", 403, {"detail": "You don't have permission to do this"}) + + +def _raising_client(exc: Exception) -> SimpleNamespace: + def _boom(**_kwargs: Any): + raise exc + + return SimpleNamespace( + releases=SimpleNamespace(create=_boom, changelog=SimpleNamespace(update=_boom)), + customers=SimpleNamespace(create=_boom, requests=SimpleNamespace(create=_boom)), + ) + + +@pytest.mark.parametrize( + ("seeder", "feature"), + [(seed_release, "releases"), (seed_customer, "customers")], +) +def test_plan_gate_becomes_a_skip_with_a_reason(seeder, feature): + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(TaskSkipped) as caught: + seeder(_raising_client(PLAN_REFUSAL), "ws", context) + assert caught.value.reason == f"env:plan-gated:{feature}" + + +@pytest.mark.parametrize("seeder", [seed_release, seed_customer]) +def test_a_non_gate_failure_still_raises(seeder): + """Only plan limits are excused. A permission or transport failure is a real error. + + Swallowing these would let the harness report a clean battery while the fixtures it + graded against were never built. + """ + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(HttpError): + seeder(_raising_client(RBAC_REFUSAL), "ws", context) + + +@pytest.mark.parametrize("seeder", [seed_release, seed_customer]) +def test_transport_failures_are_not_excused(seeder): + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(RuntimeError): + seeder(_raising_client(RuntimeError("connection reset")), "ws", context) + + +def test_customer_gate_does_not_leave_a_half_built_fixture(): + """A gate on the follow-up request must not leave a customer recorded as seeded. + + The customer is created, then its request is refused. Recording the customer while + the task skips would leave a verifier reading a fixture that was never finished. + """ + created: list[str] = [] + + def _create_customer(**_kwargs: Any): + created.append("customer") + return SimpleNamespace(id="cust-1") + + def _refuse(**_kwargs: Any): + raise PLAN_REFUSAL + + plane = SimpleNamespace( + customers=SimpleNamespace( + create=_create_customer, + requests=SimpleNamespace(create=_refuse), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(TaskSkipped): + seed_customer(plane, "ws", context) + + assert created == ["customer"] + assert "customer_request" not in context + + +def test_release_changelog_write_behaviours(): + cases = ( + ("write failure is fatal", RuntimeError("connection reset"), RuntimeError, None), + ("plan gate remains a skip", PLAN_REFUSAL, TaskSkipped, "env:plan-gated:releases"), + ) + for label, error, expected_error, expected_reason in cases: + with pytest.MonkeyPatch.context(): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda error=error, **kw: (_ for _ in ()).throw(error), + ), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + with pytest.raises(expected_error) as caught: + seed_release(plane, "ws", context) + + if expected_reason is not None: + assert caught.value.reason == expected_reason, label + assert context["release"]["id"] == "release-1", label + assert context["workspace_objects"] == [{"kind": "release", "id": "release-1"}], label + assert "release_changelog_text" not in context, label + + +def test_release_changelog_readback_sets_the_api_confirmed_baseline(): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda **kw: None, + retrieve=lambda **kw: SimpleNamespace( + description_html="

Changelog entry one: API-confirmed fact.

", + ), + ), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + + seed_release(plane, "ws", context) + + assert context["release_changelog_text"] == "Changelog entry one: API-confirmed fact." + + +def test_c2_release_truth_is_randomized_and_api_confirmed(): + contexts: list[dict[str, Any]] = [] + for run_id in ("c2000000aaaaaaaa", "c2000000bbbbbbbb"): + stored: dict[str, str] = {} + + def create(*, data, _stored=stored, _run_id=run_id, **kwargs): + _stored["name"] = data.name + return SimpleNamespace(id=f"release-{_run_id[-4:]}", name=data.name) + + def update(*, data, _stored=stored, **kwargs): + _stored["html"] = data.description_html + + def retrieve(*, _stored=stored, **kwargs): + return SimpleNamespace(description_html=_stored["html"]) + + plane = SimpleNamespace( + releases=SimpleNamespace( + create=create, + changelog=SimpleNamespace( + update=update, + retrieve=retrieve, + ), + ) + ) + context: dict[str, Any] = { + "run_id": run_id, + "task_id": "C2", + "workspace_objects": [], + "randomized_truth": {}, + } + seed_release(plane, "ws", context) + contexts.append(context) + + assert context["release"]["name"] == stored["name"] + assert context["randomized_truth"]["C2.release"]["confirmed"]["changelog"] == context["release_changelog_text"] + assert context["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + assert contexts[0]["release_name"] != contexts[1]["release_name"] + assert contexts[0]["release_changelog_text"] != contexts[1]["release_changelog_text"] + + +@pytest.mark.parametrize( + ("error", "expected_error", "expected_reason"), + [ + (RuntimeError("readback failed"), RuntimeError, None), + (PLAN_REFUSAL, TaskSkipped, "env:plan-gated:releases"), + ], +) +def test_release_changelog_readback_failures(error, expected_error, expected_reason): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda **kw: None, + retrieve=lambda **kw: (_ for _ in ()).throw(error), + ), + ) + ) + context: dict[str, Any] = {"workspace_objects": []} + + with pytest.raises(expected_error) as caught: + seed_release(plane, "ws", context) + + if expected_reason is not None: + assert caught.value.reason == expected_reason + assert "release_changelog_text" not in context + + +def test_empty_release_changelog_readback_is_a_seed_failure(): + plane = SimpleNamespace( + releases=SimpleNamespace( + create=lambda **kw: SimpleNamespace(id="release-1"), + changelog=SimpleNamespace( + update=lambda **kw: None, + retrieve=lambda **kw: SimpleNamespace(description_html="

"), + ), + ) + ) + + with pytest.raises(RuntimeError, match="readback was empty after seeding"): + seed_release(plane, "ws", {"workspace_objects": []}) diff --git a/tests/evals/seed/test_plan_gate.py b/tests/evals/seed/test_plan_gate.py new file mode 100644 index 0000000..886a6ba --- /dev/null +++ b/tests/evals/seed/test_plan_gate.py @@ -0,0 +1,90 @@ +"""Characterization of `is_plan_gate` against the payloads `api/` actually returns. + +A gate becomes an environment skip and anything else stays a real error, so a +a wrong category here either hides a defect or invents one. +""" + +from __future__ import annotations + +from plane.errors.errors import HttpError + +from evals.seed import is_plan_gate + +# --- genuine plan refusals ------------------------------------------------------------ + +PLAN_GATES = [ + ( + HttpError("Payment required", 402, {"error": "Payment required", "error_code": 1999}), + "402-check_feature_flag-decorator", + ), + (HttpError("Payment required", 402, None), "402-with-no-body"), + ( + HttpError( + "Forbidden", + 403, + {"detail": "Payment required. Upgrade your plan to access Initiatives"}, + ), + "403-initiatives-permission-class", + ), + ( + HttpError( + "Forbidden", + 403, + {"detail": "Payment required. Upgrade your plan to access Teamspaces"}, + ), + "403-teamspaces-permission-class", + ), + (HttpError("Bad request", 400, {"error": "Upgrade your plan to enable formula properties"}), "400-with-plan-prose"), +] + +# --- refusals that are NOT plan limits ------------------------------------------------- + +NOT_PLAN_GATES = [ + ( + HttpError("Forbidden", 403, {"detail": "You don't have permission to create this project"}), + "bare-403-is-rbac-not-a-plan-limit", + ), + ( + HttpError( + "Forbidden", + 403, + {"error": "Customer feature is not enabled for this workspace"}, + ), + "403-customer-toggle-is-configuration-the-harness-controls", + ), + (HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}), "404-worklog-toggle"), + ( + HttpError("Bad request", 400, {"non_field_errors": ["Cycles are not enabled for this project"]}), + "400-cycle-toggle", + ), + ( + HttpError("Bad request", 400, {"non_field_errors": ["Modules are not enabled for this project"]}), + "400-module-toggle", + ), + (HttpError("Bad request", 400, {"name": ["This field is required."]}), "400-ordinary-validation-error"), + (HttpError("Server error", 500, {"error": "Internal server error"}), "500-never-a-gate"), + (HttpError("Too many requests", 429, {"error": "Rate limit exceeded"}), "429-never-a-gate"), +] + + +def test_only_refusals_that_name_a_plan_limit_are_gates(): + """402 is unambiguous; 403/400 need the body to say so, since 403 is also plain RBAC.""" + for exc, label in PLAN_GATES: + assert is_plan_gate(exc) is True, label + for exc, label in NOT_PLAN_GATES: + assert is_plan_gate(exc) is False, label + + +def test_non_http_exceptions_are_never_gates(): + """A transport failure is infrastructure, not a plan limit to be excused.""" + for exc in (RuntimeError("connection reset"), TimeoutError(), ValueError("upgrade your plan")): + assert is_plan_gate(exc) is False, repr(exc) + + +def test_a_bare_403_would_previously_have_been_swallowed(): + """The regression this exists for: RBAC denial and a plan gate share status and shape.""" + rbac = HttpError("Forbidden", 403, {"detail": "You don't have permission to view this issue"}) + gate = HttpError("Forbidden", 403, {"detail": "Payment required. Upgrade your plan to access Initiatives"}) + assert rbac.status_code == gate.status_code + assert is_plan_gate(rbac) is False + assert is_plan_gate(gate) is True diff --git a/tests/evals/seed/test_read_randomization.py b/tests/evals/seed/test_read_randomization.py new file mode 100644 index 0000000..019278a --- /dev/null +++ b/tests/evals/seed/test_read_randomization.py @@ -0,0 +1,273 @@ +"""Focused seed/readback tests for per-row read-task truth.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels +from evals.seed.cycles import seed_cycles +from evals.seed.states import seed_r7_state_oracle +from evals.seed.work_items import require_activities, seed_work_items + + +class _Page: + def __init__(self, results: list[Any]): + self.results = results + self.next_page_results = False + self.next_cursor = None + + +class _ReadSeedPlane: + def __init__(self): + self._states = [ + SimpleNamespace(id="state-started", name="In Progress", group="started", default=False), + SimpleNamespace(id="state-done", name="Done", group="completed", default=False), + ] + self._items: dict[str, SimpleNamespace] = {} + self._comments: dict[str, list[SimpleNamespace]] = {} + self._attachments: dict[str, list[SimpleNamespace]] = {} + self._cycles: dict[str, SimpleNamespace] = {} + self._cycle_items: dict[str, list[str]] = {} + self.states = SimpleNamespace( + list=lambda **kwargs: _Page(list(self._states)), + create=self._create_state, + retrieve=self._retrieve_state, + ) + self.users = SimpleNamespace(get_me=lambda: SimpleNamespace(id="user-me")) + self.work_items = SimpleNamespace( + create=self._create_item, + update=self._update_item, + retrieve=self._retrieve_item, + list=lambda **kwargs: _Page(list(self._items.values())), + comments=SimpleNamespace(create=self._create_comment, list=self._list_comments), + activities=SimpleNamespace(list=self._list_activities), + attachments=SimpleNamespace(upload_from_bytes=self._upload_attachment, list=self._list_attachments), + ) + self._work_logs: dict[str, list[SimpleNamespace]] = {} + self.work_items.work_logs = SimpleNamespace(create=self._create_work_log) + self.projects = SimpleNamespace(get_worklog_summary=self._worklog_summary) + self.cycles = SimpleNamespace( + create=self._create_cycle, + update=self._update_cycle, + retrieve=self._retrieve_cycle, + add_work_items=self._add_cycle_items, + list_work_items=self._list_cycle_items, + ) + + def _create_work_log(self, *, work_item_id, data, **kwargs): + # The SDK resource takes a mapping here, so the fake must reject a model. + assert isinstance(data, dict), f"work_logs.create needs a mapping, got {type(data).__name__}" + log = SimpleNamespace(id=f"log-{work_item_id}", duration=data["duration"], description=data["description"]) + self._work_logs.setdefault(str(work_item_id), []).append(log) + return log + + def _worklog_summary(self, **kwargs): + return [SimpleNamespace(issue_id=item_id) for item_id in sorted(self._work_logs)] + + def _create_state(self, *, data, **kwargs): + state = SimpleNamespace( + id=f"state-{len(self._states) + 1}", + name=data.name, + group=data.group, + default=False, + ) + self._states.append(state) + return state + + def _retrieve_state(self, *, state_id, **kwargs): + return next(state for state in self._states if state.id == state_id) + + def _create_item(self, *, data, **kwargs): + work_item_id = f"item-{len(self._items) + 1}" + item = SimpleNamespace( + id=work_item_id, + sequence_id=len(self._items) + 1, + name=data.name, + priority=data.priority, + state=data.state or "state-started", + target_date=data.target_date, + assignees=list(data.assignees or []), + ) + self._items[work_item_id] = item + return item + + def _update_item(self, *, work_item_id, data, **kwargs): + item = self._items[work_item_id] + for key, value in data.model_dump(exclude_none=True).items(): + setattr(item, key, value) + return item + + def _retrieve_item(self, *, work_item_id, **kwargs): + return self._items[work_item_id] + + def _create_comment(self, *, work_item_id, data, **kwargs): + rows = self._comments.setdefault(work_item_id, []) + comment = SimpleNamespace( + id=f"comment-{len(rows) + 1}", + comment_html=data.comment_html, + comment_stripped=None, + ) + rows.append(comment) + return comment + + def _list_comments(self, *, work_item_id, **kwargs): + return _Page(list(self._comments.get(work_item_id, []))) + + def _list_activities(self, *, work_item_id, **kwargs): + rows = [ + SimpleNamespace(id=f"activity-{row.id}", comment=row.comment_html) + for row in self._comments.get(work_item_id, []) + ] + return _Page(rows) + + def _upload_attachment(self, *, work_item_id, name, **kwargs): + rows = self._attachments.setdefault(work_item_id, []) + attachment = SimpleNamespace(id=f"attachment-{len(rows) + 1}", name=name) + rows.append(attachment) + return attachment + + def _list_attachments(self, *, work_item_id, **kwargs): + return _Page(list(self._attachments.get(work_item_id, []))) + + def _create_cycle(self, *, data, **kwargs): + cycle_id = f"cycle-{len(self._cycles) + 1}" + cycle = SimpleNamespace(id=cycle_id, name=data.name, end_date=data.end_date) + self._cycles[cycle_id] = cycle + self._cycle_items[cycle_id] = [] + return cycle + + def _update_cycle(self, *, cycle_id, data, **kwargs): + cycle = self._cycles[cycle_id] + cycle.end_date = data.end_date + return cycle + + def _retrieve_cycle(self, *, cycle_id, **kwargs): + return self._cycles[cycle_id] + + def _add_cycle_items(self, *, cycle_id, issue_ids, **kwargs): + self._cycle_items[cycle_id].extend(str(value) for value in issue_ids) + + def _list_cycle_items(self, *, cycle_id, **kwargs): + return _Page([SimpleNamespace(work_item_id=value) for value in self._cycle_items[cycle_id]]) + + +def _context(task_id: str, run_id: str) -> dict[str, Any]: + return { + "run_id": run_id, + "run8": run_id[:8], + "task_id": task_id, + "project_id": "project-1", + "project_identifier": "EVTEST", + "items": {}, + "item_ids": [], + "item_identifiers": {}, + "fixture_item_ids": {}, + "fixture_item_titles": {}, + "randomized_truth": {}, + } + + +@pytest.mark.parametrize( + ("task_id", "oracle_key", "truth_key"), + [ + ("R1", "r1_state_name", "R1.state"), + ("R2", "r2_urgent_open_count", "R2.urgent_open_count"), + ("R3", "r3_due_titles", "R3.due_templates"), + ("R5", "r5_comment_phrases", "R5.comments"), + ("I2", "i2_state_name", "I2.state"), + ("L2", "l2_activity_count", "L2.activity_count"), + ("L5", "l5_attachment_count", "L5.attachment_count"), + ], +) +def test_work_item_read_truth_is_randomized_and_api_confirmed(task_id, oracle_key, truth_key): + plane = _ReadSeedPlane() + ctx = _context(task_id, f"{task_id.lower():0<8}0123456789abcdef") + seed_work_items(plane, "ws", ctx) + if task_id == "L2": + require_activities(plane, "ws", ctx) + + assert ctx[oracle_key] not in (None, "", []) + assert "confirmed" in ctx["randomized_truth"][truth_key] + if task_id == "L2": + # L2 binds the activity count, not a sentinel value: Plane's activity payload carries + # the creation row and never the seeded comment text (revision 8). + assert ctx["evidence_aggregates"][TARGET_ENTITY_EVIDENCE] == ( + {"kind": "total_count", "value": ctx[oracle_key]}, + ) + else: + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + # Whichever kind of evidence the task registered, the live seed gate must accept it. + assert configured_evidence_labels( + ctx.get("evidence_sentinels"), + ctx.get("evidence_targets"), + ctx.get("evidence_aggregates"), + ) == (TARGET_ENTITY_EVIDENCE,) + + +def test_r2_randomized_counts_differ_between_rows_after_api_readback(): + contexts = [] + for run_id in ("00000000aaaaaaaa", "11111111bbbbbbbb"): + plane = _ReadSeedPlane() + ctx = _context("R2", run_id) + seed_work_items(plane, "ws", ctx) + contexts.append(ctx) + + counts = [ctx["r2_urgent_open_count"] for ctx in contexts] + assert counts == [6, 3] + for ctx in contexts: + truth = ctx["randomized_truth"]["R2.urgent_open_count"] + assert truth["confirmed"] == ctx["r2_urgent_open_count"] + + +def test_r4_cycle_inventory_is_randomized_and_api_confirmed(): + plane = _ReadSeedPlane() + ctx = _context("R4", "44444444aaaaaaaa") + seed_work_items(plane, "ws", ctx) + seed_cycles(plane, "ws", ctx) + + truth = ctx["randomized_truth"]["R4.cycle_inventory"] + assert ctx["r4_cycle_name"].startswith("Sprint ") + assert ctx["r4_cycle_name"] != "Sprint 13" + assert ctx["r4_active_titles"] + assert ctx["r4_overdue_titles"] + assert truth["confirmed"] == { + "cycle": ctx["r4_cycle_name"], + "active_titles": ctx["r4_active_titles"], + "overdue_titles": ctx["r4_overdue_titles"], + } + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + +def test_r7_state_truth_is_randomized_api_confirmed_and_evidence_bearing(): + contexts: list[dict[str, Any]] = [] + for run_id in ("77777777aaaaaaaa", "88888888bbbbbbbb"): + plane = _ReadSeedPlane() + ctx = _context("R7", run_id) + seed_r7_state_oracle(plane, "ws", ctx) + contexts.append(ctx) + + assert contexts[0]["r7_state_pairs"] != contexts[1]["r7_state_pairs"] + for ctx in contexts: + truth = ctx["randomized_truth"]["R7.states"] + assert truth["confirmed"] == ctx["r7_state_pairs"] + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] + + +def test_l1_seed_oracle_is_the_api_confirmed_target_id(): + plane = _ReadSeedPlane() + ctx = _context("L1", "11111111cccccccc") + + seed_work_items(plane, "ws", ctx) + + target_id = ctx["fixture_item_ids"]["Payment webhook drops retries"] + # The oracle holds the agent's own row plus a row it was never told about, so reporting + # the id it already has is no longer the whole answer. + assert target_id in ctx["l1_expected_summary_ids"] + assert len(ctx["l1_expected_summary_ids"]) == 2 + seeded_id = next(i for i in ctx["l1_expected_summary_ids"] if i != target_id) + assert plane._work_logs[seeded_id] + # Provenance is the seeded row's id: the target's id is echoed by the agent's own write. + assert ctx["evidence_sentinels"][TARGET_ENTITY_EVIDENCE] == (seeded_id,) diff --git a/tests/evals/seed/test_seed.py b/tests/evals/seed/test_seed.py new file mode 100644 index 0000000..8286d93 --- /dev/null +++ b/tests/evals/seed/test_seed.py @@ -0,0 +1,1965 @@ +"""Offline eval tests for seed.""" + +from __future__ import annotations + +import asyncio +import inspect +import re +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +import pytest +from plane.errors.errors import HttpError + +from evals import cleanup as cleanup_mod +from evals import seed as seed_mod +from evals.core.errors import TaskSkipped +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.seed import ( + R5_TITLE, + create_project_with_collision_retry, + is_identifier_collision, + is_name_collision, + seed_plan, + seed_second_project, +) +from evals.seed import projects as projects_mod +from evals.tasks.debias import ( + L3_TAG_VERSION, + L4_PROP_DISPLAY, +) +from evals.tasks.read import verify_r6 +from tests.evals.conftest import case_params + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +class _TeardownPlane: + def __init__(self): + self.deleted: list[tuple[str, str]] = [] + self.releases = SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="tag-1", version=L3_TAG_VERSION)]), + delete=lambda **kw: self.deleted.append(("release_tag", kw["tag_id"])), + ), + delete=lambda **kw: self.deleted.append(("release", kw.get("release_id"))), + ) + self.customers = SimpleNamespace( + properties=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace( + id="prop-1", + display_name=L4_PROP_DISPLAY, + name="eval-industry", + ) + ] + ), + delete=lambda **kw: self.deleted.append(("customer_property", kw["property_id"])), + ), + list=lambda **kw: _Page([]), + delete=lambda **kw: None, + ) + self.projects = SimpleNamespace(delete=lambda **kw: None) + self.work_item_types = SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None) + self.workspace_work_item_types = SimpleNamespace(delete=lambda **kw: None) + self.workspace_work_item_properties = SimpleNamespace(delete=lambda **kw: None) + + +def test_r6_random_truth_is_per_seed_and_oracle_is_api_confirmed(): + class Projects: + def __init__(self, main_id: str, main_name: str): + self.names = {main_id: main_name} + + def create(self, workspace_slug, data): + project_id = f"second-{len(self.names)}" + self.names[project_id] = data.name + return SimpleNamespace(id=project_id, name=data.name, identifier=data.identifier) + + def update(self, **kwargs): + return None + + def update_features(self, **kwargs): + return None + + def retrieve(self, *, project_id, **kwargs): + return SimpleNamespace(id=project_id, name=self.names[project_id]) + + class WorkItems: + def __init__(self, *, mark_second_non_bug: int): + self.rows: dict[str, SimpleNamespace] = {} + self.project_ids: dict[str, list[str]] = {} + self.mark_second_non_bug = mark_second_non_bug + + def create(self, *, project_id, data, **kwargs): + project_rows = self.project_ids.setdefault(str(project_id), []) + work_item_id = f"{project_id}-wi-{len(project_rows) + 1}" + project_rows.append(work_item_id) + self.rows[work_item_id] = SimpleNamespace( + id=work_item_id, + name=data.name, + type_id=data.type_id, + completed_at=None, + archived_at=None, + ) + return self.rows[work_item_id] + + def retrieve(self, *, project_id, work_item_id, **kwargs): + row = self.rows[work_item_id] + project_rows = self.project_ids[str(project_id)] + if ( + self.mark_second_non_bug + and str(project_id).startswith("second-") + and work_item_id in project_rows[-self.mark_second_non_bug :] + ): + return SimpleNamespace(**{**vars(row), "type_id": "not-bug"}) + return row + + def seeded(run_id: str, *, mark_second_non_bug: int = 0): + run8 = run_id[:8] + main_id = f"main-{run8}" + main_name = f"EVAL {run8}" + work_items = WorkItems(mark_second_non_bug=mark_second_non_bug) + # Types are project-owned in this fake, so the second project must be given its own + # Bug type: creating its items with the main project's type id is what made an agent + # resolving 'Bug' inside that project count zero. + project_types: dict[str, str] = {} + + def create_project_type(*, project_id, data, **kwargs): + type_id = f"bug-{project_id}" + project_types[str(project_id)] = type_id + return SimpleNamespace(id=type_id, name=data.name) + + plane = SimpleNamespace( + projects=Projects(main_id, main_name), + work_items=work_items, + work_item_types=SimpleNamespace( + list=lambda **kwargs: [], + create=create_project_type, + ), + ) + ctx = { + "run_id": run_id, + "run8": run8, + "task_id": "R6", + "project_id": main_id, + "project_name": main_name, + "items": {}, + "item_ids": [], + "bug_type": {"id": "bug-1", "name": "Bug"}, + "bug_type_workspace_level": False, + "randomized_truth": {}, + } + seed_second_project(plane, "ws", ctx) + return plane, ctx + + first_plane, first = seeded("22222222cccccccc") # deterministic intended counts 3 / 2 + second_plane, second = seeded("aabbccdd11223344", mark_second_non_bug=2) # intended 4 / 5 + assert "second_project_ids" not in first + assert "second_project_ids" not in second + + first_intended = first["randomized_truth"]["R6.open_bug_counts"] + second_truth = second["randomized_truth"]["R6.open_bug_counts"] + assert (first_intended["intended_main"], first_intended["intended_second"]) != ( + second_truth["intended_main"], + second_truth["intended_second"], + ) + assert (first["r6_main_bug_count"], first["r6_second_bug_count"]) == (3, 2) + assert (second["r6_main_bug_count"], second["r6_second_bug_count"]) == (4, 3) + # Intended counts say B wins 5-to-4. API readback says main wins 4-to-3; + # the verifier must use the API-confirmed seed oracle. + assert second["r6_more_bugs_project"] == second["project_name"] + assert second_truth["confirmed"]["winner"] == second["project_name"] + + run = { + "final_text": f"project: {second['project_name']}", + "calls": [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + "call_source": "test", + "evidence_trace_available": True, + } + ok, note = asyncio.run(verify_r6(second_plane, second, run)) + assert ok is True, note + wrong_run = {**run, "final_text": f"project: {second['second_project_name']}"} + wrong_ok, wrong_note = asyncio.run(verify_r6(second_plane, second, wrong_run)) + assert wrong_ok is False, wrong_note + + +def test_baseline_snapshot_failure_surfaces_before_workspace_mutation(monkeypatch): + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "ws") + + def fail_list(**kwargs): + raise RuntimeError("customers unreadable") + + plane = SimpleNamespace( + projects=SimpleNamespace(create=lambda **kwargs: SimpleNamespace(id="project-1", identifier="EVDEADBEEF")), + customers=SimpleNamespace( + list=fail_list, + properties=SimpleNamespace(list=lambda **kwargs: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kwargs: _Page([]))), + ) + context: dict[str, Any] = {} + + with pytest.raises(RuntimeError, match="workspace baseline snapshot: list customers failed"): + seed_mod.seed(plane, "deadbeefcafebabe", set(), context, task_id="W10") + + assert context["project_id"] == "project-1" + + +def test_workspace_feature_snapshot_failure_prevents_mutation(): + updates: list[Any] = [] + plane = SimpleNamespace( + workspaces=SimpleNamespace( + get_features=lambda **kwargs: (_ for _ in ()).throw(RuntimeError("feature read failed")), + update_features=lambda **kwargs: updates.append(kwargs), + ) + ) + + with pytest.raises(RuntimeError, match="workspace feature snapshot failed before mutation"): + seed_mod.enable_workspace_features(plane, "ws") + + assert updates == [] + + +@pytest.mark.parametrize( + "context_key,context_value", + [("second_project_id", "second-1"), ("second_project_ids", ["second-1"])], +) +def test_teardown_deletes_second_project_from_current_or_legacy_context_key(context_key, context_value): + deleted: list[str] = [] + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kwargs: _Page([]), + properties=SimpleNamespace(list=lambda **kwargs: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kwargs: _Page([]))), + projects=SimpleNamespace(delete=lambda **kwargs: deleted.append(str(kwargs["project_id"]))), + work_item_types=SimpleNamespace(list=lambda **kwargs: []), + ) + context = { + "workspace_slug": "ws", + "project_id": "main-1", + context_key: context_value, + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + } + + seed_mod.teardown(plane, context) + + assert deleted == ["second-1", "main-1"] + + +def _seed_plan_covers_all_groups(_monkeypatch): + groups = { + "items", + "labels", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + } + lines = seed_plan(groups) + blob = "\n".join(lines) + for g in groups: + assert ( + g.split("_")[0] in blob or g in blob or g.replace("_", " ") in blob or any(g in line for line in lines) + ), f"seed_plan missing {g}: {lines}" + # Specific fixtures named + assert "Sprint 12" in blob + assert "Checkout revamp" in blob + assert "1.2.0" in blob + assert "Acme Corp" in blob + + +def _seed_plan_empty_needs_only_project(_monkeypatch): + lines = seed_plan(set()) + assert any("project" in line for line in lines) + # project line + default workspace customers enable note + assert any("customers" in line for line in lines) + assert len(lines) == 2 + + +def _seed_module_ast_has_all_group_handlers(_monkeypatch): + src = inspect.getsource(seed_mod.seed) + for group in ( + "labels", + "items", + "bug_type", + "cycles", + "module", + "intake", + "customer", + "release", + "second_project", + ): + assert f'"{group}"' in src or f"'{group}'" in src, group + + +def _seed_enables_project_features_immediately_after_create(monkeypatch): + from types import SimpleNamespace + + from plane.models.projects import ProjectFeature, UpdateProject + from plane.models.workspaces import WorkspaceFeature + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + calls.append(("create", workspace_slug, getattr(data, "name", None))) + return SimpleNamespace(id="proj-main") + + def update(self, workspace_slug, project_id, data): + assert isinstance(data, UpdateProject) + calls.append(("update", project_id, data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + assert isinstance(data, ProjectFeature) + calls.append(("update_features", project_id, data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"customers": False}) + + def update_features(self, workspace_slug, data): + assert isinstance(data, WorkspaceFeature) + calls.append(("ws_update_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="tag-unrelated", version=L3_TAG_VERSION)]), + delete=lambda **kw: None, + ) + ), + ) + ctx: dict = {} + seed_mod.seed(plane, run_id="deadbeefcafebabe", needs=set(), ctx=ctx, task_id="W10") + + assert ctx["project_id"] == "proj-main" + kinds = [c[0] for c in calls] + assert kinds == ["create", "ws_update_features", "update", "update_features"] + # Workspace customers enabled for C1 preconditions + assert calls[1][1].get("customers") is True + assert "work_item_types" not in calls[1][1] + # Project enable calls target the created id + assert calls[2][1] == "proj-main" + assert calls[3][1] == "proj-main" + upd = calls[2][2] + assert upd.get("cycle_view") is True + assert upd.get("is_time_tracking_enabled") is True + feat = calls[3][2] + assert feat.get("cycles") is True + assert ctx["workspace_baseline"] == { + "customers": set(), + "release_tags": {"tag-unrelated"}, + "customer_properties": set(), + "work_item_types": None, + "work_item_properties": None, + } + + +def _seed_collision_skips_before_create(monkeypatch): + from evals.tasks.skip import TaskSkipped + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + creates: list[Any] = [] + plane = SimpleNamespace( + projects=SimpleNamespace(create=lambda **kw: creates.append(kw)), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace( + tags=SimpleNamespace(list=lambda **kw: _Page([SimpleNamespace(id="tag-collision", version=L3_TAG_VERSION)])) + ), + ) + ctx: dict[str, Any] = {} + + with pytest.raises(TaskSkipped, match=r"^env:fixture-collision:release_tags:eval-rc1"): + seed_mod.seed(plane, run_id="collision123456", needs=set(), ctx=ctx, task_id="L3") + + assert creates == [] + assert ctx["project_id"] is None + assert ctx["workspace_baseline"] == { + "customers": None, + "release_tags": None, + "customer_properties": None, + "work_item_types": None, + "work_item_properties": None, + } + + +def _seed_s5_leaves_cycles_worklogs_and_customers_off(monkeypatch): + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-s5") + + def update(self, workspace_slug, project_id, data): + calls.append(("update", data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + calls.append(("features", data.model_dump(exclude_none=True))) + return data + + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"customers": True}) + + def update_features(self, workspace_slug, data): + calls.append(("ws_features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects(), workspaces=_Workspaces()) + ctx: dict = {} + seed_mod.seed(plane, run_id="s5s5s5s5s5s5s5s5", needs={"leave_cycles_worklogs_off"}, ctx=ctx) + assert ctx["feature_exclude"] == ["cycles", "worklogs"] + assert ctx["ws_feature_exclude"] == ["customers"] + assert ctx["s5_left_customers_off"] is True + # Excluded features are written OFF, not omitted. The workspace outlives the run, so + # omitting the write leaves the previous rep's value and S5's precondition never holds. + ws = next(c[1] for c in calls if c[0] == "ws_features") + assert ws.get("customers") is False + assert ctx["workspace_features_prior"] == {"customers": True} + upd = next(c[1] for c in calls if c[0] == "update") + assert upd.get("cycle_view") is False + assert upd.get("is_time_tracking_enabled") is False + assert upd.get("module_view") is True + feat = next(c[1] for c in calls if c[0] == "features") + assert feat.get("cycles") is False + assert feat.get("modules") is True + + +def _seed_cycles_create_add_then_backdate(monkeypatch): + from types import SimpleNamespace + + from plane.models.cycles import CreateCycle, UpdateCycle + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + cycle_seq = {"n": 0} + + class _Cycles: + def create(self, workspace_slug, project_id, data): + assert isinstance(data, CreateCycle) + cycle_seq["n"] += 1 + cid = f"cyc-{cycle_seq['n']}" + calls.append( + ( + "create", + { + "name": data.name, + "start_date": data.start_date, + "end_date": data.end_date, + "id": cid, + }, + ) + ) + return SimpleNamespace(id=cid, name=data.name, end_date=data.end_date) + + def add_work_items(self, workspace_slug, project_id, cycle_id, issue_ids): + calls.append(("add_work_items", {"cycle_id": cycle_id, "n": len(issue_ids)})) + + def update(self, workspace_slug, project_id, cycle_id, data): + assert isinstance(data, UpdateCycle) + calls.append(("update", {"cycle_id": cycle_id, "end_date": data.end_date})) + return SimpleNamespace(id=cycle_id, end_date=data.end_date) + + class _Projects: + def create(self, workspace_slug, data): + return SimpleNamespace(id="proj-1") + + def update(self, workspace_slug, project_id, data): + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + return data + + class _Workspaces: + def update_features(self, workspace_slug, data): + return data + + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {}) + + class _Users: + def get_me(self): + return SimpleNamespace(id="user-1") + + class _States: + def list(self, workspace_slug, project_id): + return SimpleNamespace( + results=[ + SimpleNamespace(id="st-started", name="In Progress", group="started", default=False), + SimpleNamespace(id="st-todo", name="Todo", group="unstarted", default=True), + ] + ) + + item_n = {"n": 0} + + class _WorkItems: + def create(self, workspace_slug, project_id, data): + item_n["n"] += 1 + return SimpleNamespace( + id=f"wi-{item_n['n']}", + name=data.name, + state="st-started", + created_at="2026-01-01", + ) + + def update(self, workspace_slug, project_id, work_item_id, data): + return SimpleNamespace(id=work_item_id, name="x", state=getattr(data, "state", None)) + + class comments: + @staticmethod + def create(**kw): + return SimpleNamespace(id="c1") + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + cycles=_Cycles(), + users=_Users(), + states=_States(), + work_items=_WorkItems(), + ) + ctx: dict = {} + seed_mod.seed(plane, run_id="cycletestabcdef", needs={"items", "cycles"}, ctx=ctx) + + # Filter to Sprint-12-related create/add/update sequence (first cycle is past). + past_id = ctx["cycle_past_id"] + # Must create both cycles before any backdate update of past. + create_idxs = [i for i, c in enumerate(calls) if c[0] == "create"] + assert len(create_idxs) == 2 + past_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_PAST) + # Created with temporary *future* end_date (active), not the final past end. + assert past_create[1]["end_date"] > past_create[1]["start_date"] + # At least one add to past cycle before its update + past_adds = [i for i, c in enumerate(calls) if c[0] == "add_work_items" and c[1]["cycle_id"] == past_id] + past_updates = [i for i, c in enumerate(calls) if c[0] == "update" and c[1]["cycle_id"] == past_id] + assert past_adds, "expected add_work_items on Sprint 12" + assert past_updates, "expected backdate update on Sprint 12" + assert max(past_adds) < min(past_updates), f"add must precede backdate; calls={calls}" + # Backdated end matches W6 seed ctx; differs from create-time active end + backdated_end = calls[past_updates[0]][1]["end_date"] + assert ctx["cycle_past_seed_end_date"] == backdated_end + assert backdated_end != past_create[1]["end_date"] + assert ctx.get("cycle_past_end_date_before_backdate") == past_create[1]["end_date"] + # Active cycle: create with future end; never backdated + cur_create = next(c for c in calls if c[0] == "create" and c[1]["name"] == seed_mod.CYCLE_CURRENT) + assert cur_create[1]["end_date"] + cur_updates = [c for c in calls if c[0] == "update" and c[1]["cycle_id"] == ctx["cycle_current_id"]] + assert cur_updates == [] + + +def _seed_enables_features_on_second_project_too(monkeypatch): + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + creates: list[str] = [] + enables: list[str] = [] + + class _Projects: + def create(self, workspace_slug, data): + pid = f"p-{len(creates)}" + creates.append(pid) + return SimpleNamespace(id=pid) + + def update(self, workspace_slug, project_id, data): + enables.append(("update", project_id)) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + enables.append(("features", project_id)) + return data + + # Minimal stubs so second_project seed gets past bug_type + work items. + class _Workspaces: + def get_features(self, workspace_slug): + return SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": False}) + + def update_features(self, workspace_slug, data): + enables.append(("ws_features", workspace_slug)) + return data + + class _WorkItemTypes: + def list(self, **kw): + return [SimpleNamespace(id="bug-1", name="Bug")] + + def create(self, **kw): + return SimpleNamespace(id="bug-1", name="Bug") + + def import_to_project(self, **kw): + return None + + class _WorkItems: + def create(self, **kw): + return SimpleNamespace(id=f"wi-{id(kw)}", name=kw["data"].name) + + plane = SimpleNamespace( + projects=_Projects(), + workspaces=_Workspaces(), + work_item_types=_WorkItemTypes(), + work_items=_WorkItems(), + ) + ctx: dict = {} + # second_project path also seeds bug_type when missing + seed_mod.seed(plane, run_id="aabbccdd11223344", needs={"second_project", "bug_type"}, ctx=ctx) + + assert len(creates) == 2 + # Each create followed by update + update_features for that project id + assert ("update", creates[0]) in enables + assert ("features", creates[0]) in enables + assert ("update", creates[1]) in enables + assert ("features", creates[1]) in enables + + +_SEED_CASES = case_params( + _seed_plan_covers_all_groups, + _seed_plan_empty_needs_only_project, + _seed_module_ast_has_all_group_handlers, + _seed_enables_project_features_immediately_after_create, + _seed_collision_skips_before_create, + _seed_s5_leaves_cycles_worklogs_and_customers_off, + _seed_cycles_create_add_then_backdate, + _seed_enables_features_on_second_project_too, +) + + +@pytest.mark.parametrize("case", _SEED_CASES) +def test_seed_behaviours(case, monkeypatch): + case(monkeypatch) + + +def test_excluding_pages_turns_page_view_off_despite_its_true_default(monkeypatch): + """``page_view`` defaults to True on a fresh project, so omission is not exclusion. + + The other excludable project features default false, which is why omitting the write + happened to work for S5. Relying on that is unsound for any feature added later. + """ + from types import SimpleNamespace + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list[tuple] = [] + + class _Projects: + def update(self, workspace_slug, project_id, data): + calls.append(("update", data.model_dump(exclude_none=True))) + return SimpleNamespace(id=project_id) + + def update_features(self, workspace_slug, project_id, data): + calls.append(("features", data.model_dump(exclude_none=True))) + return data + + plane = SimpleNamespace(projects=_Projects()) + seed_mod.enable_project_features(plane, "test-ws", "proj-1", exclude={"pages"}) + + upd = next(c[1] for c in calls if c[0] == "update") + assert upd.get("page_view") is False + feat = next(c[1] for c in calls if c[0] == "features") + assert feat.get("pages") is False + assert feat.get("cycles") is True + + +@pytest.mark.parametrize("prior", [True, False]) +def test_teardown_restores_the_workspace_value_it_found(monkeypatch, prior): + """Teardown puts the toggle back, rather than forcing the value this run wanted. + + The harness runs against an instance it does not own. Forcing ``customers=True`` on + the way out is configuration drift for anyone whose workspace had it off. + """ + from types import SimpleNamespace + + from plane.models.workspaces import WorkspaceFeature + + monkeypatch.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.delenv("REDIS_PORT", raising=False) + + calls: list = [] + + class _Workspaces: + def update_features(self, workspace_slug, data): + assert isinstance(data, WorkspaceFeature) + calls.append(data.model_dump(exclude_none=True)) + return data + + plane = SimpleNamespace( + workspaces=_Workspaces(), + projects=SimpleNamespace(delete=lambda **k: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + ) + seed_mod.teardown( + plane, + { + "workspace_slug": "test-ws", + "workspace_features_prior": {"customers": prior}, + "project_id": None, + }, + ) + assert calls and calls[0].get("customers") is prior + + +def _teardown_leaves_workspace_alone_when_prior_unknown(): + from types import SimpleNamespace + + calls: list = [] + + class _Workspaces: + def update_features(self, workspace_slug, data): + calls.append(data) + return data + + plane = SimpleNamespace( + workspaces=_Workspaces(), + projects=SimpleNamespace(delete=lambda **k: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + ) + seed_mod.teardown( + plane, + { + "workspace_slug": "test-ws", + "workspace_features_prior": {"customers": None}, + "project_id": None, + }, + ) + assert calls == [] + + +def _teardown_deletes_release_tag_and_customer_property(): + from evals.seed import teardown + + plane = _TeardownPlane() + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "project_name": "EVAL x", + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + "workspace_objects": [ + {"kind": "release_tag", "id": "tag-tracked"}, + {"kind": "customer_property", "id": "prop-tracked"}, + ], + } + teardown(plane, ctx) + kinds = {k for k, _ in plane.deleted} + assert "release_tag" in kinds + assert "customer_property" in kinds + # Tracked ids deleted + assert ("release_tag", "tag-tracked") in plane.deleted + assert ("customer_property", "prop-tracked") in plane.deleted + + +@pytest.mark.parametrize( + "case", + case_params( + _teardown_leaves_workspace_alone_when_prior_unknown, + _teardown_deletes_release_tag_and_customer_property, + ), +) +def test_teardown_behaviours(case): + case() + + +def test_teardown_aggregates_failures_after_attempting_every_object(): + from evals.seed import TeardownError, teardown + + delete_calls: list[tuple[str, str]] = [] + + def fail_delete(kind: str, object_id: str) -> None: + delete_calls.append((kind, object_id)) + raise RuntimeError(f"cannot delete {kind} {object_id}") + + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + delete=lambda **kw: fail_delete("customer", kw["customer_id"]), + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + releases=SimpleNamespace( + delete=lambda **kw: fail_delete("release", kw["release_id"]), + tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + projects=SimpleNamespace(delete=lambda **kw: fail_delete("project", kw["project_id"])), + work_item_types=SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None), + ) + context = { + "workspace_slug": "ws", + "project_id": "project-main", + "project_name": "EVAL cleanup", + "second_project_ids": ["project-second"], + "workspace_objects": [ + {"kind": "customer", "id": "customer-1"}, + {"kind": "release", "id": "release-1"}, + ], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + }, + } + + with pytest.raises(TeardownError) as caught: + teardown(plane, context) + + assert delete_calls == [ + ("customer", "customer-1"), + ("release", "release-1"), + ("project", "project-second"), + ("project", "project-main"), + ] + assert len(caught.value.failures) == 4 + assert {failure.target for failure in caught.value.failures} == { + "customer-1", + "release-1", + "project-second", + "EVAL cleanup", + } + + +def test_teardown_customer_baseline_behaviours(capsys): + cases = ( + { + "name": "pre-existing name match", + "customer_id": "customer-existing", + "baseline": {"customer-existing"}, + "tracked": False, + "deleted": False, + "warns": False, + }, + { + "name": "agent-created name match", + "customer_id": "customer-agent", + "baseline": set(), + "tracked": False, + "deleted": True, + "warns": False, + }, + { + "name": "tracked id wins over baseline", + "customer_id": "customer-tracked", + "baseline": {"customer-tracked"}, + "tracked": True, + "deleted": True, + "warns": False, + }, + { + "name": "unavailable baseline fails closed", + "customer_id": "customer-unknown", + "baseline": None, + "tracked": False, + "deleted": False, + "warns": True, + }, + ) + + for case in cases: + with pytest.MonkeyPatch.context() as mp: + mp.setenv("EVAL_PLANE_WORKSPACE_SLUG", "test-ws") + deleted: list[str] = [] + customer = SimpleNamespace(id=case["customer_id"], name="Acme Corp") + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda customer=customer, **kw: _Page([customer]), + delete=lambda deleted=deleted, **kw: deleted.append(str(kw["customer_id"])), + properties=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]), delete=lambda **kw: None)), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + workspace_objects = [{"kind": "customer", "id": case["customer_id"]}] if case["tracked"] else [] + context = { + "workspace_slug": "test-ws", + "project_id": None, + "workspace_objects": workspace_objects, + "workspace_baseline": { + "customers": case["baseline"], + "release_tags": set(), + "customer_properties": set(), + }, + } + if case["warns"]: + with pytest.raises(seed_mod.TeardownError) as caught: + seed_mod.teardown(plane, context) + assert "baseline unavailable" in str(caught.value) + else: + seed_mod.teardown(plane, context) + output = capsys.readouterr().out + assert (case["customer_id"] in deleted) is case["deleted"], case["name"] + assert ("customers baseline unavailable" in output) is case["warns"], case["name"] + if case["warns"]: + assert case["customer_id"] in output + + +def test_preexisting_workspace_bug_is_reused_and_not_deleted(): + deleted_types: list[str] = [] + imported_types: list[str] = [] + bug = SimpleNamespace(id="bug-existing", name="Bug") + plane = SimpleNamespace( + workspaces=SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [bug], + create=lambda **kw: pytest.fail("pre-existing Bug must be reused"), + delete=lambda **kw: deleted_types.append(str(kw["type_id"])), + properties=SimpleNamespace(list=lambda **kw: []), + ), + workspace_work_item_properties=SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None), + work_item_types=SimpleNamespace( + import_to_project=lambda **kw: imported_types.extend(str(value) for value in kw["work_item_type_ids"]), + ), + work_item_properties=SimpleNamespace(list=lambda **kw: [], delete=lambda **kw: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + context = { + "task_id": "S1", + "workspace_slug": "ws", + "project_id": "project-1", + "workspace_objects": [], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + "work_item_types": {"bug-existing"}, + "work_item_properties": set(), + }, + } + + seed_mod.seed_item_type(plane, "ws", context) + seed_mod.teardown(plane, context) + + assert imported_types == ["bug-existing"] + assert context["bug_type_created"] is False + assert context["workspace_objects"] == [] + assert deleted_types == [] + + +@pytest.mark.parametrize( + ("baseline", "should_delete"), + [ + pytest.param({"severity-existing"}, False, id="pre-existing-preserved"), + pytest.param(set(), True, id="agent-created-deleted"), + ], +) +def test_teardown_severity_property_uses_seed_baseline(baseline, should_delete): + deleted: list[str] = [] + bug = SimpleNamespace(id="bug-existing", name="Bug") + severity = SimpleNamespace(id="severity-existing", display_name="Severity") + plane = SimpleNamespace( + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [bug], + properties=SimpleNamespace(list=lambda **kw: [severity.id]), + ), + workspace_work_item_properties=SimpleNamespace( + list=lambda **kw: [severity], + delete=lambda **kw: deleted.append(str(kw["property_id"])), + ), + work_item_properties=SimpleNamespace(list=lambda **kw: [severity], delete=lambda **kw: None), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + context = { + "task_id": "S1", + "workspace_slug": "ws", + "project_id": "project-1", + "bug_type": {"id": bug.id, "name": bug.name}, + "bug_type_workspace_level": True, + "workspace_objects": [], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + "work_item_types": {bug.id}, + "work_item_properties": baseline, + }, + } + + seed_mod.teardown(plane, context) + + assert (severity.id in deleted) is should_delete + + +@pytest.mark.parametrize( + ("baseline", "should_delete"), + [ + pytest.param({"incident-existing"}, False, id="pre-existing-preserved"), + pytest.param(set(), True, id="agent-created-deleted"), + ], +) +def test_teardown_workspace_incident_uses_seed_baseline(baseline, should_delete): + deleted: list[str] = [] + incident = SimpleNamespace(id="incident-existing", name="Incident") + plane = SimpleNamespace( + workspaces=SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [incident], + delete=lambda **kw: deleted.append(str(kw["type_id"])), + ), + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + projects=SimpleNamespace(delete=lambda **kw: None), + ) + context = { + "task_id": "S3", + "workspace_slug": "ws", + "project_id": "project-1", + "workspace_objects": [], + "workspace_baseline": { + "customers": set(), + "release_tags": set(), + "customer_properties": set(), + "work_item_types": baseline, + "work_item_properties": None, + }, + } + + seed_mod.teardown(plane, context) + + assert (incident.id in deleted) is should_delete + + +def test_preclean_behaviours(): + from evals.seed import check_workspace_fixture_collisions + from evals.tasks.skip import TaskSkipped + + cases = ( + { + "name": "release tag collision", + "customers": [], + "tags": [SimpleNamespace(id="tag-old", version=L3_TAG_VERSION)], + "properties": [], + "category": "release_tags", + "fixture_name": L3_TAG_VERSION, + "checked_categories": {"release_tags"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "customer property collision", + "customers": [], + "tags": [], + "properties": [SimpleNamespace(id="prop-old", display_name=L4_PROP_DISPLAY, name="x")], + "category": "customer_properties", + "fixture_name": L4_PROP_DISPLAY, + "checked_categories": {"customer_properties"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "customer collision", + "customers": [SimpleNamespace(id="customer-old", name="Acme")], + "tags": [], + "properties": [], + "category": "customers", + "fixture_name": "Acme Corp", + "checked_categories": {"customers"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "Bug Severity collision", + "customers": [], + "tags": [], + "properties": [], + "workspace_types": [SimpleNamespace(id="type-bug", name="Bug")], + "workspace_properties": [SimpleNamespace(id="severity-old", display_name="Severity")], + "category": "work_item_properties", + "fixture_name": "Severity", + "checked_categories": {"work_item_properties"}, + }, + { + "name": "Incident collision", + "customers": [], + "tags": [], + "properties": [], + "workspace_types": [SimpleNamespace(id="incident-old", name="Incident")], + "workspace_properties": [], + "category": "work_item_types", + "fixture_name": "Incident", + "checked_categories": {"work_item_types"}, + }, + { + "name": "clean workspace", + "customers": [SimpleNamespace(id="customer-other", name="Other Corp")], + "tags": [SimpleNamespace(id="tag-other", version="v2")], + "properties": [SimpleNamespace(id="prop-other", display_name="Region", name="region")], + "category": None, + "fixture_name": None, + "checked_categories": {"customers", "release_tags", "customer_properties"}, + "workspace_types": [], + "workspace_properties": [], + }, + { + "name": "release tag irrelevant to checked category", + "customers": [], + "tags": [SimpleNamespace(id="tag-unrelated", version=L3_TAG_VERSION)], + "properties": [], + "category": None, + "fixture_name": None, + "checked_categories": {"customers"}, + "workspace_types": [SimpleNamespace(id="incident-unrelated", name="Incident")], + "workspace_properties": [], + }, + ) + + for case in cases: + with pytest.MonkeyPatch.context(): + deleted: list[tuple[str, str]] = [] + customers = case["customers"] + tags = case["tags"] + properties = case["properties"] + workspace_types = case["workspace_types"] + workspace_properties = case["workspace_properties"] + plane = SimpleNamespace( + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda tags=tags, **kw: _Page(tags), + delete=lambda deleted=deleted, **kw: deleted.append(("tag", kw["tag_id"])), + ) + ), + customers=SimpleNamespace( + list=lambda customers=customers, **kw: _Page(customers), + delete=lambda deleted=deleted, **kw: deleted.append(("customer", kw["customer_id"])), + properties=SimpleNamespace( + list=lambda properties=properties, **kw: _Page(properties), + delete=lambda deleted=deleted, **kw: deleted.append(("property", kw["property_id"])), + ), + ), + workspaces=SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda workspace_types=workspace_types, **kw: workspace_types, + properties=SimpleNamespace( + list=lambda workspace_properties=workspace_properties, **kw: ( + [row.id for row in workspace_properties] if kw.get("type_id") == "type-bug" else [] + ) + ), + ), + workspace_work_item_properties=SimpleNamespace( + list=lambda workspace_properties=workspace_properties, **kw: workspace_properties + ), + ) + + if case["category"] is None: + check_workspace_fixture_collisions(plane, "ws", case["checked_categories"]) + else: + expected = f"env:fixture-collision:{case['category']}:{case['fixture_name']}" + with pytest.raises(TaskSkipped) as caught: + check_workspace_fixture_collisions(plane, "ws", case["checked_categories"]) + assert caught.value.reason.startswith(expected), case["name"] + assert case["fixture_name"] in caught.value.reason + assert "python -m evals.cleanup --sentinels --yes" in caught.value.reason + assert deleted == [], case["name"] + + +def test_collision_category_coverage_matches_task_prompts(): + from evals.seed import ( + CUSTOMER_NAME, + EVALUATION_CUSTOMER_PROPERTY_NAME, + EVALUATION_RELEASE_TAG_VERSION, + INCIDENT_TYPE_NAME, + SEVERITY_PROPERTY_NAME, + collision_categories, + ) + from evals.tasks.catalog import TASKS + + prompt_categories = ( + (CUSTOMER_NAME, "customers"), + (EVALUATION_RELEASE_TAG_VERSION, "release_tags"), + (EVALUATION_CUSTOMER_PROPERTY_NAME, "customer_properties"), + (SEVERITY_PROPERTY_NAME, "work_item_properties"), + (INCIDENT_TYPE_NAME, "work_item_types"), + ) + for task in TASKS: + task_id = str(task["id"]) + categories = collision_categories(set(task.get("needs") or set()), task_id) + prompt = str(task.get("prompt") or "") + for fixture_name, category in prompt_categories: + if fixture_name in prompt: + assert category in categories, f"task {task_id} prompt references {fixture_name!r}; missing {category}" + + +@pytest.mark.parametrize( + ("context", "read_result", "expected_error", "match"), + [ + pytest.param( + {"project_id": "p1", "items": {}}, + [], + RuntimeError, + "fixture error: missing work_item_id", + id="missing-work-item-is-fixture-error", + ), + pytest.param( + {"project_id": None, "items": {R5_TITLE: "wi-r5"}}, + [], + RuntimeError, + "fixture error: missing project_id", + id="missing-project-is-fixture-error", + ), + pytest.param( + {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}}, + ConnectionError("activity backend unavailable"), + ConnectionError, + "activity backend unavailable", + id="read-failure-propagates-as-infrastructure", + ), + pytest.param( + {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}}, + [], + TaskSkipped, + "^env:no-activity-worker$", + id="successful-empty-read-is-capability-skip", + ), + pytest.param( + {"project_id": "p1", "items": {R5_TITLE: "wi-r5"}}, + [SimpleNamespace(id="a1")], + None, + None, + id="successful-nonempty-read-proceeds", + ), + pytest.param( + # Revision 8: a non-empty read is sufficient, and evidence is the activity count. + # This case used to require the seeded comment phrase in the readback and expect a + # fixture error without it — a contract Plane's activity API can never satisfy, + # because it returns the creation row and never the comment text. + { + "task_id": "L2", + "project_id": "p1", + "items": {R5_TITLE: "wi-r5"}, + "l2_comment_phrases": ["hidden seeded comment"], + }, + [SimpleNamespace(id="a1", comment="unrelated activity")], + None, + None, + id="nonempty-read-without-comment-text-binds-count-evidence", + ), + ], +) +def test_l2_activity_gate_outcomes(context, read_result, expected_error, match): + from evals.seed import _gate_activity_worker + + def list_activities(**kwargs): + if isinstance(read_result, BaseException): + raise read_result + return SimpleNamespace(results=read_result) + + plane = SimpleNamespace(work_items=SimpleNamespace(activities=SimpleNamespace(list=list_activities))) + if expected_error is None: + _gate_activity_worker(plane, "ws", context) + if context.get("task_id") == "L2": + aggregates = context["evidence_aggregates"][TARGET_ENTITY_EVIDENCE] + assert aggregates == ({"kind": "total_count", "value": 1},) + assert context["evidence_targets"][TARGET_ENTITY_EVIDENCE] == ("wi-r5",) + else: + with pytest.raises(expected_error, match=match): + _gate_activity_worker(plane, "ws", context) + + +def _create_project_retries_409_then_succeeds(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + ident = data.identifier + attempts.append(ident) + if len(attempts) < 3: + raise HttpError("Project identifier already taken", 409) + return MagicMock(id="proj-ok", identifier=ident) + + plane = MagicMock() + plane.projects = FakeProjects() + + # Force deterministic retries after first collision. + suffixes = iter(["AAAAAAAA", "BBBBBBBB"]) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + project = create_project_with_collision_retry( + plane, + "ws", + name="EVAL abcd", + identifier_prefix="EV", + initial_suffix="DEADBEEF", + ) + assert project.id == "proj-ok" + assert attempts[0] == "EVDEADBEEF" + assert len(attempts) == 3 + assert attempts[1] != attempts[0] + assert attempts[2] != attempts[1] + assert attempts[1] == "EVAAAAAAAA" + assert attempts[2] == "EVBBBBBBBB" + + +def _create_project_raises_after_max_409s(monkeypatch): + attempts: list[str] = [] + + class Always409: + def create(self, *, workspace_slug, data): + attempts.append(data.identifier) + raise HttpError("identifier already taken", 409) + + plane = MagicMock() + plane.projects = Always409() + suffixes = iter( + [ + "11111111", + "22222222", + "33333333", + "44444444", + "55555555", + "66666666", + "77777777", + "should-not-use", + ] + ) + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: next(suffixes)) + + with pytest.raises(HttpError) as ei: + create_project_with_collision_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="00000000", + ) + assert ei.value.status_code == 409 + assert len(attempts) == 8 + assert attempts[0] == "EV00000000" + assert attempts[1] != attempts[0] + assert attempts[1] == "EV11111111" + assert attempts[-1] == "EV77777777" + + +def _create_project_non_collision_error_does_not_retry(_monkeypatch): + class Fail500: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + plane = MagicMock() + plane.projects = Fail500() + with pytest.raises(HttpError) as ei: + create_project_with_collision_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="00000000", + ) + assert ei.value.status_code == 500 + + +def _identifier_stays_within_plane_limit(_monkeypatch): + plane = SimpleNamespace( + projects=SimpleNamespace( + create=lambda **kwargs: SimpleNamespace(id="project", identifier=kwargs["data"].identifier) + ) + ) + project = create_project_with_collision_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="12345678", + ) + assert project.identifier == "EV12345678" + assert len(project.identifier) <= seed_mod.PLANE_PROJECT_IDENTIFIER_MAX_LENGTH + + with pytest.raises(ValueError, match="12-character limit"): + create_project_with_collision_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="TOO-LONG", + initial_suffix="12345678", + ) + + +# The observed payload, verbatim: this is what failed L3 in the 177-tool arm on 2026-08-19. +_NAME_TAKEN = "Conflict: name: The project name is already taken" + + +def test_name_and_identifier_collisions_are_told_apart(): + """Both are 409s carrying the same collision wording; only the named field separates them.""" + assert is_name_collision(HttpError(_NAME_TAKEN, 409)) is True + assert is_identifier_collision(HttpError(_NAME_TAKEN, 409)) is False + + assert is_name_collision(HttpError("identifier already taken", 409)) is False + assert is_identifier_collision(HttpError("identifier already taken", 409)) is True + + # A body naming both fields is treated as the identifier case: retrying a suffix is cheap + # and is what this did before names could be retried at all. + both = HttpError("name already taken; identifier already taken", 409) + assert is_name_collision(both) is False + assert is_identifier_collision(both) is True + + # Collision language is still required, and the status still gates it. + assert is_name_collision(HttpError("name is required", 400)) is False + assert is_name_collision(HttpError(_NAME_TAKEN, 500)) is False + + +def _create_project_advances_name_on_name_collision(monkeypatch): + seen: list[tuple[str, str]] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + seen.append((data.name, data.identifier)) + if data.name == "EVAL Delivery Planning Plover": + raise HttpError(_NAME_TAKEN, 409) + # SimpleNamespace, not MagicMock: `name` is reserved on a Mock constructor. + return SimpleNamespace(id="proj-ok", name=data.name, identifier=data.identifier) + + plane = MagicMock() + plane.projects = FakeProjects() + + def _fail_token_hex(_n): + raise AssertionError("a name collision must not regenerate the identifier suffix") + + monkeypatch.setattr(seed_mod.secrets, "token_hex", _fail_token_hex) + + project = create_project_with_collision_retry( + plane, + "ws", + name="EVAL Delivery Planning Plover", + identifier_prefix="EV", + initial_suffix="DEADBEEF", + name_variants=iter(["EVAL Delivery Planning Godwit"]), + ) + assert project.name == "EVAL Delivery Planning Godwit" + # Two attempts, same identifier: the suffix was never the problem. + assert seen == [ + ("EVAL Delivery Planning Plover", "EVDEADBEEF"), + ("EVAL Delivery Planning Godwit", "EVDEADBEEF"), + ] + + +def _create_project_name_collision_raises_without_variants(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + attempts.append(data.name) + raise HttpError(_NAME_TAKEN, 409) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: "AAAAAAAA") + + with pytest.raises(HttpError) as ei: + create_project_with_collision_retry( + plane, + "ws", + name="EVAL x", + identifier_prefix="EV", + initial_suffix="00000000", + ) + assert ei.value.status_code == 409 + # Raised on the first refusal rather than spending the budget on a fine identifier. + assert attempts == ["EVAL x"] + + +class _ReadTimeout(Exception): + """Stands in for requests' ReadTimeout: an error carrying no HTTP status.""" + + +def _create_project_adopts_the_project_a_timeout_orphaned(monkeypatch): + """A create that times out after the server made the project must not orphan it. + + This is where the leftover projects came from: the client stopped waiting, the id was + never returned, so the caller never put it in the teardown context and teardown reported + cleanup_error 0 while the project sat in the workspace. + """ + listed: list[str] = [] + + class TimeoutThenPresent: + def create(self, *, workspace_slug, data): + raise _ReadTimeout("HTTPConnectionPool(host='localhost', port=8000): Read timed out.") + + def list(self, *, workspace_slug, params=None): + listed.append(workspace_slug) + return SimpleNamespace( + results=[ + SimpleNamespace(id="other", identifier="EVZZZZZZZZ", name="EVAL other"), + SimpleNamespace(id="orphan", identifier="EVDEADBEEF", name="EVAL Delivery Planning Wren"), + ], + next_page_results=False, + next_cursor="100:0:0", + ) + + plane = MagicMock() + plane.projects = TimeoutThenPresent() + + project = create_project_with_collision_retry( + plane, + "ws", + name="EVAL Delivery Planning Wren", + identifier_prefix="EV", + initial_suffix="DEADBEEF", + ) + # Adopted by identifier, so the caller learns the id and teardown can delete it. + assert project.id == "orphan" + assert listed == ["ws"] + + +def _create_project_reraises_when_nothing_was_created(monkeypatch): + """A timeout where the server created nothing must still fail, not invent a project.""" + + class TimeoutAndAbsent: + def create(self, *, workspace_slug, data): + raise _ReadTimeout("Read timed out.") + + def list(self, *, workspace_slug, params=None): + return SimpleNamespace(results=[], next_page_results=False, next_cursor="100:0:0") + + plane = MagicMock() + plane.projects = TimeoutAndAbsent() + with pytest.raises(_ReadTimeout): + create_project_with_collision_retry( + plane, "ws", name="EVAL x", identifier_prefix="EV", initial_suffix="00000000" + ) + + +def _create_project_does_not_adopt_after_an_http_refusal(monkeypatch): + """An HTTP error is a known outcome: nothing was created, so do not go looking.""" + listed: list[str] = [] + + class Refuses: + def create(self, *, workspace_slug, data): + raise HttpError("server error", 500) + + def list(self, *, workspace_slug, params=None): + listed.append(workspace_slug) + return SimpleNamespace(results=[], next_page_results=False, next_cursor=None) + + plane = MagicMock() + plane.projects = Refuses() + with pytest.raises(HttpError): + create_project_with_collision_retry( + plane, "ws", name="EVAL x", identifier_prefix="EV", initial_suffix="00000000" + ) + assert listed == [], "an HTTP refusal must not trigger an adoption lookup" + + +def _create_project_exhausts_name_variants(monkeypatch): + attempts: list[str] = [] + + class FakeProjects: + def create(self, *, workspace_slug, data): + attempts.append(data.name) + raise HttpError(_NAME_TAKEN, 409) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr(seed_mod.secrets, "token_hex", lambda n: "AAAAAAAA") + + with pytest.raises(HttpError): + create_project_with_collision_retry( + plane, + "ws", + name="EVAL a", + identifier_prefix="EV", + initial_suffix="00000000", + name_variants=iter(["EVAL b", "EVAL c"]), + ) + # Stops when variants run out, well inside the attempt budget. + assert attempts == ["EVAL a", "EVAL b", "EVAL c"] + assert len(attempts) < projects_mod.PROJECT_CREATE_ATTEMPT_LIMIT + + +@pytest.mark.parametrize( + "case", + case_params( + _create_project_retries_409_then_succeeds, + _create_project_raises_after_max_409s, + _create_project_non_collision_error_does_not_retry, + _identifier_stays_within_plane_limit, + _create_project_advances_name_on_name_collision, + _create_project_name_collision_raises_without_variants, + _create_project_exhausts_name_variants, + _create_project_adopts_the_project_a_timeout_orphaned, + _create_project_reraises_when_nothing_was_created, + _create_project_does_not_adopt_after_an_http_refusal, + ), +) +def test_create_behaviours(case, monkeypatch): + case(monkeypatch) + + +def test_identifier_collision_requires_status_and_language(): + assert is_identifier_collision(HttpError("identifier already taken", 409)) is True + assert is_identifier_collision(HttpError("project exists", 400)) is True + # Validation-shaped: mentions identifier but not collision language → no retry + assert is_identifier_collision(HttpError("identifier is required", 400)) is False + assert is_identifier_collision(HttpError("identifier already taken", 500)) is False + + +def test_project_name_variants_start_deterministic_and_cover_the_pool(): + from evals.core.fixtures import ( + PROJECT_SUFFIX_WORDS, + eval_project_name, + eval_project_name_variants, + ) + + for second in (False, True): + variants = list(eval_project_name_variants("3c128f21", second=second)) + # First name unchanged, so a resumed run and its teardown still agree on it. + assert variants[0] == eval_project_name("3c128f21", second=second) + assert len(variants) == len(PROJECT_SUFFIX_WORDS) + assert len(set(variants)) == len(variants) + + # Reproducible, and the two titles never collide with each other. + assert list(eval_project_name_variants("3c128f21")) == list(eval_project_name_variants("3c128f21")) + assert not set(eval_project_name_variants("3c128f21")) & set(eval_project_name_variants("3c128f21", second=True)) + # Non-hex prefixes fall back to a character sum rather than raising. + assert len(list(eval_project_name_variants("not-hex-at-all"))) == len(PROJECT_SUFFIX_WORDS) + + +def _cleanup_dry_run_never_calls_delete(monkeypatch, capsys, _yes): + projects = [ + SimpleNamespace(id="p1", name="EVAL deadbeef", identifier="EVDEAD"), + SimpleNamespace(id="p2", name="EVAL cafe", identifier="EVCAFE"), + SimpleNamespace(id="p3", name="Production", identifier="PROD"), + ] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + rc = cleanup_mod.main([]) # dry-run + assert rc == 0 + assert delete_calls == [] + out = capsys.readouterr().out + assert "EVAL deadbeef" in out + assert "dry-run" in out + assert "Production" not in out # prefix filter + + +def _cleanup_yes_deletes(monkeypatch, capsys, _yes): + projects = [SimpleNamespace(id="p1", name="EVAL x", identifier="EVX")] + delete_calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + return SimpleNamespace(results=projects, next_page_results=False, next_cursor="100:0:0") + + def delete(self, **kwargs): + delete_calls.append(kwargs) + + plane = MagicMock() + plane.projects = FakeProjects() + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + rc = cleanup_mod.main(["--yes"]) + assert rc == 0 + assert len(delete_calls) == 1 + assert delete_calls[0]["project_id"] == "p1" + + +def _cleanup_sentinel_mode(monkeypatch, capsys, yes): + delete_calls: list[tuple[str, str]] = [] + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace(id="customer-eval", name="Acme Corp"), + SimpleNamespace(id="customer-short", name="Acme"), + SimpleNamespace(id="customer-other", name="Other Corp"), + ] + ), + delete=lambda **kw: delete_calls.append(("customer", kw["customer_id"])), + properties=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace(id="property-eval", display_name="Eval Industry", name="eval-industry"), + SimpleNamespace(id="property-other", display_name="Region", name="region"), + ] + ), + delete=lambda **kw: delete_calls.append(("customer_property", kw["property_id"])), + ), + ), + releases=SimpleNamespace( + tags=SimpleNamespace( + list=lambda **kw: _Page( + [ + SimpleNamespace(id="tag-eval", version="eval-rc1"), + SimpleNamespace(id="tag-other", version="v2"), + ] + ), + delete=lambda **kw: delete_calls.append(("release_tag", kw["tag_id"])), + ) + ), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace(id="type-bug", name="Bug"), + SimpleNamespace(id="type-incident", name="Incident"), + SimpleNamespace(id="type-epic", name="Epic"), + ], + properties=SimpleNamespace(list=lambda **kw: ["severity-eval"] if kw.get("type_id") == "type-bug" else []), + delete=lambda **kw: delete_calls.append(("work_item_type", kw["type_id"])), + ), + workspace_work_item_properties=SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace(id="severity-eval", display_name="Severity"), + SimpleNamespace(id="property-region", display_name="Region"), + ], + delete=lambda **kw: delete_calls.append(("work_item_property", kw["property_id"])), + ), + ) + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + args = ["--sentinels", "--yes"] if yes else ["--sentinels"] + assert cleanup_mod.main(args) == 0 + output = capsys.readouterr().out + # Bug is a fixture name too, so a leftover Bug type is now a deletion target rather than + # only the type whose Severity property is removed. + for fixture_name in ("Acme Corp", "eval-rc1", "Eval Industry", "Bug", "Incident", "Severity"): + assert fixture_name in output + assert "Other Corp" not in output + assert "Region" not in output + # Reported so a zero match count cannot imply a clean workspace, but not deleted. + assert "Epic" in output + assert "did not create" in output + if yes: + assert set(delete_calls) == { + ("customer", "customer-eval"), + ("customer", "customer-short"), + ("release_tag", "tag-eval"), + ("customer_property", "property-eval"), + ("work_item_type", "type-bug"), + ("work_item_type", "type-incident"), + ("work_item_property", "severity-eval"), + } + assert ("work_item_type", "type-epic") not in set(delete_calls) + assert "deleted sentinel" in output + assert "would delete sentinel" not in output + else: + assert delete_calls == [] + assert output.count("would delete sentinel") == 7 + assert "dry-run" in output + + +def _cleanup_unowned_types_deleted_only_when_asked(monkeypatch, capsys, _yes): + delete_calls: list[tuple[str, str]] = [] + plane = SimpleNamespace( + customers=SimpleNamespace( + list=lambda **kw: _Page([]), + properties=SimpleNamespace(list=lambda **kw: _Page([])), + ), + releases=SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page([]))), + workspace_work_item_types=SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace(id="type-task-a", name="Task"), + SimpleNamespace(id="type-task-b", name="Task"), + ], + properties=SimpleNamespace(list=lambda **kw: []), + delete=lambda **kw: delete_calls.append(("work_item_type", kw["type_id"])), + ), + workspace_work_item_properties=SimpleNamespace(list=lambda **kw: []), + ) + monkeypatch.setattr("evals.seed.make_plane_client", lambda: (plane, "test-ws")) + + # No fixture-named types: without --unowned this reports nothing to delete, but must still + # surface the two it will not touch. This is the exact shape that read as clean before. + assert cleanup_mod.main(["--sentinels", "--yes"]) == 0 + output = capsys.readouterr().out + assert "sentinel_matches=0" in output + assert "nothing to delete" in output + assert output.count("'Task'") == 2 + assert delete_calls == [] + + assert cleanup_mod.main(["--sentinels", "--unowned", "--yes"]) == 0 + assert set(delete_calls) == {("work_item_type", "type-task-a"), ("work_item_type", "type-task-b")} + + # --unowned is meaningless for the project cleaner and must not be silently ignored. + assert cleanup_mod.main(["--unowned"]) == 2 + + +@pytest.mark.parametrize( + ("case", "yes"), + [ + pytest.param(_cleanup_dry_run_never_calls_delete, None, id="project-dry-run"), + pytest.param(_cleanup_yes_deletes, None, id="project-delete"), + pytest.param(_cleanup_sentinel_mode, False, id="sentinel-dry-run"), + pytest.param(_cleanup_sentinel_mode, True, id="sentinel-delete"), + pytest.param(_cleanup_unowned_types_deleted_only_when_asked, None, id="sentinel-unowned"), + ], +) +def test_cleanup_behaviours(case, yes, monkeypatch, capsys): + case(monkeypatch, capsys, yes) + + +def _list_projects_with_prefix_filters(): + projects = [ + SimpleNamespace(id="1", name="EVAL a"), + SimpleNamespace(id="2", name="Other"), + SimpleNamespace(id="3", name="EVAL b"), + SimpleNamespace(id="4", name="EVALUATION"), # must NOT match "EVAL " + ] + calls: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + calls.append({"workspace_slug": workspace_slug, "params": params}) + assert params is not None + assert params.per_page == 100 + # SDK always populates next_cursor even on last page. + return SimpleNamespace( + results=projects, + next_page_results=False, + next_cursor="100:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "3"] + assert len(calls) == 1 # one page only — no infinite loop on next_cursor + assert calls[0]["params"].cursor is None + + +def _list_projects_two_page_pagination(): + page1 = [SimpleNamespace(id="1", name="EVAL one")] + page2 = [SimpleNamespace(id="2", name="EVAL two")] + seen_cursors: list[Any] = [] + + class FakeProjects: + def list(self, workspace_slug=None, params=None): + seen_cursors.append(getattr(params, "cursor", None)) + if params.cursor is None: + return SimpleNamespace( + results=page1, + next_page_results=True, + next_cursor="100:0:0", + ) + assert params.cursor == "100:0:0" + return SimpleNamespace( + results=page2, + next_page_results=False, + next_cursor="200:0:0", + ) + + plane = MagicMock() + plane.projects = FakeProjects() + got = cleanup_mod.list_projects_with_prefix(plane, "ws", "EVAL ") + assert [p.id for p in got] == ["1", "2"] + assert seen_cursors == [None, "100:0:0"] + + +@pytest.mark.parametrize( + "case", + case_params(_list_projects_with_prefix_filters, _list_projects_two_page_pagination), +) +def test_list_projects_behaviours(case): + case() + + +def test_a_seeded_project_name_contains_nothing_that_looks_like_an_id(): + """The name is all an agent gets, and it must not offer a substring to submit as an id. + + Measured: with "EVAL 3c128f21" a weaker model sent project_id="EVAL 3c128f21" verbatim; + with "EVAL Delivery Planning (3c128f21)" it extracted the bare hex and sent that, 17 + times across six repetitions. The hex was the bait either way, so it is gone. + """ + from evals.core.fixtures import EVAL_PROJECT_PREFIX, PROJECT_SUFFIX_WORDS, eval_project_name + + for run_prefix in ("3c128f21", "cad7f69b", "deadbeef", "00000000", "ffffffff"): + for second in (False, True): + name = eval_project_name(run_prefix, second=second) + assert name.startswith(EVAL_PROJECT_PREFIX), f"cleanup --prefix must still match: {name}" + # No hex run of 6+ characters, which is what the model was pattern-matching on. + assert not re.search(r"\b[0-9a-f]{6,}\b", name, re.I), name + # No bare digits at all: a number is the other thing an id looks like. + assert not re.search(r"\d", name), name + assert name.split()[-1] in PROJECT_SUFFIX_WORDS, name + + # Deterministic in the run prefix, so a resumed run and its teardown agree on the name. + assert eval_project_name("3c128f21") == eval_project_name("3c128f21") + # The two projects stay distinguishable — R6's answer is a project name. + assert eval_project_name("3c128f21") != eval_project_name("3c128f21", second=True) + + +def test_a_non_hex_run_prefix_still_produces_a_name(): + """Never raise on the naming path: a fixture seed id shape change must not break seeding.""" + from evals.core.fixtures import eval_project_name + + assert eval_project_name("not-hex-at-all").startswith("EVAL ") diff --git a/tests/evals/tasks/__init__.py b/tests/evals/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/evals/tasks/test_answers.py b/tests/evals/tasks/test_answers.py new file mode 100644 index 0000000..0d2bcb0 --- /dev/null +++ b/tests/evals/tasks/test_answers.py @@ -0,0 +1,30 @@ +"""Offline eval tests for answers.""" + +from __future__ import annotations + + +def test_reports_contract_int_unit(): + """Direct unit cases for the contract helper.""" + from evals.tasks.answers import reports_contract_int + + assert reports_contract_int("count: 3", 3) is True + assert reports_contract_int("count: 2", 3) is False + assert reports_contract_int("-3", 3) is False + assert reports_contract_int("count: -3", 3) is False + assert reports_contract_int("0", 0) is True + assert reports_contract_int("Some prose only", 0) is False + assert reports_contract_int("preamble\ncount: 0\n", 0) is True + # Last contract line wins + assert reports_contract_int("count: 9\ncount: 3", 3) is True + assert reports_contract_int("count: 9\ncount: 3", 9) is False + + +def test_exact_line_contract_helpers_unit(): + from evals.tasks.answers import contract_values, reports_contract_value, reports_contract_values + + text = "prose mentions state Done\nSTATE: In Progress\nitem: B\nitem: A" + assert contract_values(text, "state") == ["In Progress"] + assert reports_contract_value(text, "state", "In Progress") is True + assert reports_contract_value("- state: In Progress", "state", "In Progress") is False + assert reports_contract_values(text, "item", ["A", "B"]) is True + assert reports_contract_values("item: A\nitem: A", "item", ["A"]) is False diff --git a/tests/evals/tasks/test_catalog.py b/tests/evals/tasks/test_catalog.py new file mode 100644 index 0000000..9068bbf --- /dev/null +++ b/tests/evals/tasks/test_catalog.py @@ -0,0 +1,334 @@ +"""Offline eval tests for catalog.""" + +from __future__ import annotations + +import hashlib +import inspect +import json + +import pytest + +from evals import tasks as tasks_mod +from evals.tasks.catalog import ( + TASKS, + TASKS_BY_ID, + battery_fingerprint, + get_tasks, + task_author, + task_fingerprint, + task_fingerprint_payload, +) +from evals.tasks.debias import ( + I1_TITLE, +) + +DESIGN_IDS = { + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "S1", + "S2", + "S3", + "S4", + "C1", + "C2", +} + +EXTRA_IDS = {"W9", "W10", "R7", "S5", "W11"} # bulk, pages, state inventory, features, gate recovery + +ID_IN_HAND_IDS = {"I1", "I2", "I3", "I4", "I5"} + +LONG_TAIL_IDS = {"L1", "L2", "L3", "L4", "L5"} + +NO_PROJECT_PROMPT_IDS = {"C2", "L3", "L4"} + +CATALOG_ID_ORDER = ( + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "W9", + "W10", + "W11", + "S1", + "S2", + "S3", + "S4", + "S5", + "C1", + "C2", + "R7", + "I1", + "I2", + "I3", + "I4", + "I5", + "L1", + "L2", + "L3", + "L4", + "L5", +) + +# "eea5abf36382" before CATALOG_REVISION entered the payload; "232036625e00" at revision 1; +# "9c148461e674" at revision 3 before fixture names joined the payload. +# "013109dc1c4c" at revision 3 after fixture names joined the payload. +# "9f2c2feb2e24" at revision 4 before read provenance and randomised truth. +# "7b8dc6bd2f8f" at revision 5 before unverifiable W8/W9 asks were removed. +# "77230f96962d" at revision 6 before target-entity response evidence. +# "ea5bdba36109" at revision 9 before R6 accepted per-project counts as provenance. +# "05176e65e594" at revision 10 before L1 got a worklog the agent did not create. +# "d9d087b763c8" at revision 11 before R6's second project got a Bug type of its own. +# This pin moves with every deliberate revision bump, and must not move otherwise — an +# unexplained change means the serialization drifted, which is what the pin exists to catch. +PINNED_SYNTHETIC_BATTERY = "7d77116e3299" + + +@pytest.mark.parametrize("case", ["design-and-extras", "id-order"]) +def test_catalog_behaviours(case): + if case == "id-order": + assert tuple(task["id"] for task in TASKS) == CATALOG_ID_ORDER + return + + ids = {task["id"] for task in TASKS} + for expected, label in ( + (DESIGN_IDS, "DESIGN"), + (EXTRA_IDS, "extra"), + (ID_IN_HAND_IDS, "I-class"), + (LONG_TAIL_IDS, "L-class"), + ): + assert expected.issubset(ids), f"missing {label}: {expected - ids}" + assert len(TASKS) >= 20 + + +@pytest.mark.parametrize("case", ["all-and-filter", "unknown-id"]) +def test_get_tasks_behaviours(case): + if case == "unknown-id": + with pytest.raises(SystemExit): + get_tasks(["NOPE"]) + return + + assert len(get_tasks(None)) == len(TASKS) + assert [task["id"] for task in get_tasks(["R1", "W9", "C2"])] == ["R1", "W9", "C2"] + + +@pytest.mark.parametrize("case", ["schema-invariants", "author-default"]) +def test_task_behaviours(case): + if case == "author-default": + assert task_author({}) == "claude" + assert task_author({"author": "alice"}) == "alice" + return + + for task in TASKS: + assert task["id"] + assert isinstance(task["tags"], set) + assert "{project}" in task["prompt"] or task["id"] in NO_PROJECT_PROMPT_IDS + assert callable(task["verify"]) + assert isinstance(task.get("needs"), set) + + +def test_debias_tasks_author(): + from evals.tasks.catalog import task_author + + for tid in ID_IN_HAND_IDS | LONG_TAIL_IDS: + t = TASKS_BY_ID[tid] + assert task_author(t) == "post-hoc-debias" + + +def test_w6_seeds_an_open_cycle(): + """W6 asks the agent to close Sprint 12, so the seed must leave it open. + + Plane rejects every edit to an ended cycle, so a pre-closed fixture makes the + task unachievable by design. + """ + assert "cycles_open_past" in TASKS_BY_ID["W6"]["needs"] + assert "cycles" in TASKS_BY_ID["W6"]["needs"] + + +def test_prompts_do_not_ask_for_unverifiable_w8_date_or_w9_batching(): + from plane.models.work_items import CreateWorkItemWorkLog, WorkItemWorkLog + + w8_prompt = str(TASKS_BY_ID["W8"]["prompt"]) + w9_prompt = str(TASKS_BY_ID["W9"]["prompt"]) + authoritative_date_fields = {"logged_at", "logged_date", "work_date", "date"} + assert not authoritative_date_fields.intersection(CreateWorkItemWorkLog.model_fields) + assert not authoritative_date_fields.intersection(WorkItemWorkLog.model_fields) + assert "yesterday" not in w8_prompt.casefold() + assert "in one batch" not in w9_prompt.casefold() + + +def test_verifiers_are_async_and_importable(): + modules = { + "R": "read", + "W": "write", + "S": "schema", + "C": "cross", + "I": "debias", + "L": "debias", + } + for t in TASKS: + fn = t["verify"] + assert inspect.iscoroutinefunction(fn), t["id"] + # Callables resolve without NameError + assert fn.__module__ == f"evals.tasks.{modules[t['id'][0]]}" + + +def test_tasks_module_has_no_hardcoded_uuids(): + """Regression: verifiers must resolve expected values at verify time.""" + src = inspect.getsource(tasks_mod) + # Crude: no UUID-shaped literals in tasks module. + assert not any(len(part) == 36 and part.count("-") == 4 for part in src.replace('"', " ").replace("'", " ").split()) + + +@pytest.mark.parametrize( + "case", + ["strict-empty", "strict-exception", "dry-run-markers", "strict-success"], +) +def test_prompt_bind_behaviours(case): + from evals.tasks.prompts import PromptBindError, format_task_prompt + + task = TASKS_BY_ID["I1"] + if case == "strict-empty": + with pytest.raises(PromptBindError): + format_task_prompt(task, {"project_name": "P", "items": {}}, strict=True) + elif case == "strict-exception": + + def boom(_ctx): + raise RuntimeError("seed broken") + + custom = {"id": "X", "prompt": "do {work_item_id}", "prompt_bind": boom} + with pytest.raises(PromptBindError, match="prompt_bind failed"): + format_task_prompt(custom, {"project_name": "P"}, strict=True) + elif case == "dry-run-markers": + text = format_task_prompt(task, {"project_name": "EVAL x"}, strict=False) + assert "" in text and "EVAL x" in text + else: + text = format_task_prompt( + task, + {"project_name": "P", "items": {I1_TITLE: "uuid-abc"}}, + strict=True, + ) + assert "uuid-abc" in text and "<" not in text + + +@pytest.mark.parametrize( + "case", + ["stable-and-sensitive", "catalog-nonempty", "debias-tasks-change-hash"], +) +def test_battery_fingerprint_behaviours(case): + if case == "catalog-nonempty": + fingerprint = battery_fingerprint() + assert len(fingerprint) == 12 + assert battery_fingerprint(list(TASKS)) == fingerprint + return + if case == "debias-tasks-change-hash": + full = battery_fingerprint() + without_debias = [task for task in TASKS if not str(task.get("id", "")).startswith(("I", "L"))] + assert without_debias, "pre-debias catalog should be non-empty" + reduced = battery_fingerprint(without_debias) + assert reduced != full + assert battery_fingerprint(without_debias + [TASKS_BY_ID["I1"]]) != reduced + return + + task_a = {"id": "A", "prompt": "p1 {project}"} + task_b = {"id": "B", "prompt": "p2"} + assert battery_fingerprint([task_b, task_a]) == battery_fingerprint([task_a, task_b]) == PINNED_SYNTHETIC_BATTERY + assert battery_fingerprint([{**task_a, "prompt": "p1 edited {project}"}, task_b]) != PINNED_SYNTHETIC_BATTERY + assert battery_fingerprint([task_a]) != PINNED_SYNTHETIC_BATTERY + + +def test_revision_bump_changes_the_fingerprint_for_an_unchanged_catalog(): + """A fixture/verifier correction is expressible even though the hash ignores them. + + The per-task payload deliberately omits verifier bodies, so without the revision a + corrected verifier could keep the old fingerprint and go on asserting that results + graded against a different contract are comparable. + """ + from evals.tasks import catalog + + tasks = list(catalog.TASKS) + before = battery_fingerprint(tasks) + original = catalog.CATALOG_REVISION + try: + catalog.CATALOG_REVISION = original + 1 + after = battery_fingerprint(tasks) + finally: + catalog.CATALOG_REVISION = original + + assert after != before, "bumping the revision must move the fingerprint" + assert battery_fingerprint(tasks) == before, "restoring the revision must restore it" + + +def test_task_and_battery_fingerprints_share_the_same_per_task_payload(monkeypatch): + from evals.tasks import catalog + + task = {"id": "T1", "prompt": "prompt", "needs": {"projects", "states"}} + expected_payload = {"id": "T1", "prompt": "prompt", "needs": ["projects", "states"]} + assert task_fingerprint_payload(task) == expected_payload + + def short_hash(document): + blob = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:12] + + assert task_fingerprint(task) == short_hash(expected_payload) + sentinel_payload = {"id": "sentinel", "prompt": "shared", "needs": []} + monkeypatch.setattr(catalog, "task_fingerprint_payload", lambda _task: sentinel_payload) + assert catalog.task_fingerprint(task) == short_hash(sentinel_payload) + assert catalog.battery_fingerprint([task]) == short_hash( + {"revision": catalog.CATALOG_REVISION, "tasks": [sentinel_payload]} + ) + + +def test_fingerprint_records_the_revision_transition(): + """Pin the current value, so a future change is read as intentional, not drift. + + ``d546d3181bdb`` was the fingerprint before the revision field existed (batteries 6-8). + Revision 3 drops author-declared tool sets and call floors and adds fixture names, so + the hash covers what the agent was asked and what it was given — never how anyone + expected it to answer. Its final full-catalog value was ``d89173c744cc``. Revision 4 + rewrites R7 to replace its unconditional-pass transition question with an exact + state-and-group listing; its full-catalog value was ``0c9b6fc0405e``. Revision 5 + adds successful Plane-call provenance and randomised API-confirmed seed truth to the + read family; its full-catalog value was ``075bbd409f15``. Revision 6 removes W8's + unverifiable logged-date ask and W9's unverifiable batching ask, and tightens the + affected end-state verifiers; its full-catalog value was ``ccf39203f656``. Revision 7 + binds read provenance to target-entity response evidence and gives C2/R7 randomised, + immutable seed-time oracles; its full-catalog value was ``9ea76bf22ba0``. Revision 8 binds + L2's provenance to the activity count its verifier already checks, because the seeded + comment phrase it used to require is never present in Plane's activity payload; its + full-catalog value was ``61bddef0dc76``. Revision 9 also binds R1/I2 provenance to the seeded + state, because a work item's state is an id and resolving its name takes a second call; + its full-catalog value was ``e9134604a0a7``. Revision 10 accepts one count per project as + R6 provenance, alongside the single count grouped by project it already accepted; its + full-catalog value was ``9cce7ce77310``. Revision 11 gives L1 a worklog the agent did not + create, because reporting the id it had just logged time on required no summary read; its + full-catalog value was ``fa1784b052cd``. Revision 12 gives R6's second project a Bug type of + its own, because its bugs carried the main project's type and an agent resolving 'Bug' there + counted zero. Results across these transitions are not comparable. + Asserting the constant rather than merely 'it changed' makes future drift visible. + """ + from evals.tasks.catalog import CATALOG_REVISION + + assert CATALOG_REVISION == 12 + assert battery_fingerprint() == "eaf35e8019aa" diff --git a/tests/evals/tasks/test_debias_verifiers.py b/tests/evals/tasks/test_debias_verifiers.py new file mode 100644 index 0000000..53fb86e --- /dev/null +++ b/tests/evals/tasks/test_debias_verifiers.py @@ -0,0 +1,400 @@ +"""Offline eval tests for debias verifiers.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.seed import W2_TITLE +from evals.tasks.debias import ( + I1_TITLE, + I3_TITLE, + I4_TITLE, + L1_TITLE, + L2_TITLE, + L3_TAG_VERSION, + L4_PROP_DISPLAY, + L4_PROP_VALUE, + L5_TITLE, + verify_i1, + verify_i2, + verify_i3, + verify_i4, + verify_i5, + verify_l1, + verify_l2, + verify_l3, + verify_l4, + verify_l5, +) + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _run(text: str = "") -> dict[str, Any]: + return { + "final_text": text, + "calls": [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + "call_source": "test", + "evidence_trace_available": True, + } + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +class _WIRetrievePlane: + """work_items.retrieve + list by name; optional labels expand.""" + + def __init__( + self, + *, + by_id: dict[str, Any], + by_name: dict[str, str] | None = None, + states: list[Any] | None = None, + ): + self._by_id = by_id + self._by_name = by_name or {} + self._states = states or [] + self.work_items = SimpleNamespace( + list=self._list, + retrieve=self._retrieve, + ) + self.states = SimpleNamespace(list=lambda **kw: _Page(self._states)) + + def _list(self, **kw): + # Minimal name filter support used by _find_item_by_name. + params = kw.get("params") + name = None + if params is not None: + name = getattr(params, "name", None) or (params.get("name") if isinstance(params, dict) else None) + if name and name in self._by_name: + wid = self._by_name[name] + row = self._by_id.get(wid) or _item(wid, name) + return _Page([row]) + return _Page([]) + + def _retrieve(self, **kw): + wid = str(kw["work_item_id"]) + if wid not in self._by_id: + raise LookupError(wid) + return self._by_id[wid] + + +class _I3Plane: + def __init__(self, cycle_item_ids: list[str]): + self.cycles = SimpleNamespace(list_work_items=lambda **kw: _Page([_item(i, f"n-{i}") for i in cycle_item_ids])) + + +class _L1Plane: + def __init__(self, durations: list[int], summary_ids: list[str] | None = None): + self.work_items = SimpleNamespace( + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + rows = [SimpleNamespace(work_item_id=i, duration=90) for i in (summary_ids or [])] + self.projects = SimpleNamespace(get_worklog_summary=lambda **kw: rows) + + +class _L2Plane: + def __init__(self, n_activities: int): + acts = [SimpleNamespace(id=f"a{i}", verb="updated") for i in range(n_activities)] + self.work_items = SimpleNamespace(activities=SimpleNamespace(list=lambda **kw: _Page(acts))) + + +class _L3Plane: + def __init__(self, versions: list[str]): + tags = [SimpleNamespace(id=f"t-{v}", version=v) for v in versions] + self.releases = SimpleNamespace(tags=SimpleNamespace(list=lambda **kw: _Page(tags))) + + +class _L4Plane: + def __init__(self, *, props: list[Any], values: dict[str, list[str]]): + self.customers = SimpleNamespace( + properties=SimpleNamespace(list=lambda **kw: _Page(props)), + property_values=SimpleNamespace(list=lambda **kw: values), + ) + + +class _L5Plane: + def __init__(self, n: int): + rows = [SimpleNamespace(id=f"att-{i}") for i in range(n)] + self.work_items = SimpleNamespace(attachments=SimpleNamespace(list=lambda **kw: _Page(rows))) + + +BACKLOG = SimpleNamespace(id="st-backlog", name="Backlog", group="unstarted") +DONE = SimpleNamespace(id="st-done", name="Done", group="completed") + + +def _i1_ctx(): + return {"workspace_slug": "ws", "project_id": "p1", "items": {I1_TITLE: "wi-1"}} + + +def test_i1_passes_only_when_the_target_item_itself_changed(): + """Priority must land on the item the task names, not merely somewhere.""" + + async def _go(): + cases = [ + ( + "untouched: target still urgent", + _WIRetrievePlane(by_id={"wi-1": SimpleNamespace(id="wi-1", priority="urgent")}), + (), + ), + ( + "right value on the wrong item", + _WIRetrievePlane( + by_id={ + "wi-1": SimpleNamespace(id="wi-1", priority="urgent"), + "wi-other": SimpleNamespace(id="wi-other", priority="high"), + } + ), + ("urgent", "high"), + ), + ] + for label, plane, expect_any in cases: + ok, note = await verify_i1(plane, _i1_ctx(), _run()) + assert ok is False, f"{label}: {note}" + if expect_any: + assert any(s in note for s in expect_any), f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_i2_requires_the_state_contract_to_name_the_real_state(): + async def _go(): + def plane_for(states): + return _WIRetrievePlane(by_id={"wi-2": SimpleNamespace(id="wi-2", state=BACKLOG)}, states=states) + + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {W2_TITLE: "wi-2"}, + "i2_state_name": "Backlog", + } + cases = [ + ("untouched: empty answer", [BACKLOG], "", False), + ("names a different state", [BACKLOG, DONE], "Done", False), + ("exact contract line", [BACKLOG], "state: Backlog", True), + ] + for label, states, text, want in cases: + ok, note = await verify_i2(plane_for(states), dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_i3_fails_unless_the_target_item_is_on_the_cycle(): + async def _go(): + cases = [ + ("untouched: target never added", ["other-1", "other-2"]), + ("added the wrong item", ["wrong-item"]), + ] + for label, on_cycle in cases: + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I3_TITLE: "footer-1"}, + "cycle_current_id": "cyc-1", + } + ok, note = await verify_i3(_I3Plane(on_cycle), ctx, _run()) + assert ok is False, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_i4_requires_the_named_label_on_the_target(): + async def _go(): + cases = [ + ("untouched: no labels", []), + ("a different label attached", [SimpleNamespace(id="lab-auth")]), + ] + for label, labels in cases: + plane = _WIRetrievePlane(by_id={"wi-4": SimpleNamespace(id="wi-4", labels=labels)}) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {I4_TITLE: "wi-4"}, + "labels": {"perf": "lab-perf"}, + } + ok, note = await verify_i4(plane, ctx, _run()) + assert ok is False, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_i5_rejects_both_untouched_and_wrong_priority(): + async def _go(): + cases = [ + ("untouched: priority none", "none", ()), + ("wrong value: high", "high", ("high",)), + ] + for label, priority, expect in cases: + plane = _WIRetrievePlane(by_id={"wi-5": SimpleNamespace(id="wi-5", priority=priority)}) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {I3_TITLE: "wi-5"}} + ok, note = await verify_i5(plane, ctx, _run()) + assert ok is False, f"{label}: {note}" + for s in expect: + assert s in note, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_l1_grades_the_duration_contract_not_the_prose(): + """Prose stating the right facts still fails: the format is part of the task. + + Covers the counterexamples — English "ninety" is not a number, and a correct log with an empty answer. + """ + + async def _go(): + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L1_TITLE: "wi-l1"}, + "l1_expected_summary_ids": ["wi-l1", "wi-seeded"], + } + cases = [ + ("untouched: no worklog", [], None, "", False, ()), + ("wrong duration 120", [120], ["wi-l1", "wi-seeded"], "Logged 120 minutes; summary ok.", False, ("90",)), + ("90m logged but empty answer", [90], ["wi-l1", "wi-seeded"], "", False, ("logged-minutes",)), + ( + "English 'ninety' is not a number", + [90], + ["wi-l1", "wi-seeded"], + "Logged one hundred ninety minutes. Project summary looks fine.", + False, + (), + ), + ("correct facts, no contract", [90], ["wi-l1", "wi-seeded"], "Logged 1.5 hours total.", False, ()), + ("bare prose, no contract", [90], ["wi-l1", "wi-seeded"], "90 minutes of work", False, ("logged-minutes",)), + ( + "exact contract", + [90], + ["wi-l1", "wi-seeded"], + "logged-minutes: 90\nsummary-work-item-id: wi-l1\nsummary-work-item-id: wi-seeded", + True, + (), + ), + ] + for label, durations, summary_ids, text, want, expect in cases: + plane = _L1Plane(durations, summary_ids=summary_ids) + ok, note = await verify_l1(plane, dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" + for s in expect: + assert s in note.lower(), f"{label}: {note}" + + # The 'ninety' case must name the duration it objected to. + plane = _L1Plane([90], summary_ids=["wi-l1", "wi-seeded"]) + _, note = await verify_l1( + plane, dict(ctx), _run("Logged one hundred ninety minutes. Project summary looks fine.") + ) + assert "duration" in note.lower() or "90" in note or "1.5" in note, note + + mutated_ok, mutated_note = await verify_l1( + _L1Plane([90], summary_ids=["wi-l1", "agent-added"]), + dict(ctx), + _run("logged-minutes: 90\nsummary-work-item-id: wi-l1\nsummary-work-item-id: agent-added"), + ) + assert mutated_ok is False + assert "mutated beyond the seeded oracle" in mutated_note + + return asyncio.run(_go()) + + +def test_l2_counts_activities_through_the_contract_only(): + async def _go(): + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L2_TITLE: "wi-l2"}, + "l2_activity_count": 3, + } + cases = [ + ("untouched: empty answer", "", False), + ("contract matches truth 3", "Saw some history.\ncount: 3", True), + ("contract says 2, truth 3", "count: 2", False), + ("bare negative", "-3", False), + ("negative contract", "count: -3", False), + ("prose without contract", "There are 3 activities and some comment phrases.", False), + ] + for label, text, want in cases: + ok, note = await verify_l2(_L2Plane(3), dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_l3_requires_the_exact_release_tag_version(): + async def _go(): + cases = [("untouched: no tags", [], ()), ("wrong version", ["v0.0.1", "other-rc"], (L3_TAG_VERSION,))] + for label, versions, expect in cases: + ok, note = await verify_l3(_L3Plane(versions), {"workspace_slug": "ws"}, _run()) + assert ok is False, f"{label}: {note}" + for s in expect: + assert s in note, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_l4_matches_the_property_on_name_type_and_value(): + """A URL-typed property whose name merely contains "Industry" must not satisfy it.""" + + async def _go(): + exact = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, name="eval-industry", property_type="TEXT") + loose = SimpleNamespace(id="prop-url", display_name="Industry", name="industry", property_type="URL") + cases = [ + ("untouched: no property", [], {}, False, ()), + ( + "right property, wrong value", + [exact], + {"prop-1": ["Startup"]}, + False, + (L4_PROP_VALUE, "Startup", "lack"), + ), + ("wrong name and type, right value", [loose], {"prop-url": [L4_PROP_VALUE]}, False, ()), + ("exact text property", [exact], {"prop-1": [L4_PROP_VALUE]}, True, ()), + ] + for label, props, values, want, expect_any in cases: + ctx = {"workspace_slug": "ws", "customer": {"id": "cust-1", "name": "Acme Corp"}} + ok, note = await verify_l4(_L4Plane(props=props, values=values), ctx, _run()) + assert ok is want, f"{label}: {note}" + if expect_any: + assert any(s in note for s in expect_any), f"{label}: {note}" + if want: + assert any(o.get("kind") == "customer_property" for o in ctx.get("workspace_objects") or []) + + return asyncio.run(_go()) + + +def test_l5_accepts_the_api_confirmed_count_only_through_the_contract(): + async def _go(): + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L5_TITLE: "wi-l5"}, + "l5_attachment_count": 2, + } + cases = [ + ("untouched: empty answer", "", False), + ("bare count as the whole answer", "2", True), + ("multiline ending in the contract", "Two files on this work item.\ncount: 2", True), + ("prose without contract", "There are 2 attachments.", False), + ("contract with the wrong count", "count: 10", False), + ] + for label, text, want in cases: + ok, note = await verify_l5(_L5Plane(2), dict(ctx), _run(text)) + assert ok is want, f"{label}: {note}" + + return asyncio.run(_go()) diff --git a/tests/evals/tasks/test_gate_recovery.py b/tests/evals/tasks/test_gate_recovery.py new file mode 100644 index 0000000..49aaf76 --- /dev/null +++ b/tests/evals/tasks/test_gate_recovery.py @@ -0,0 +1,115 @@ +"""W11: the agent must clear a disabled project feature before it can do the work. + +W8 logs two hours against a project where time tracking is already on. W11 is the same +end state reached from an obstacle — the worklog endpoints refuse until the feature is +enabled, which the prompt explicitly permits. The verifier separates the ways it can go +wrong, because "no work log" alone does not say whether the agent gave up, half-finished, +or claimed a success it never earned. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.tasks.catalog import TASKS_BY_ID +from evals.tasks.verification import VerifierReadError +from evals.tasks.write import W11_TITLE, verify_w11 + +WORKLOG_DISABLED = HttpError("Not found", 404, {"message": "Worklog is not enabled for the project"}) + + +class _Page: + """The paginated envelope the SDK returns from a list endpoint.""" + + def __init__(self, results: list[Any] | None = None, next_page_results: bool = False) -> None: + self.results = results or [] + self.next_page_results = next_page_results + self.next_cursor = None + + +def _plane(*, logs: Any, time_tracking: bool | None = True, item: bool = True) -> SimpleNamespace: + def _list_items(**_kwargs: Any): + return _Page([SimpleNamespace(id="item-1", name=W11_TITLE)] if item else []) + + def _list_logs(**_kwargs: Any): + if isinstance(logs, Exception): + raise logs + return logs + + def _retrieve_project(**_kwargs: Any): + return SimpleNamespace(id="proj-1", is_time_tracking_enabled=time_tracking) + + return SimpleNamespace( + work_items=SimpleNamespace( + list=_list_items, + work_logs=SimpleNamespace(list=_list_logs), + ), + projects=SimpleNamespace(retrieve=_retrieve_project), + ) + + +CTX = {"workspace_slug": "ws", "project_id": "proj-1"} + + +def test_passes_when_the_work_log_landed(): + plane = _plane(logs=[SimpleNamespace(duration=120)]) + ok, note = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "done"})) + assert ok is True + assert "120" in note + + +def test_still_refused_means_the_feature_was_never_enabled(): + plane = _plane(logs=WORKLOG_DISABLED, time_tracking=False) + ok, note = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "I could not log the time."})) + assert ok is False + assert "never enabled" in note + + +def test_enabled_but_nothing_logged_is_reported_as_half_finished(): + plane = _plane(logs=[], time_tracking=True) + ok, note = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "Enabled time tracking."})) + assert ok is False + assert "enabled but no 120-minute log" in note + + +def test_reporting_the_limitation_is_distinguished_from_giving_up_silently(): + """Both fail, but they are different behaviours and the report should say which.""" + reported = _plane(logs=[], time_tracking=False) + ok, note = asyncio.run( + verify_w11(reported, dict(CTX), {"final_text": "Time tracking is not enabled for this project."}) + ) + assert ok is False + assert "rather than enabling it" in note + + silent = _plane(logs=[], time_tracking=False) + ok, note = asyncio.run(verify_w11(silent, dict(CTX), {"final_text": "All done!"})) + assert ok is False + assert "never mentions" in note + + +def test_a_do_nothing_agent_fails(): + """The canary asserts this for every verifier; a gate task must not pass vacuously.""" + plane = _plane(logs=[], time_tracking=False) + ok, _ = asyncio.run(verify_w11(plane, dict(CTX), {"final_text": "", "calls": []})) + assert ok is False + + +def test_an_unexpected_error_is_not_swallowed_as_a_disabled_feature(): + """Only the 'worklog disabled' 404 is read as the obstacle; anything else is a bug.""" + plane = _plane(logs=HttpError("Server error", 500, {"error": "boom"})) + with pytest.raises(VerifierReadError, match="W11 verifier read failed while listing work logs"): + asyncio.run(verify_w11(plane, dict(CTX), {"final_text": ""})) + + +def test_task_seeds_time_tracking_off_and_authorises_turning_it_on(): + task = TASKS_BY_ID["W11"] + assert "leave_worklogs_off" in task["needs"], "the obstacle must actually be seeded" + assert "items" in task["needs"] + # Without explicit permission, an agent that declines to change project-wide config is + # arguably behaving better, and scoring the enable as success would reward overreach. + assert "permission" in task["prompt"].lower() diff --git a/tests/evals/tasks/test_lookups.py b/tests/evals/tasks/test_lookups.py new file mode 100644 index 0000000..f63a655 --- /dev/null +++ b/tests/evals/tasks/test_lookups.py @@ -0,0 +1,23 @@ +"""Shape-tolerance tests for shared verifier lookups.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from evals.tasks.lookups import state_group + + +@pytest.mark.parametrize( + "response", + [ + [SimpleNamespace(id="state-1", group="started")], + SimpleNamespace(results=[SimpleNamespace(id="state-1", group="started")]), + ], + ids=["raw-list", "paginated-page"], +) +def test_state_group_accepts_raw_and_paginated_list_shapes(response): + plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: response)) + + assert state_group(plane, "ws", "project", "state-1") == "started" diff --git a/tests/evals/tasks/test_output_contracts.py b/tests/evals/tasks/test_output_contracts.py new file mode 100644 index 0000000..4fd3865 --- /dev/null +++ b/tests/evals/tasks/test_output_contracts.py @@ -0,0 +1,558 @@ +"""Offline eval tests for structural output contracts.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.seed import CYCLE_CURRENT, R1_TITLE, R5_COMMENT_PHRASES, W2_TITLE, W8_TITLE +from evals.tasks.cross import verify_c2 +from evals.tasks.debias import verify_i2, verify_l2, verify_l5 +from evals.tasks.read import verify_r1, verify_r2, verify_r3, verify_r4, verify_r5, verify_r6, verify_r7 +from evals.tasks.write import verify_w2, verify_w4, verify_w8 +from tests.evals.conftest import case_params + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _run(text: str = "", *, calls: list[dict[str, Any]] | None = None) -> dict[str, Any]: + return { + "final_text": text, + "calls": ( + [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ] + if calls is None + else calls + ), + "call_source": "test", + "evidence_trace_available": True, + } + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +class _R1Plane: + def __init__(self, state_name: str): + st = SimpleNamespace(id="st-1", name=state_name, group="started") + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("r1", R1_TITLE, state=st)]), + retrieve=lambda **kw: SimpleNamespace(id="r1", name=R1_TITLE, state=st), + ) + self.states = SimpleNamespace( + list=lambda **kw: _Page( + [ + st, + SimpleNamespace(id="st-2", name="Done", group="completed"), + SimpleNamespace(id="st-3", name="Backlog", group="unstarted"), + ] + ) + ) + + +class _W2Plane: + def __init__(self, group: str, name: str): + st = SimpleNamespace(id="st", name=name, group=group) + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w2", W2_TITLE, state=st)]), + retrieve=lambda **kw: SimpleNamespace(id="w2", state=st), + ) + self.states = SimpleNamespace(list=lambda **kw: _Page([st])) + + +class _W4Plane: + def __init__(self, name: str): + self.labels = SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(id=kw["label_id"], name=name), + list=lambda **kw: _Page([SimpleNamespace(id="triage-id", name=name)]), + ) + + +class _W8Plane: + def __init__(self, durations: list[int]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w8", W8_TITLE)]), + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + + +class _C2Plane: + def __init__(self, changelog_html: str = "", error: Exception | None = None, release_name: str = "1.2.0"): + def retrieve(**kwargs): + if error is not None: + raise error + return SimpleNamespace(description_html=changelog_html) + + self.releases = SimpleNamespace( + retrieve=lambda **kwargs: SimpleNamespace(name=release_name), + changelog=SimpleNamespace(retrieve=retrieve), + ) + + +def _r2_written_number_prose_fails_and_count_contract_passes(): + async def _go(): + state = SimpleNamespace(id="started", name="Started", group="started") + items = [SimpleNamespace(id=str(index), priority="urgent", state=state) for index in range(4)] + plane = SimpleNamespace( + states=SimpleNamespace(list=lambda **kwargs: _Page([state])), + work_items=SimpleNamespace(list=lambda **kwargs: _Page(items)), + ) + ctx = {"workspace_slug": "ws", "project_id": "project", "r2_urgent_open_count": 4} + + prose_ok, _ = await verify_r2(plane, ctx, _run("There are four urgent open work items.")) + contract_ok, note = await verify_r2(plane, ctx, _run("count: 4")) + + assert prose_ok is False + assert contract_ok is True, note + + return asyncio.run(_go()) + + +def _r2_rejects_a_count_that_disagrees_with_the_api(): + async def _go(): + from evals.tasks.read import verify_r2 as _vr2 + + urgent = [_item(str(i), "x", priority="urgent", state=SimpleNamespace(group="started")) for i in range(4)] + + class Plane: + work_items = SimpleNamespace(list=lambda **kw: _Page(urgent)) + states = SimpleNamespace( + list=lambda **kw: _Page([SimpleNamespace(id="s", name="S", group="started", default=False)]) + ) + + ctx = {"workspace_slug": "ws", "project_id": "p1", "r2_urgent_open_count": 4} + ok, note = await _vr2(Plane(), ctx, _run("0")) + assert ok is False, note + + return asyncio.run(_go()) + + +@pytest.mark.parametrize( + "case", + case_params( + _r2_written_number_prose_fails_and_count_contract_passes, + _r2_rejects_a_count_that_disagrees_with_the_api, + ), +) +def test_r2_behaviours(case): + case() + + +def test_r4_contract_requires_cycle_items_and_exact_overdue_title(): + async def _go(): + overdue = "Session cookie not rotated after login" + ctx = { + "r4_cycle_name": CYCLE_CURRENT, + "r4_active_titles": [R1_TITLE, overdue], + "r4_overdue_titles": [overdue], + } + text = f"cycle: {CYCLE_CURRENT}\nitem: {R1_TITLE}\nitem: {overdue}\noverdue: {overdue}" + + ok, note = await verify_r4(object(), ctx, _run(text)) + keyword_only_ok, _ = await verify_r4(object(), ctx, _run(f"cycle: {CYCLE_CURRENT}\noverdue")) + + assert ok is True, note + assert keyword_only_ok is False + + return asyncio.run(_go()) + + +def test_r5_exact_comment_lines_pass_but_free_prose_does_not(): + async def _go(): + ctx = {"r5_comment_phrases": list(R5_COMMENT_PHRASES)} + contract = "\n".join(f"comment: {phrase}" for phrase in reversed(R5_COMMENT_PHRASES)) + prose = f"The discussion covered {R5_COMMENT_PHRASES[0]} and {R5_COMMENT_PHRASES[1]}." + + contract_ok, note = await verify_r5(object(), ctx, _run(contract)) + prose_ok, _ = await verify_r5(object(), ctx, _run(prose)) + + assert contract_ok is True, note + assert prose_ok is False + + return asyncio.run(_go()) + + +def test_r6_exact_project_contract_passes_and_shorthand_fails(): + async def _go(): + expected = "EVAL deadbeef B" + ctx = {"r6_more_bugs_project": expected} + + exact_ok, note = await verify_r6(object(), ctx, _run(f"project: {expected}")) + shorthand_ok, _ = await verify_r6(object(), ctx, _run("The B project has more bugs.")) + + assert exact_ok is True, note + assert shorthand_ok is False + + return asyncio.run(_go()) + + +def test_read_provenance_matrix_and_canary_coverage(): + async def _go(): + ctx = {"r2_urgent_open_count": 4} + no_call_ok, no_call_note = await verify_r2(object(), ctx, _run("count: 4", calls=[])) + assert no_call_ok is False + assert "answer_correct=true" in no_call_note + assert "provenance=missing" in no_call_note + + successful_ok, successful_note = await verify_r2(object(), ctx, _run("count: 4")) + assert successful_ok is True, successful_note + assert "answer_correct=true" in successful_note + assert "provenance=observed" in successful_note + + unrelated_ok, unrelated_note = await verify_r2( + object(), + ctx, + _run( + "count: 4", + calls=[{"tool": "plane_call", "is_error": False, "observed_sentinels": []}], + ), + ) + assert unrelated_ok is False + assert "answer_correct=true" in unrelated_note + assert "0 evidence-bearing of 1 successful" in unrelated_note + + failed_call_ok, failed_call_note = await verify_r2( + object(), + ctx, + _run("count: 4", calls=[{"tool": "plane_call", "is_error": True}]), + ) + assert failed_call_ok is False + assert "answer_correct=true" in failed_call_note + assert "0 evidence-bearing of 0 successful" in failed_call_note + + wrong_ok, wrong_note = await verify_r2(object(), ctx, _run("count: 3")) + assert wrong_ok is False + assert "answer_correct=false" in wrong_note + assert "provenance=observed" in wrong_note + + unavailable_ok, unavailable_note = await verify_r2( + object(), + ctx, + {"final_text": "count: 4"}, + ) + assert unavailable_ok is False + assert "provenance=unavailable" in unavailable_note + + incomplete_ok, incomplete_note = await verify_r2( + object(), + ctx, + { + **_run("count: 4"), + "driver_notes": ["proxy_sidecar_incomplete:skipped_rows=1"], + "trace_integrity": False, + "trace_integrity_reason": "recorder_loss", + }, + ) + assert incomplete_ok is False + assert "answer_correct=true" in incomplete_note + assert "provenance=trace incomplete" in incomplete_note + assert "sentinel" not in incomplete_note + + # The real canary supplies this exact empty trace. Every affected read verifier + # must reject even when its text happens to be correct. + empty_run = { + "final_text": "", + "calls": [], + "call_source": "canary", + "evidence_trace_available": False, + } + cases = [ + ("R1", verify_r1, {"r1_state_name": "Investigating 4821"}, "state: Investigating 4821"), + ("R2", verify_r2, {"r2_urgent_open_count": 6}, "count: 6"), + ("R3", verify_r3, {"r3_due_titles": ["Due case 4821"]}, "item: Due case 4821"), + ( + "R4", + verify_r4, + { + "r4_cycle_name": "Sprint 47", + "r4_active_titles": ["Active case 4821"], + "r4_overdue_titles": ["Active case 4821"], + }, + "cycle: Sprint 47\nitem: Active case 4821\noverdue: Active case 4821", + ), + ("R5", verify_r5, {"r5_comment_phrases": ["comment ref-4821"]}, "comment: comment ref-4821"), + ("R6", verify_r6, {"r6_more_bugs_project": "EVAL deadbeef"}, "project: EVAL deadbeef"), + ("I2", verify_i2, {"i2_state_name": "Investigating 4821"}, "state: Investigating 4821"), + ("L2", verify_l2, {"l2_activity_count": 3}, "count: 3"), + ("L5", verify_l5, {"l5_attachment_count": 2}, "count: 2"), + ] + for task_id, verifier, task_ctx, correct_text in cases: + ok, note = await verifier(object(), task_ctx, {**empty_run, "final_text": correct_text}) + assert ok is False, f"{task_id}: {note}" + assert "answer_correct=true" in note, f"{task_id}: {note}" + assert "provenance=unavailable" in note, f"{task_id}: {note}" + + return asyncio.run(_go()) + + +def test_r7_state_group_contract_matches_live_api_exactly(): + async def _go(): + states = [ + SimpleNamespace(name="Backlog", group="backlog"), + SimpleNamespace(name="In Progress", group="started"), + SimpleNamespace(name="Done", group="completed"), + ] + plane = SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(states))) + ctx = { + "workspace_slug": "ws", + "project_id": "project", + "r7_state_pairs": [ + "Backlog | group: backlog", + "In Progress | group: started", + "Done | group: completed", + ], + } + + exact = "\n".join( + [ + "state: Done | group: completed", + "state: Backlog | group: backlog", + "state: In Progress | group: started", + ] + ) + exact_ok, note = await verify_r7(plane, ctx, _run(exact)) + unrestricted_ok, _ = await verify_r7(plane, ctx, _run("state: unrestricted")) + wrong_group_ok, _ = await verify_r7( + plane, + ctx, + _run(exact.replace("Done | group: completed", "Done | group: started")), + ) + names_only_ok, _ = await verify_r7(plane, ctx, _run("state: Backlog\nstate: In Progress\nstate: Done")) + prose_ok, _ = await verify_r7(plane, ctx, _run("It can move to Done.")) + empty_ok, _ = await verify_r7(plane, ctx, _run()) + + assert exact_ok is True, note + assert unrestricted_ok is False + assert wrong_group_ok is False + assert names_only_ok is False + assert prose_ok is False + assert empty_ok is False + + return asyncio.run(_go()) + + +def test_r7_rejects_oracle_mutation_and_zero_call_default_state_cans(): + async def _go(): + baseline = [ + "Backlog | group: backlog", + "In Progress | group: started", + "Done | group: completed", + "Review 7b0a1f9c | group: started", + ] + ctx = {"workspace_slug": "ws", "project_id": "project", "r7_state_pairs": baseline} + mutated_states = [ + SimpleNamespace(name="Backlog", group="backlog"), + SimpleNamespace(name="In Progress", group="started"), + SimpleNamespace(name="Done", group="completed"), + SimpleNamespace(name="Agent Rewrite", group="completed"), + ] + mutated_answer = "\n".join( + f"state: {value}" + for value in [ + "Backlog | group: backlog", + "In Progress | group: started", + "Done | group: completed", + "Agent Rewrite | group: completed", + ] + ) + mutated_ok, mutated_note = await verify_r7( + SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(mutated_states))), + ctx, + _run(mutated_answer), + ) + assert mutated_ok is False + assert "oracle was mutated after seeding" in mutated_note + + live_baseline = [ + SimpleNamespace(name=value.split(" | group: ")[0], group=value.split(" | group: ")[1]) for value in baseline + ] + canned = "state: Backlog | group: backlog\nstate: In Progress | group: started\nstate: Done | group: completed" + canned_ok, canned_note = await verify_r7( + SimpleNamespace(states=SimpleNamespace(list=lambda **kwargs: _Page(live_baseline))), + ctx, + _run(canned, calls=[]), + ) + assert canned_ok is False + assert "answer_correct=false" in canned_note + assert "provenance=missing" in canned_note + + return asyncio.run(_go()) + + +CHANGELOG = "Changelog entry one: OAuth login hardening. Changelog entry two: webhook retry backoff." +R1_CTX = { + "workspace_slug": "ws", + "project_id": "p1", + "r1_state_name": "In Progress", + "state_names": ["In Progress", "Done", "Backlog"], +} + + +def test_r1_accepts_only_the_exact_state_contract(): + async def _go(): + cases = [ + ("untouched: empty answer", "", False), + ("names a different state", "Done", False), + ("exact contract line", "state: In Progress", True), + ] + for label, text, want in cases: + ok, note = await verify_r1(_R1Plane("In Progress"), dict(R1_CTX), _run(text)) + assert ok is want, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_w2_requires_the_exact_done_state(): + """Other terminal and completed-group states are not the requested Done state.""" + + async def _go(): + cases = [ + ("untouched: still in progress", "started", "In Progress"), + ("cancelled, not done", "cancelled", "Cancelled"), + ("different completed-group state", "completed", "Closed"), + ] + for label, group, name in cases: + ok, note = await verify_w2(_W2Plane(group, name), {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_w4_requires_the_label_renamed_to_the_exact_target(): + async def _go(): + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + for label, name in [ + ("untouched: still triage", "triage"), + ("renamed to something else", "needs-review"), + ("space is not the requested hyphen", "needs triage"), + ]: + ok, note = await verify_w4(_W4Plane(name), dict(ctx), _run()) + assert ok is False, f"{label}: {note}" + + fallback_ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = await verify_w4(_W4Plane("needs triage"), fallback_ctx, _run()) + assert ok is False, f"name-scan fallback accepted a space-separated label: {note}" + + return asyncio.run(_go()) + + +def test_w8_requires_a_log_of_exactly_the_asked_duration(): + async def _go(): + for label, durations in [("untouched: no log", []), ("wrong duration", [60])]: + ok, note = await verify_w8(_W8Plane(durations), {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is False, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_c2_grades_the_release_contract_not_correct_prose(): + """Prose naming the right release and entries still fails; the format is the task.""" + + async def _go(): + cases = [ + ("untouched: empty answer", "", False), + ("wrong release name", "Release 9.9.9 shipped nothing useful.", False), + ("correct facts as prose", "Release 1.2.0 shipped OAuth login hardening and webhook retry backoff.", False), + ( + "exact contract", + "release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff", + True, + ), + ] + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": CHANGELOG, + } + plane = _C2Plane(f"

{CHANGELOG}

") + for label, text, want in cases: + ok, note = await verify_c2(plane, ctx, _run(text)) + assert ok is want, f"{label}: {note}" + + return asyncio.run(_go()) + + +def test_c2_live_changelog_behaviours(): + mutated = "Changelog entry one: Live API fact. Changelog entry two: Different live item." + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": CHANGELOG, + } + baseline_answer = "release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff" + + async def _go(): + identical_ok, identical_note = await verify_c2(_C2Plane(f"

{CHANGELOG}

"), ctx, _run(baseline_answer)) + assert identical_ok is True, identical_note + + mutated_ok, mutated_note = await verify_c2(_C2Plane(f"

{mutated}

"), ctx, _run(baseline_answer)) + assert mutated_ok is False + assert "changelog was mutated after seeding" in mutated_note + + live_answer = "release: 1.2.0\nshipped: Live API fact\nshipped: Different live item" + exploit_ok, exploit_note = await verify_c2(_C2Plane(f"

{mutated}

"), ctx, _run(live_answer)) + assert exploit_ok is False, exploit_note + + renamed_ok, renamed_note = await verify_c2( + _C2Plane(f"

{CHANGELOG}

", release_name="9.9.9-agent"), + ctx, + _run(baseline_answer), + ) + assert renamed_ok is False + assert "release name was mutated after seeding" in renamed_note + + empty_live_ok, empty_live_note = await verify_c2(_C2Plane("

"), ctx, _run(baseline_answer)) + assert empty_live_ok is False + assert "mutated after seeding" in empty_live_note + assert "live changelog is empty" in empty_live_note + + empty_seed_ctx = {**ctx, "release_changelog_text": ""} + empty_seed_ok, empty_seed_note = await verify_c2(_C2Plane("

"), empty_seed_ctx, _run()) + assert empty_seed_ok is False + assert "fixture missing" in empty_seed_note + assert "seeded changelog baseline is empty" in empty_seed_note + + with pytest.raises(RuntimeError, match="C2 verifier read failed while reading release"): + await verify_c2(_C2Plane(error=RuntimeError("503 unavailable")), ctx, _run()) + + return asyncio.run(_go()) + + +def test_c2_repository_constant_answer_without_reading_fails(): + randomized = ( + "Changelog entry one: OAuth login hardening ticket EVAL-a91c7e20. " + "Changelog entry two: webhook retry backoff window 7-a91c7e20." + ) + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-random", "name": "1.8.14-eval.a91c7e20"}, + "release_changelog_text": randomized, + } + repository_constant_answer = "release: 1.2.0\nshipped: OAuth login hardening\nshipped: webhook retry backoff" + + ok, note = asyncio.run( + verify_c2( + _C2Plane(f"

{randomized}

", release_name="1.8.14-eval.a91c7e20"), + ctx, + _run(repository_constant_answer, calls=[]), + ) + ) + + assert ok is False + assert "answer_correct=false" in note + assert "provenance=missing" in note diff --git a/tests/evals/tasks/test_pagination.py b/tests/evals/tasks/test_pagination.py new file mode 100644 index 0000000..30109a3 --- /dev/null +++ b/tests/evals/tasks/test_pagination.py @@ -0,0 +1,142 @@ +"""Regression tests for verifier reads whose target can land after page one.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +from plane.errors.errors import HttpError + +from evals.seed import CUSTOMER_NAME, CUSTOMER_REQUEST_NAME, R1_TITLE +from evals.tasks.cross import verify_c1 +from evals.tasks.debias import L3_TAG_VERSION, L4_PROP_DISPLAY, L4_PROP_VALUE, verify_l3, verify_l4 +from evals.tasks.write import W10_PAGE_BODY, W10_PAGE_NAME, verify_w5, verify_w10 + + +class _Page: + def __init__(self, results: list[Any], *, more: bool, cursor: str = ""): + self.results = results + self.next_page_results = more + self.next_cursor = cursor + + +def _cursor(params: Any) -> str | None: + if isinstance(params, dict): + return params.get("cursor") + return getattr(params, "cursor", None) + + +class _TwoPageList: + def __init__(self, target: Any): + self.target = target + self.cursors: list[str | None] = [] + + def list(self, *, params=None, **kwargs): + cursor = _cursor(params) + self.cursors.append(cursor) + if cursor is None: + return _Page([], more=True, cursor="cursor-2") + assert cursor == "cursor-2" + return _Page([self.target], more=False) + + +def test_c1_paginates_customers_requests_and_customer_work_items(): + customer = SimpleNamespace(id="customer-1", name=CUSTOMER_NAME) + request = SimpleNamespace(id="request-1", name=CUSTOMER_REQUEST_NAME) + linked = SimpleNamespace(id="wi-r1") + customers = _TwoPageList(customer) + requests = _TwoPageList(request) + customer_work_items = _TwoPageList(linked) + project_work_items = SimpleNamespace( + list=lambda **kwargs: _Page( + [SimpleNamespace(id="wi-r1", name=R1_TITLE, created_at="2026-01-01")], + more=False, + ) + ) + plane = SimpleNamespace( + work_items=project_work_items, + customers=SimpleNamespace( + list=customers.list, + requests=SimpleNamespace(list=requests.list), + work_items=SimpleNamespace(list=customer_work_items.list), + ), + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "workspace_objects": []} + + ok, note = asyncio.run(verify_c1(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert customers.cursors == [None, "cursor-2"] + assert requests.cursors == [None, "cursor-2"] + assert customer_work_items.cursors == [None, "cursor-2"] + + +def test_l3_paginates_release_tags(): + tags = _TwoPageList(SimpleNamespace(id="tag-1", version=L3_TAG_VERSION)) + plane = SimpleNamespace(releases=SimpleNamespace(tags=SimpleNamespace(list=tags.list))) + ctx = {"workspace_slug": "ws", "workspace_objects": []} + + ok, note = asyncio.run(verify_l3(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert tags.cursors == [None, "cursor-2"] + + +def test_l4_paginates_customer_properties(): + prop = SimpleNamespace(id="prop-1", display_name=L4_PROP_DISPLAY, property_type="TEXT") + props = _TwoPageList(prop) + plane = SimpleNamespace( + customers=SimpleNamespace( + properties=SimpleNamespace(list=props.list), + property_values=SimpleNamespace(list=lambda **kwargs: {"prop-1": [L4_PROP_VALUE]}), + ) + ) + ctx = { + "workspace_slug": "ws", + "customer": {"id": "customer-1"}, + "workspace_objects": [], + } + + ok, note = asyncio.run(verify_l4(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert props.cursors == [None, "cursor-2"] + + +def test_w5_paginates_archived_work_items(): + archived = _TwoPageList(SimpleNamespace(id="wi-archived")) + + def missing(**kwargs): + raise HttpError("not found", status_code=404, response={}) + + plane = SimpleNamespace( + work_items=SimpleNamespace( + retrieve=missing, + list_archived=archived.list, + ) + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "module_completed_ids": ["wi-archived"], + } + + ok, note = asyncio.run(verify_w5(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert archived.cursors == [None, "cursor-2"] + + +def test_w10_paginates_pages_then_retrieves_the_page_two_body(): + pages = _TwoPageList(SimpleNamespace(id="page-1", name=W10_PAGE_NAME)) + plane = SimpleNamespace( + pages=SimpleNamespace( + list_project_pages=pages.list, + retrieve_project_page=lambda **kwargs: SimpleNamespace( + id="page-1", + description_html=f"

{W10_PAGE_BODY}

", + ), + ) + ) + ctx = {"workspace_slug": "ws", "project_id": "p1"} + + ok, note = asyncio.run(verify_w10(plane, ctx, {"final_text": "", "calls": []})) + assert ok is True, note + assert pages.cursors == [None, "cursor-2"] diff --git a/tests/evals/tasks/test_verifier_read_errors.py b/tests/evals/tasks/test_verifier_read_errors.py new file mode 100644 index 0000000..d0abebe --- /dev/null +++ b/tests/evals/tasks/test_verifier_read_errors.py @@ -0,0 +1,510 @@ +"""Verifier API-read failures are infrastructure; explicit fallbacks stay tolerant.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from datetime import date +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.core.changelog import normalize_changelog_text +from evals.core.fixtures import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + CYCLE_CURRENT, + INTAKE_BILLING_TITLE, + INTAKE_SPAM_TITLE, + R1_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, +) +from evals.tasks.cross import verify_c1, verify_c2 +from evals.tasks.debias import L1_TITLE, L4_PROP_DISPLAY, verify_l1, verify_l3, verify_l4 +from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.verification import VerifierReadError +from evals.tasks.write import ( + W10_PAGE_NAME, + W11_TITLE, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w10, + verify_w11, +) + + +class _Page: + def __init__(self, results: list[Any] | None = None): + self.results = results or [] + self.next_page_results = False + self.next_cursor = None + + +def _http(status: int) -> HttpError: + return HttpError("read unavailable" if status != 404 else "not found", status, {}) + + +def _raise(exc: BaseException) -> Callable[..., Any]: + def fail(**kwargs: Any) -> Any: + raise exc + + return fail + + +def _run() -> dict[str, Any]: + return {"final_text": "", "calls": []} + + +def _s1(site: str) -> tuple[Any, dict[str, Any]]: + severity = SimpleNamespace(id="severity-1", display_name="Severity", property_type="OPTION", options=[]) + if site == "properties": + properties = SimpleNamespace(list=_raise(_http(500))) + else: + properties = SimpleNamespace(list=lambda **kw: [severity], options=SimpleNamespace(list=_raise(_http(500)))) + return SimpleNamespace(work_item_properties=properties), { + "workspace_slug": "ws", + "project_id": "p1", + "bug_type": {"id": "bug-1"}, + } + + +def _s3(site: str) -> tuple[Any, dict[str, Any]]: + incident = SimpleNamespace(id="incident-1", name="Incident") + project_types = [] if site in {"ownership", "workspace-types"} else [incident] + workspace_features = ( + _raise(ConnectionError("features down")) + if site == "ownership" + else lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": True}) + ) + workspace_types = _raise(ConnectionError("types down")) if site == "workspace-types" else lambda **kw: [] + property_list = _raise(_http(500)) if site == "properties" else lambda **kw: [] + return SimpleNamespace( + work_item_types=SimpleNamespace(list=lambda **kw: project_types), + workspaces=SimpleNamespace(get_features=workspace_features), + workspace_work_item_types=SimpleNamespace(list=workspace_types), + work_item_properties=SimpleNamespace(list=property_list), + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _s5(site: str) -> tuple[Any, dict[str, Any]]: + project_features = ( + _raise(ConnectionError("project features down")) + if site == "project" + else lambda **kw: SimpleNamespace(model_dump=lambda: {"cycles": True}) + ) + workspace_features = _raise(ConnectionError("workspace features down")) + return SimpleNamespace( + projects=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(cycle_view=True, is_time_tracking_enabled=True), + get_features=project_features, + ), + workspaces=SimpleNamespace(get_features=workspace_features), + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _l4(site: str) -> tuple[Any, dict[str, Any]]: + prop = SimpleNamespace(id="property-1", display_name=L4_PROP_DISPLAY, property_type="TEXT") + properties = _raise(ConnectionError("properties down")) if site == "properties" else lambda **kw: _Page([prop]) + values = _raise(ConnectionError("values down")) + return SimpleNamespace( + customers=SimpleNamespace( + properties=SimpleNamespace(list=properties), + property_values=SimpleNamespace(list=values), + ) + ), {"workspace_slug": "ws", "customer": {"id": "customer-1"}} + + +def _c1() -> tuple[Any, dict[str, Any]]: + r1 = SimpleNamespace(id="item-r1", name=R1_TITLE, created_at="2026-01-01") + customer = SimpleNamespace(id="customer-1", name=CUSTOMER_NAME) + request = SimpleNamespace(id="request-1", name=CUSTOMER_REQUEST_NAME) + return SimpleNamespace( + work_items=SimpleNamespace(list=lambda **kw: _Page([r1])), + customers=SimpleNamespace( + list=lambda **kw: _Page([customer]), + requests=SimpleNamespace(list=lambda **kw: _Page([request])), + work_items=SimpleNamespace(list=_raise(ConnectionError("links down"))), + ), + ), {"workspace_slug": "ws", "project_id": "p1", "workspace_objects": []} + + +def _w5(site: str) -> tuple[Any, dict[str, Any]]: + retrieve = _raise(_http(500) if site == "retrieve" else _http(404)) + return SimpleNamespace( + work_items=SimpleNamespace( + retrieve=retrieve, + list_archived=_raise(ConnectionError("archive list down")), + ) + ), {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ["item-1"]} + + +def _w7(site: str) -> tuple[Any, dict[str, Any]]: + rows = [ + SimpleNamespace(id="source-1", name=W7_SOURCE_TITLE, created_at="2026-01-02"), + SimpleNamespace(id="target-1", name=W7_TARGET_TITLE, created_at="2026-01-01"), + ] + dependencies = ( + _raise(ConnectionError("dependencies down")) + if site == "dependencies" + else lambda **kw: {"blocking": [{"id": "target-1"}]} + ) + links = _raise(ConnectionError("links down")) + return SimpleNamespace( + work_items=SimpleNamespace( + list=lambda **kw: _Page(rows), + dependencies=SimpleNamespace(list=dependencies), + links=SimpleNamespace(list=links), + ) + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _w10(site: str) -> tuple[Any, dict[str, Any]]: + page = SimpleNamespace(id="page-1", name=W10_PAGE_NAME) + listing = _raise(ConnectionError("pages down")) if site == "list" else lambda **kw: _Page([page]) + return SimpleNamespace( + pages=SimpleNamespace( + list_project_pages=listing, + retrieve_project_page=_raise(TimeoutError("page read timed out")), + ) + ), {"workspace_slug": "ws", "project_id": "p1"} + + +def _infra_cases() -> list[Any]: + cases: list[Any] = [] + + def add(case_id: str, task: str, reading: str, verifier: Any, plane: Any, ctx: dict[str, Any]) -> None: + cases.append(pytest.param(task, reading, lambda: verifier(plane, ctx, _run()), id=case_id)) + + plane, ctx = _s1("properties") + add("S1-type-properties", "S1", "listing Bug type properties", verify_s1, plane, ctx) + plane, ctx = _s1("options") + add("S1-options", "S1", "listing Severity options", verify_s1, plane, ctx) + add( + "S2-estimate", + "S2", + "retrieving the project estimate", + verify_s2, + SimpleNamespace(estimates=SimpleNamespace(retrieve=_raise(_http(500)))), + {"workspace_slug": "ws", "project_id": "p1"}, + ) + for site, reading in ( + ("ownership", "reading workspace work-item-type ownership"), + ("workspace-types", "listing workspace work-item types"), + ("properties", "listing Incident type properties"), + ): + plane, ctx = _s3(site) + add(f"S3-{site}", "S3", reading, verify_s3, plane, ctx) + add( + "S4-intake-list-fallback", + "S4", + "listing intake while resolving", + verify_s4, + SimpleNamespace( + intake=SimpleNamespace( + retrieve=_raise(ConnectionError("retrieve down")), + list=_raise(ConnectionError("list down")), + ) + ), + { + "workspace_slug": "ws", + "project_id": "p1", + "intake": {"billing": {"issue_id": "billing-1"}, "spam": {"issue_id": "spam-1"}}, + }, + ) + for site, reading in (("project", "reading project feature flags"), ("workspace", "reading workspace customer")): + plane, ctx = _s5(site) + add(f"S5-{site}-features", "S5", reading, verify_s5, plane, ctx) + add( + "L1-worklog-summary", + "L1", + "reading the project worklog summary", + verify_l1, + SimpleNamespace( + work_items=SimpleNamespace(work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=90)])), + projects=SimpleNamespace(get_worklog_summary=_raise(ConnectionError("summary down"))), + ), + { + "workspace_slug": "ws", + "project_id": "p1", + "items": {L1_TITLE: "item-1"}, + "l1_expected_summary_ids": ["item-1", "item-seeded"], + }, + ) + add( + "L3-release-tags", + "L3", + "listing workspace release tags", + verify_l3, + SimpleNamespace(releases=SimpleNamespace(tags=SimpleNamespace(list=_raise(ConnectionError("tags down"))))), + {"workspace_slug": "ws"}, + ) + for site, reading in ( + ("properties", "listing workspace customer properties"), + ("values", "reading property values"), + ): + plane, ctx = _l4(site) + add(f"L4-{site}", "L4", reading, verify_l4, plane, ctx) + plane, ctx = _c1() + add("C1-customer-work-items", "C1", "listing work items linked to customer", verify_c1, plane, ctx) + baseline = "Changelog entry one: One. Changelog entry two: Two." + add( + "C2-release-changelog", + "C2", + "reading release release-1 and its changelog", + verify_c2, + SimpleNamespace( + releases=SimpleNamespace( + retrieve=_raise(_http(500)), + changelog=SimpleNamespace(retrieve=lambda **kw: normalize_changelog_text(baseline)), + ) + ), + { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": baseline, + }, + ) + add( + "W4-triage-label", + "W4", + "retrieving seeded triage label", + verify_w4, + SimpleNamespace(labels=SimpleNamespace(retrieve=_raise(_http(500)))), + {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "label-1"}}, + ) + for site, reading in (("retrieve", "retrieving module work item"), ("archived", "listing archived items")): + plane, ctx = _w5(site) + add(f"W5-{site}", "W5", reading, verify_w5, plane, ctx) + add( + "W6-cycle-items", + "W6", + f"listing {CYCLE_CURRENT} work items", + verify_w6, + SimpleNamespace( + cycles=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace( + end_date=date.today().isoformat(), archived_at=None, progress_snapshot=None + ), + list_work_items=_raise(TimeoutError("cycle items read timed out")), + ) + ), + { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "past-1", + "cycle_current_id": "current-1", + "w6_unfinished_titles": ["unfinished"], + }, + ) + for site, reading in (("dependencies", "listing dependencies"), ("links", "listing links")): + plane, ctx = _w7(site) + add(f"W7-{site}", "W7", reading, verify_w7, plane, ctx) + item = SimpleNamespace(id="item-1", name=W11_TITLE, created_at="2026-01-01") + add( + "W11-worklogs", + "W11", + "listing work logs for item", + verify_w11, + SimpleNamespace( + work_items=SimpleNamespace( + list=lambda **kw: _Page([item]), + work_logs=SimpleNamespace(list=_raise(_http(500))), + ) + ), + {"workspace_slug": "ws", "project_id": "p1"}, + ) + for site, reading in (("list", "listing project pages"), ("retrieve", "retrieving project page")): + plane, ctx = _w10(site) + add(f"W10-{site}", "W10", reading, verify_w10, plane, ctx) + return cases + + +@pytest.mark.parametrize(("task_id", "reading", "invoke"), _infra_cases()) +def test_required_verifier_read_failures_are_infrastructure(task_id: str, reading: str, invoke: Callable[[], Any]): + with pytest.raises(VerifierReadError) as caught: + asyncio.run(invoke()) + + message = str(caught.value) + assert message.startswith(f"{task_id} verifier read failed while ") + assert reading in message + assert caught.value.__cause__ is not None + + +async def _negative_s2_estimate_404() -> tuple[bool, str]: + plane = SimpleNamespace(estimates=SimpleNamespace(retrieve=_raise(_http(404)))) + return await verify_s2(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +async def _negative_c2_oracle_404() -> tuple[bool, str]: + baseline = "Changelog entry one: One. Changelog entry two: Two." + plane = SimpleNamespace( + releases=SimpleNamespace( + retrieve=_raise(_http(404)), + changelog=SimpleNamespace(retrieve=lambda **kw: normalize_changelog_text(baseline)), + ) + ) + ctx = { + "workspace_slug": "ws", + "release": {"id": "release-1", "name": "1.2.0"}, + "release_changelog_text": baseline, + } + return await verify_c2(plane, ctx, _run()) + + +async def _negative_w6_current_cycle_404() -> tuple[bool, str]: + plane = SimpleNamespace( + cycles=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace( + end_date=date.today().isoformat(), archived_at=None, progress_snapshot=None + ), + list_work_items=_raise(_http(404)), + ) + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "past-1", + "cycle_current_id": "current-1", + "w6_unfinished_titles": ["unfinished"], + } + return await verify_w6(plane, ctx, _run()) + + +async def _negative_w10_page_retrieve_404() -> tuple[bool, str]: + page = SimpleNamespace(id="page-1", name=W10_PAGE_NAME) + plane = SimpleNamespace( + pages=SimpleNamespace( + list_project_pages=lambda **kw: _Page([page]), + retrieve_project_page=_raise(_http(404)), + ) + ) + return await verify_w10(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +@pytest.mark.parametrize( + ("invoke", "note_fragment"), + [ + pytest.param(_negative_s2_estimate_404, "estimate not found", id="S2-missing-created-estimate"), + pytest.param(_negative_c2_oracle_404, "no longer exists", id="C2-missing-seed-oracle"), + pytest.param(_negative_w6_current_cycle_404, "not found", id="W6-missing-rollover-cycle"), + pytest.param(_negative_w10_page_retrieve_404, "not found after listing", id="W10-missing-created-page"), + ], +) +def test_authoritative_not_found_is_an_agent_failure(invoke: Callable[[], Any], note_fragment: str): + success, note = asyncio.run(invoke()) + assert success is False + assert note_fragment in note + + +async def _tolerant_s1_property_404() -> tuple[bool, str]: + plane = SimpleNamespace(work_item_properties=SimpleNamespace(list=_raise(_http(404)))) + return await verify_s1(plane, {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug"}}, _run()) + + +async def _tolerant_s1_options_404() -> tuple[bool, str]: + severity = SimpleNamespace(id="severity", display_name="Severity", property_type="OPTION", options=[]) + plane = SimpleNamespace( + work_item_properties=SimpleNamespace( + list=lambda **kw: [severity], + options=SimpleNamespace(list=_raise(_http(404))), + ) + ) + return await verify_s1(plane, {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug"}}, _run()) + + +async def _tolerant_s3_property_404() -> tuple[bool, str]: + incident = SimpleNamespace(id="incident", name="Incident") + plane = SimpleNamespace( + work_item_types=SimpleNamespace(list=lambda **kw: [incident]), + work_item_properties=SimpleNamespace(list=_raise(_http(404))), + ) + return await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +async def _tolerant_s4_retrieve_fallback() -> tuple[bool, str]: + rows = [ + SimpleNamespace(issue_detail=SimpleNamespace(name=INTAKE_BILLING_TITLE), status=1), + SimpleNamespace(issue_detail=SimpleNamespace(name=INTAKE_SPAM_TITLE), status=-1), + ] + plane = SimpleNamespace( + intake=SimpleNamespace(retrieve=_raise(ConnectionError("retrieve down")), list=lambda **kw: rows) + ) + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "intake": {"billing": {"issue_id": "billing"}, "spam": {"issue_id": "spam"}}, + } + return await verify_s4(plane, ctx, _run()) + + +async def _tolerant_w4_not_found() -> tuple[bool, str]: + plane = SimpleNamespace(labels=SimpleNamespace(retrieve=_raise(_http(404)))) + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "label-1"}} + return await verify_w4(plane, ctx, _run()) + + +async def _tolerant_w5_retrieve_fallback() -> tuple[bool, str]: + archived = SimpleNamespace(id="item-1") + plane = SimpleNamespace( + work_items=SimpleNamespace( + retrieve=_raise(_http(404)), + list_archived=lambda **kw: _Page([archived]), + ) + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ["item-1"]} + return await verify_w5(plane, ctx, _run()) + + +async def _tolerant_w5_optional_archive_crosscheck() -> tuple[bool, str]: + plane = SimpleNamespace( + work_items=SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace(id="item-1", archived_at=None), + list_archived=_raise(ConnectionError("optional list down")), + ) + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": ["item-1"]} + return await verify_w5(plane, ctx, _run()) + + +async def _tolerant_w11_diagnostic_read() -> tuple[bool, str]: + item = SimpleNamespace(id="item-1", name=W11_TITLE, created_at="2026-01-01") + plane = SimpleNamespace( + work_items=SimpleNamespace(list=lambda **kw: _Page([item]), work_logs=SimpleNamespace(list=lambda **kw: [])), + projects=SimpleNamespace(retrieve=_raise(ConnectionError("diagnostic read down"))), + ) + return await verify_w11(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +async def _tolerant_w11_gate_404() -> tuple[bool, str]: + item = SimpleNamespace(id="item-1", name=W11_TITLE, created_at="2026-01-01") + plane = SimpleNamespace( + work_items=SimpleNamespace( + list=lambda **kw: _Page([item]), + work_logs=SimpleNamespace(list=_raise(_http(404))), + ) + ) + return await verify_w11(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + + +@pytest.mark.parametrize( + ("invoke", "want_success"), + [ + pytest.param(_tolerant_s1_property_404, False, id="S1-property-404-is-absence"), + pytest.param(_tolerant_s1_options_404, False, id="S1-options-404-is-absence"), + pytest.param(_tolerant_s3_property_404, False, id="S3-property-404-is-absence"), + pytest.param(_tolerant_s4_retrieve_fallback, True, id="S4-retrieve-has-list-fallback"), + pytest.param(_tolerant_w4_not_found, False, id="W4-seed-id-404-is-deletion"), + pytest.param(_tolerant_w5_retrieve_fallback, True, id="W5-retrieve-404-has-archive-fallback"), + pytest.param(_tolerant_w5_optional_archive_crosscheck, False, id="W5-archive-list-crosscheck-is-optional"), + pytest.param(_tolerant_w11_diagnostic_read, False, id="W11-feature-read-is-diagnostic-only"), + pytest.param(_tolerant_w11_gate_404, False, id="W11-worklog-404-is-disabled-gate"), + ], +) +def test_deliberately_tolerant_verifier_reads_are_pinned(invoke: Callable[[], Any], want_success: bool): + success, note = asyncio.run(invoke()) + assert success is want_success, note diff --git a/tests/evals/tasks/test_verifiers.py b/tests/evals/tasks/test_verifiers.py new file mode 100644 index 0000000..9684380 --- /dev/null +++ b/tests/evals/tasks/test_verifiers.py @@ -0,0 +1,1063 @@ +"""Offline eval tests for R, W, S, and C verifiers.""" + +from __future__ import annotations + +import asyncio +from datetime import date, timedelta +from types import SimpleNamespace +from typing import Any + +import pytest +from plane.errors.errors import HttpError + +from evals.core.errors import TaskSkipped +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.fixtures import INTAKE_BILLING_TITLE, INTAKE_SPAM_TITLE +from evals.seed import ( + CUSTOMER_NAME, + CUSTOMER_REQUEST_NAME, + CYCLE_PAST, + R1_TITLE, + W3_TITLE, + W7_SOURCE_TITLE, + W7_TARGET_TITLE, + W7_URL, + W8_TITLE, +) +from evals.tasks.cross import verify_c1 +from evals.tasks.read import verify_r3 +from evals.tasks.schema import verify_s1, verify_s2, verify_s3, verify_s4, verify_s5 +from evals.tasks.verification import VerifierReadError +from evals.tasks.write import ( + W10_PAGE_BODY, + W10_PAGE_NAME, + verify_w1, + verify_w3, + verify_w4, + verify_w5, + verify_w6, + verify_w7, + verify_w8, + verify_w9, + verify_w10, +) + + +class _Page: + def __init__(self, results: list[Any] | None = None, next_page_results: bool = False): + self.results = results or [] + self.next_page_results = next_page_results + self.next_cursor = None + + +def _http404() -> HttpError: + return HttpError("not found", status_code=404, response={}) + + +def _item(id: str, name: str, **kw: Any) -> SimpleNamespace: + return SimpleNamespace(id=id, name=name, **kw) + + +def _run() -> dict[str, Any]: + return {"final_text": "", "calls": []} + + +class _DepsDump: + def __init__(self, data: dict): + self._data = data + + def model_dump(self) -> dict: + return self._data + + +class _W7Plane: + """Fake where dependencies.list returns dump with tgt only in blocked_by.""" + + def __init__(self, deps_dump: dict, urls: list[str] | None = None): + self._deps_dump = deps_dump + self._urls = urls if urls is not None else [W7_URL] + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("src-1", W7_SOURCE_TITLE), _item("tgt-1", W7_TARGET_TITLE)]), + dependencies=SimpleNamespace(list=self._deps_list), + links=SimpleNamespace(list=self._links_list), + ) + + def _deps_list(self, **kw): + return _DepsDump(self._deps_dump) + + def _links_list(self, **kw): + return _Page([SimpleNamespace(url=u) for u in self._urls]) + + +@pytest.mark.parametrize( + ("dependencies", "want", "expect_wrong_direction"), + [ + pytest.param( + {"blocking": [], "blocked_by": [{"id": "tgt-1"}]}, + False, + True, + id="blocked-by-is-wrong-direction", + ), + pytest.param( + {"blocking": [{"id": "tgt-1"}], "blocked_by": []}, + True, + False, + id="blocking-passes", + ), + ], +) +def test_f1_w7_behaviours(dependencies, want, expect_wrong_direction): + ok, note = asyncio.run( + verify_w7( + _W7Plane(dependencies), + {"workspace_slug": "ws", "project_id": "p1"}, + _run(), + ) + ) + assert ok is want, note + if expect_wrong_direction: + assert "blocking" in note.lower() or "no blocking" in note.lower() + assert "wrong direction" in note or "tgt-1" in note + + +class _W6Plane: + def __init__( + self, + *, + past_end: str, + archived_at=None, + snapshot=None, + sprint13_names: list[str] | None = None, + list_error: Exception | None = None, + ): + self._past_end = past_end + self._archived_at = archived_at + self._snapshot = snapshot + self._s13 = sprint13_names or [] + self._list_error = list_error + self.cycles = SimpleNamespace(retrieve=self._retrieve, list_work_items=self._list_wi) + + def _retrieve(self, **kw): + return SimpleNamespace( + id="c12", + end_date=self._past_end, + archived_at=self._archived_at, + progress_snapshot=self._snapshot, + ) + + def _list_wi(self, **kw): + if self._list_error is not None: + raise self._list_error + return _Page([_item(f"i{i}", n) for i, n in enumerate(self._s13)]) + + +@pytest.mark.parametrize( + ("end_offset", "timestamp", "seed_offset", "names", "want", "expect"), + [ + pytest.param( + -14, False, -14, ["Inventory count goes negative under load"], False, "not closed", id="seeded-past-end" + ), + pytest.param(1, True, 1, ["Inventory count goes negative under load"], False, "not closed", id="noop-tomorrow"), + pytest.param( + 0, True, 1, ["Inventory count goes negative under load"], True, "end_date", id="closed-today-timestamp" + ), + pytest.param( + 0, + False, + -14, + ["Inventory count goes negative under load", "Tooltip clipped inside modal dialog"], + True, + "", + id="closed-today-date", + ), + ], +) +def test_f2_w6_closes_only_on_a_real_end_date_signal(end_offset, timestamp, seed_offset, names, want, expect): + """A timestamp and a bare date must provide the same real closure signal.""" + end_date = (date.today() + timedelta(days=end_offset)).isoformat() + if timestamp: + end_date = f"{end_date}T00:00:00Z" + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": (date.today() + timedelta(days=seed_offset)).isoformat(), + "w6_unfinished_titles": names, + "cycles": {CYCLE_PAST: "c12"}, + } + ok, note = asyncio.run(verify_w6(_W6Plane(past_end=end_date, sprint13_names=names), ctx, _run())) + assert ok is want, note + if expect: + assert expect in note.lower() or expect in note, note + + +@pytest.mark.parametrize( + ("sprint13_names", "list_error", "ctx_override", "want", "expect_note", "raises"), + [ + pytest.param(None, None, {}, True, "", None, id="healthy"), + pytest.param( + ["Inventory count goes negative under load"], + None, + {}, + False, + "unfinished not on Sprint 13", + None, + id="missing-rollover-item", + ), + pytest.param(None, RuntimeError("503 unavailable"), {}, None, "", "W6.*Sprint 13", id="listing-error"), + pytest.param( + None, + None, + {"cycle_current_id": None}, + None, + "", + "W6 fixture error.*Sprint 13 id missing", + id="missing-current-cycle", + ), + pytest.param( + None, + None, + {"cycle_past_id": None}, + None, + "", + "W6 fixture error.*Sprint 12 id missing", + id="missing-past-cycle", + ), + pytest.param( + None, + None, + {"w6_unfinished_titles": []}, + None, + "", + "W6 fixture error.*unfinished items.*empty", + id="empty-unfinished-fixture", + ), + ], +) +def test_w6_rollover_verification_is_fail_closed(sprint13_names, list_error, ctx_override, want, expect_note, raises): + expected = ["Inventory count goes negative under load", "Tooltip clipped inside modal dialog"] + ctx = { + "workspace_slug": "ws", + "project_id": "p1", + "cycle_past_id": "c12", + "cycle_current_id": "c13", + "cycle_past_seed_end_date": (date.today() + timedelta(days=1)).isoformat(), + "w6_unfinished_titles": expected, + **ctx_override, + } + plane = _W6Plane( + past_end=date.today().isoformat(), + sprint13_names=expected if sprint13_names is None else sprint13_names, + list_error=list_error, + ) + if raises: + with pytest.raises(RuntimeError, match=raises): + asyncio.run(verify_w6(plane, ctx, _run())) + return + + ok, note = asyncio.run(verify_w6(plane, ctx, _run())) + assert ok is want, note + if expect_note: + assert expect_note in note + + +class _W5Plane: + def __init__(self, *, retrieve_map: dict[str, Any], archived_ids: list[str]): + self._retrieve_map = retrieve_map + self._archived_ids = archived_ids + self.work_items = SimpleNamespace( + retrieve=self._retrieve, + list_archived=self._list_archived, + list=lambda **kw: _Page([]), + ) + + def _retrieve(self, **kw): + wid = str(kw["work_item_id"]) + if wid not in self._retrieve_map: + raise _http404() + val = self._retrieve_map[wid] + if val is None: + raise _http404() + return val + + def _list_archived(self, **kw): + return _Page([_item(i, f"n-{i}") for i in self._archived_ids]) + + +@pytest.mark.parametrize( + ("plane", "module_ids", "want", "expect"), + [ + pytest.param( + _W5Plane(retrieve_map={}, archived_ids=[]), + ["m1", "m2", "m3"], + False, + "not archived", + id="deleted-not-archived", + ), + pytest.param( + _W5Plane(retrieve_map={}, archived_ids=["m1", "m2", "m3"]), ["m1", "m2", "m3"], True, "", id="archived-list" + ), + pytest.param( + _W5Plane( + retrieve_map={"m1": SimpleNamespace(id="m1", archived_at="2026-01-01T00:00:00Z")}, + archived_ids=[], + ), + ["m1"], + True, + "", + id="archived-at", + ), + ], +) +def test_f3_w5_accepts_archive_but_not_deletion(plane, module_ids, want, expect): + """A 404 alone is not evidence of archiving — deleting every item would also 404.""" + ctx = {"workspace_slug": "ws", "project_id": "p1", "module_completed_ids": module_ids} + ok, note = asyncio.run(verify_w5(plane, ctx, _run())) + assert ok is want, note + if expect: + assert expect in note, note + + +class _C1Plane: + def __init__( + self, + *, + customers: list[Any], + requests: list[Any], + linked: list[Any], + project_items: list[Any], + ): + self.customers = SimpleNamespace( + list=lambda **kw: _Page(customers), + requests=SimpleNamespace(list=lambda **kw: _Page(requests)), + work_items=SimpleNamespace(list=lambda **kw: _Page(linked)), + ) + self.work_items = SimpleNamespace(list=lambda **kw: _Page(project_items)) + + +@pytest.mark.parametrize( + ("customers", "linked", "project_items", "want", "expect_any"), + [ + pytest.param( + [SimpleNamespace(id="c1", name="Acme Industries")], + [SimpleNamespace(id="wi-r1")], + [_item("wi-r1", R1_TITLE)], + False, + (CUSTOMER_NAME,), + id="lookalike-customer", + ), + pytest.param( + [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + [SimpleNamespace(id="wi-other")], + [_item("wi-r1", R1_TITLE), _item("wi-other", "Other")], + False, + ("not linked", "wi-r1"), + id="wrong-linked-item", + ), + pytest.param( + [SimpleNamespace(id="c1", name=CUSTOMER_NAME)], + [SimpleNamespace(id="wi-r1")], + [_item("wi-r1", R1_TITLE)], + True, + (), + id="exact-customer-and-item", + ), + ], +) +def test_f4_c1_requires_the_named_customer_linked_to_the_named_item(customers, linked, project_items, want, expect_any): + """Both ends are checked: a lookalike customer name and a link to any other item fail.""" + plane = _C1Plane( + customers=customers, + requests=[SimpleNamespace(id="r1", name=CUSTOMER_REQUEST_NAME)], + linked=linked, + project_items=project_items, + ) + ctx = {"workspace_slug": "ws", "project_id": "p1", "items": {R1_TITLE: "wi-r1"}} + ok, note = asyncio.run(verify_c1(plane, ctx, _run())) + assert ok is want, note + if expect_any: + assert any(s in note for s in expect_any), note + + +class _W3Plane: + def __init__(self, comments: list[Any]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w3", W3_TITLE)]), + comments=SimpleNamespace(list=lambda **kw: _Page(comments)), + ) + + +@pytest.mark.parametrize( + ("comment_html", "stripped", "want", "expect"), + [ + pytest.param("

lgtm

", "lgtm", False, "contrast tokens", id="unrelated-comment"), + pytest.param( + "

Reviewed contrast tokens — needs design pass

", + "Reviewed contrast tokens — needs design pass", + True, + "", + id="exact-stripped-text", + ), + pytest.param( + "

Reviewed contrast tokens — needs design pass

", + None, + True, + "", + id="normalized-html", + ), + pytest.param( + "

Reviewed contrast tokens — needs design pass and accessibility review

", + "Reviewed contrast tokens — needs design pass and accessibility review", + False, + "exact normalized comment", + id="substring-only", + ), + ], +) +def test_f5_w3_requires_exact_normalized_comment_text(comment_html, stripped, want, expect): + plane = _W3Plane([SimpleNamespace(comment_html=comment_html, comment_stripped=stripped)]) + ok, note = asyncio.run(verify_w3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + assert ok is want, note + if expect: + assert expect in note, note + + +class _Pages: + def __init__(self, body: str): + self.body = body + + def list_project_pages(self, **kwargs): + return _Page([SimpleNamespace(id="page-1", name=W10_PAGE_NAME)]) + + def retrieve_project_page(self, **kwargs): + return SimpleNamespace(id="page-1", name=W10_PAGE_NAME, description_html=self.body) + + +@pytest.mark.parametrize( + ("body", "want", "expect"), + [ + pytest.param("

Unrelated runbook body

", False, "body mismatch", id="wrong-body"), + pytest.param(f"

{W10_PAGE_BODY}

", True, "", id="normalized-exact-body"), + ], +) +def test_w10_requires_the_exact_normalized_page_body(body, want, expect): + ok, note = asyncio.run( + verify_w10( + SimpleNamespace(pages=_Pages(body)), + {"workspace_slug": "ws", "project_id": "p1"}, + _run(), + ) + ) + assert ok is want, note + if expect: + assert expect in note + + +class _W4Plane: + def __init__(self, *, by_id: dict[str, Any], listed: list[Any]): + self._by_id = by_id + self.labels = SimpleNamespace(retrieve=self._retrieve, list=lambda **kw: _Page(listed)) + + def _retrieve(self, **kw): + lid = str(kw["label_id"]) + if lid not in self._by_id: + raise _http404() + return self._by_id[lid] + + +@pytest.mark.parametrize( + ("by_id", "listed", "want", "expect"), + [ + pytest.param( + {"triage-id": SimpleNamespace(id="triage-id", name="triage")}, + [SimpleNamespace(id="triage-id", name="triage"), SimpleNamespace(id="other", name="needs-triage")], + False, + "triage-id", + id="decoy-label", + ), + pytest.param( + {"triage-id": SimpleNamespace(id="triage-id", name="needs-triage")}, + [SimpleNamespace(id="triage-id", name="needs-triage")], + True, + "", + id="seeded-label-renamed", + ), + ], +) +def test_f7_w4_follows_the_seeded_label_id_not_the_name(by_id, listed, want, expect): + """A name scan would accept a different label renamed to the target.""" + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"triage": "triage-id"}} + ok, note = asyncio.run(verify_w4(_W4Plane(by_id=by_id, listed=listed), ctx, _run())) + assert ok is want, note + if expect: + assert expect in note, note + + +class _S3Plane: + def __init__(self, *, props: list[Any], types: list[Any] | None = None, workspace_owns: bool = False): + self._props = props + self._types = types if types is not None else [SimpleNamespace(id="t-inc", name="Incident")] + self._ws_owns = workspace_owns + self.work_item_types = SimpleNamespace(list=lambda **kw: self._types) + self.work_item_properties = SimpleNamespace(list=lambda **kw: self._props) + self.workspaces = SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"is_work_item_types_enabled": self._ws_owns}) + ) + self.workspace_work_item_types = SimpleNamespace(list=lambda **kw: []) + + +@pytest.mark.parametrize( + ("display_name", "property_type", "want", "expect"), + [ + pytest.param("Severity", "OPTION", False, "TEXT", id="required-option-fails"), + pytest.param("Impact summary", "TEXT", True, "", id="required-text-passes"), + ], +) +def test_f6_s3_behaviours(display_name, property_type, want, expect): + plane = _S3Plane( + props=[ + SimpleNamespace( + id="p1", + display_name=display_name, + property_type=property_type, + is_required=True, + ) + ] + ) + ok, note = asyncio.run(verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + assert ok is want, note + if expect: + assert expect in note + + +def test_f9_s3_workspace_type_via_features_probe(): + async def _go(): + """Incident only on workspace types; empty project list; needs empty (no seed flag).""" + plane = _S3Plane(props=[], types=[], workspace_owns=True) + # Override workspace type list to include Incident + plane.workspace_work_item_types = SimpleNamespace( + list=lambda **kw: [SimpleNamespace(id="ws-inc", name="Incident")] + ) + plane.work_item_properties = SimpleNamespace( + list=lambda **kw: [ + SimpleNamespace( + id="p1", + display_name="Impact summary", + property_type="TEXT", + is_required=True, + ) + ] + ) + # No bug_type_workspace_level in ctx — old code would miss Incident. + ok, note = await verify_s3(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run()) + assert ok is True, note + + return asyncio.run(_go()) + + +@pytest.mark.parametrize( + "weekday", + range(7), + ids=["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"], +) +def test_f8_r3_due_date_clamped_to_iso_week(weekday): + monday = date(2026, 8, 10) + today = monday + timedelta(days=weekday) + days_to_week_end = 6 - today.weekday() + due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) + week_end = today + timedelta(days=days_to_week_end) + week_start = today - timedelta(days=today.weekday()) + assert week_start <= due <= week_end, f"weekday={weekday} due={due}" + if weekday >= 5: + assert due <= week_end + assert due == week_end or due == today + + +@pytest.mark.parametrize( + ("today", "expected"), + [ + pytest.param(date(2026, 8, 15), date(2026, 8, 16), id="saturday-clamps-to-sunday"), + pytest.param(date(2026, 8, 16), date(2026, 8, 16), id="sunday-stays-sunday"), + ], +) +def test_f8_seed_r3_due_date_function_matches(today, expected): + days_to_week_end = 6 - today.weekday() + due = min(today + timedelta(days=2), today + timedelta(days=days_to_week_end)) + assert due == expected + + +class _W8Plane: + def __init__(self, durations: list[int]): + self.work_items = SimpleNamespace( + list=lambda **kw: _Page([_item("w8", W8_TITLE)]), + work_logs=SimpleNamespace(list=lambda **kw: [SimpleNamespace(duration=d) for d in durations]), + ) + + +@pytest.mark.parametrize( + ("duration", "want", "expect"), + [ + pytest.param(480, False, "120", id="480-minutes-fails"), + pytest.param(120, True, "", id="exactly-120-passes"), + ], +) +def test_minor_w8_behaviours(duration, want, expect): + ok, note = asyncio.run( + verify_w8( + _W8Plane([duration]), + {"workspace_slug": "ws", "project_id": "p1"}, + _run(), + ) + ) + assert ok is want, note + if expect: + assert expect in note + + +@pytest.mark.parametrize( + ("run", "want", "expect"), + [ + pytest.param( + {"final_text": "There are 2 items due this week.", "calls": []}, + False, + "item contract", + id="count-without-titles-fails", + ), + pytest.param( + { + "final_text": "item: Onboarding email template stale\nitem: Webhook secret rotation docs missing", + "calls": [ + { + "tool": "plane_call", + "is_error": False, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + "call_source": "test", + "evidence_trace_available": True, + }, + True, + "", + id="exact-items-any-order", + ), + ], +) +def test_minor_r3_behaviours(run, want, expect): + titles = ["Webhook secret rotation docs missing", "Onboarding email template stale"] + ok, note = asyncio.run(verify_r3(object(), {"r3_due_titles": titles, "r3_due_count": 2}, run)) + assert ok is want, note + if expect: + assert expect in note.lower() + + +class _S5Plane: + def __init__( + self, + *, + cycle_view: bool, + time_tracking: bool, + customers: bool = True, + features_cycles: bool | None = None, + ): + self.projects = SimpleNamespace( + retrieve=lambda **kw: SimpleNamespace( + id=kw["project_id"], + cycle_view=cycle_view, + is_time_tracking_enabled=time_tracking, + ), + get_features=lambda **kw: SimpleNamespace( + model_dump=lambda: { + "cycles": features_cycles if features_cycles is not None else cycle_view, + "modules": False, + } + ), + ) + self.workspaces = SimpleNamespace( + get_features=lambda **kw: SimpleNamespace(model_dump=lambda: {"customers": customers}) + ) + + +@pytest.mark.parametrize( + ("cycle_view", "tracking", "customers", "features_cycles", "want", "expect"), + [ + pytest.param(True, False, True, False, False, ("is_time_tracking_enabled",), id="worklogs-off"), + pytest.param(False, True, True, False, False, ("cycle_view",), id="cycles-off"), + pytest.param(False, False, True, False, False, ("cycle_view", "is_time_tracking_enabled"), id="customers-only"), + pytest.param(True, True, False, True, False, ("customers",), id="customers-off"), + pytest.param(True, True, True, True, True, (), id="all-three-enabled"), + ], +) +def test_s5_requires_all_three_features_not_a_majority(cycle_view, tracking, customers, features_cycles, want, expect): + plane = _S5Plane( + cycle_view=cycle_view, + time_tracking=tracking, + customers=customers, + features_cycles=features_cycles, + ) + ok, note = asyncio.run(verify_s5(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + assert ok is want, note + for text in expect: + assert text in note, note + + +# --------------------------------------------------------------------------- +# The five verifiers that shipped without a behavioural test: W1, W9, S1, S2, S4. +# Each case below is one a *wrong* verifier would accept, so a regression that +# loosens a check fails here rather than inflating a battery score. +# --------------------------------------------------------------------------- + +W1_TITLE = "Login page 500s on empty password" +ME_ID = "me-1" + + +class _W1Plane: + def __init__(self, items: list[Any], detail: Any): + self._items = items + self._detail = detail + self.work_items = SimpleNamespace(list=lambda **kw: _Page(self._items), retrieve=self._retrieve) + self.users = SimpleNamespace(get_me=lambda **kw: SimpleNamespace(id=ME_ID)) + + def _retrieve(self, **kw): + # W1 verifies the newest duplicate, so the detail must be keyed by the id it asked for. + return self._detail(kw["work_item_id"]) if callable(self._detail) else self._detail + + +def _w1_detail( + *, priority: str = "urgent", assignees: tuple[str, ...] = (ME_ID,), labels: tuple[str, ...] = ("auth-id",) +): + return SimpleNamespace( + priority=priority, + assignees=[SimpleNamespace(id=a) for a in assignees], + labels=[SimpleNamespace(id=lid) for lid in labels], + ) + + +@pytest.mark.parametrize( + ("items", "detail", "ctx_labels", "want", "expect"), + [ + pytest.param( + [_item("w1", W1_TITLE)], _w1_detail(), {"auth": "auth-id"}, True, "auth label attached", id="all-three-met" + ), + pytest.param([], _w1_detail(), {"auth": "auth-id"}, False, "not found", id="never-created"), + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(priority="high"), + {"auth": "auth-id"}, + False, + "want urgent", + id="priority-close-but-wrong", + ), + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(assignees=("someone-else",)), + {"auth": "auth-id"}, + False, + "missing me", + id="assigned-to-the-wrong-person", + ), + # A label *named* auth is not the seeded label. Matching on name would pass this. + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(labels=("decoy-id",)), + {"auth": "auth-id"}, + False, + "missing auth", + id="decoy-label-with-the-right-name", + ), + # Fail closed: a seed that never produced the label must not make the requirement vanish. + pytest.param( + [_item("w1", W1_TITLE)], + _w1_detail(), + {}, + False, + "auth label id missing from seed ctx", + id="seed-lost-the-label", + ), + ], +) +def test_w1_requires_all_three_conditions_and_the_seeded_label_id(items, detail, ctx_labels, want, expect): + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": ctx_labels} + ok, note = asyncio.run(verify_w1(_W1Plane(items, detail), ctx, _run())) + assert ok is want, note + assert expect in note, note + + +def test_w1_verifies_the_newest_duplicate_and_says_so(): + """Two items share the title; only the newest satisfies the ask.""" + items = [ + _item("old", W1_TITLE, created_at="2026-01-01T00:00:00Z"), + _item("new", W1_TITLE, created_at="2026-06-01T00:00:00Z"), + ] + details = {"old": _w1_detail(priority="low"), "new": _w1_detail()} + ctx = {"workspace_slug": "ws", "project_id": "p1", "labels": {"auth": "auth-id"}} + ok, note = asyncio.run(verify_w1(_W1Plane(items, lambda wid: details[wid]), ctx, _run())) + assert ok is True, note + assert "2 items with title" in note, note + + +W9_TITLES = ( + "Checkout times out on 3DS challenge", + "Session cookie not rotated after login", + "Inventory count goes negative under load", +) + + +class _W9Plane: + def __init__(self, priorities: dict[str, str], missing: tuple[str, ...] = ()): + self._priorities = priorities + rows = [_item(f"id-{i}", t) for i, t in enumerate(W9_TITLES) if t not in missing] + self.work_items = SimpleNamespace( + list=lambda **kw: _Page(rows), + retrieve=lambda **kw: SimpleNamespace(priority=self._priorities.get(kw["work_item_id"])), + ) + + +@pytest.mark.parametrize( + ("priorities", "missing", "want", "expect"), + [ + pytest.param( + {"id-0": "high", "id-1": "high", "id-2": "high"}, (), True, "3 items priority=high", id="all-three" + ), + # The majority trap: two of three is a failed task, not a pass. + pytest.param({"id-0": "high", "id-1": "high", "id-2": "medium"}, (), False, "Inventory", id="two-of-three"), + pytest.param({"id-0": "high", "id-1": "high"}, (W9_TITLES[2],), False, "missing", id="one-never-existed"), + pytest.param({"id-0": "High", "id-1": "HIGH", "id-2": "high"}, (), True, "3 items", id="priority-case-varies"), + pytest.param({"id-0": None, "id-1": "high", "id-2": "high"}, (), False, "Checkout", id="priority-unset"), + ], +) +def test_w9_requires_all_three_items_not_a_majority(priorities, missing, want, expect): + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = asyncio.run(verify_w9(_W9Plane(priorities, missing), ctx, _run())) + assert ok is want, note + assert expect in note, note + + +class _S1Props: + """Type-scoped property listing, with the options collection as a separate endpoint.""" + + def __init__(self, props: list[Any] | Exception, options: list[Any] | Exception | None = None): + self._props = props + self._options = options if options is not None else [] + self.list = self._list + self.options = SimpleNamespace(list=self._options_list) + + def _list(self, **kw): + if isinstance(self._props, Exception): + raise self._props + return self._props + + def _options_list(self, **kw): + if isinstance(self._options, Exception): + raise self._options + return self._options + + +def _severity(*, property_type: Any = "OPTION", options: tuple[str, ...] = ("Critical", "Major", "Minor"), **kw: Any): + return SimpleNamespace( + id="sev-1", + display_name="Severity", + property_type=property_type, + options=[SimpleNamespace(name=name) for name in options], + **kw, + ) + + +@pytest.mark.parametrize( + ("props", "options", "want", "expect"), + [ + pytest.param( + [_severity()], None, True, "Severity OPTION with Critical/Major/Minor", id="option-with-all-three" + ), + pytest.param([_severity(property_type="TEXT")], None, False, "want OPTION", id="text-instead-of-option"), + pytest.param( + [_severity(options=("Critical", "Major"))], None, False, "missing ['minor']", id="one-choice-short" + ), + pytest.param( + [_severity(options=("CRITICAL", "major", "MiNoR"))], + None, + True, + "Severity OPTION", + id="choice-casing-varies", + ), + # A Severity bound to some other work item type does not satisfy "on the Bug type". + pytest.param([_severity(issue_type="other-type")], None, False, "not found", id="attached-to-the-wrong-type"), + pytest.param([], None, False, "not found", id="never-created"), + # An empty inline collection falls back to the options endpoint. + pytest.param( + [_severity(options=())], + [SimpleNamespace(name="Critical"), SimpleNamespace(name="Major"), SimpleNamespace(name="Minor")], + True, + "Severity OPTION", + id="options-only-on-the-endpoint", + ), + # A 404 on the options endpoint means the choices definitively are not there. + pytest.param([_severity(options=())], _http404(), False, "missing", id="options-endpoint-404"), + # A type-scoped 404 is authoritative absence, not an infrastructure failure. + pytest.param(_http404(), None, False, "type-scoped list empty/404", id="type-scoped-404-is-a-real-failure"), + ], +) +def test_s1_requires_an_option_severity_attached_to_the_bug_type(props, options, want, expect): + plane = SimpleNamespace(work_item_properties=_S1Props(props, options)) + ctx = {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug-type-1"}} + ok, note = asyncio.run(verify_s1(plane, ctx, _run())) + assert ok is want, note + assert expect in note, note + + +def test_s1_skips_when_the_bug_type_was_never_seeded(): + """No fixture means the question was never asked — not an agent failure.""" + plane = SimpleNamespace(work_item_properties=_S1Props([_severity()])) + with pytest.raises(TaskSkipped): + asyncio.run(verify_s1(plane, {"workspace_slug": "ws", "project_id": "p1"}, _run())) + + +def test_s1_surfaces_a_non_404_read_failure_as_infrastructure(): + """A 500 while reading authoritative state must not be scored as a failed task.""" + plane = SimpleNamespace(work_item_properties=_S1Props(HttpError("boom", status_code=500, response={}))) + ctx = {"workspace_slug": "ws", "project_id": "p1", "bug_type": {"id": "bug-type-1"}} + with pytest.raises(VerifierReadError): + asyncio.run(verify_s1(plane, ctx, _run())) + + +class _S2Plane: + def __init__( + self, + *, + values: tuple[str, ...], + estimate_point: Any, + item: bool = True, + retrieve_error: Exception | None = None, + ): + self._error = retrieve_error + self._points = [SimpleNamespace(id=f"pt-{v}", value=v) for v in values] + self._estimate_point = estimate_point + rows = [_item("w8", W8_TITLE)] if item else [] + self.estimates = SimpleNamespace(retrieve=self._retrieve, list_points=lambda **kw: self._points) + self.work_items = SimpleNamespace( + list=lambda **kw: _Page(rows), + retrieve=lambda **kw: SimpleNamespace(estimate_point=self._estimate_point), + ) + + def _retrieve(self, **kw): + if self._error: + raise self._error + return SimpleNamespace(id="est-1") + + +FIB = ("1", "2", "3", "5", "8") + + +@pytest.mark.parametrize( + ("plane", "want", "expect"), + [ + pytest.param( + _S2Plane(values=FIB, estimate_point="pt-5"), True, "item estimate_point=5", id="scale-and-item-both-right" + ), + # Both halves are required; a correct scale with the wrong point is not a pass. + pytest.param(_S2Plane(values=FIB, estimate_point="pt-3"), False, "want 5", id="item-points-at-the-wrong-value"), + pytest.param( + _S2Plane(values=("1", "2", "3", "5"), estimate_point="pt-5"), + False, + "missing fib subset", + id="scale-missing-8", + ), + pytest.param(_S2Plane(values=FIB, estimate_point=None), False, "want 5", id="item-has-no-estimate"), + pytest.param(_S2Plane(values=FIB, estimate_point="pt-5", item=False), False, "missing", id="target-item-gone"), + # estimate_point may arrive expanded rather than as a UUID; both are the same end state. + pytest.param( + _S2Plane(values=FIB, estimate_point=SimpleNamespace(value="5")), + True, + "item estimate value=5", + id="expanded-estimate-point", + ), + # No estimate at all reads as "the requested scale was never created", not an error. + pytest.param( + _S2Plane(values=FIB, estimate_point="pt-5", retrieve_error=_http404()), + False, + "was not created", + id="estimate-404", + ), + ], +) +def test_s2_requires_both_the_fibonacci_scale_and_the_item_estimate(plane, want, expect): + ctx = {"workspace_slug": "ws", "project_id": "p1"} + ok, note = asyncio.run(verify_s2(plane, ctx, _run())) + assert ok is want, note + assert expect in note, note + + +class _S4Intake: + def __init__( + self, statuses: dict[str, int | None], *, retrieve_works: bool = True, list_error: Exception | None = None + ): + self._statuses = statuses + self._retrieve_works = retrieve_works + self._list_error = list_error + + def retrieve(self, **kw): + if not self._retrieve_works: + raise _http404() + return SimpleNamespace(status=self._statuses.get(kw["work_item_id"])) + + def list(self, **kw): + if self._list_error: + raise self._list_error + return _Page( + [ + SimpleNamespace( + status=self._statuses.get("billing-1"), issue_detail=SimpleNamespace(name=INTAKE_BILLING_TITLE) + ), + SimpleNamespace( + status=self._statuses.get("spam-1"), issue_detail=SimpleNamespace(name=INTAKE_SPAM_TITLE) + ), + ] + ) + + +def _s4_ctx(*, billing: str | None = "billing-1", spam: str | None = "spam-1"): + return { + "workspace_slug": "ws", + "project_id": "p1", + "intake": {"billing": {"issue_id": billing}, "spam": {"issue_id": spam}}, + } + + +@pytest.mark.parametrize( + ("statuses", "ctx", "want", "expect"), + [ + pytest.param( + {"billing-1": 1, "spam-1": -1}, _s4_ctx(), True, "billing accepted; spam declined", id="right-call-on-both" + ), + # The sign is the whole task: triaging both the same way is not partial credit. + pytest.param({"billing-1": -1, "spam-1": 1}, _s4_ctx(), False, "billing status=-1", id="decisions-swapped"), + pytest.param({"billing-1": 1, "spam-1": 1}, _s4_ctx(), False, "spam status=1", id="accepted-the-spam-too"), + pytest.param( + {"billing-1": None, "spam-1": -1}, _s4_ctx(), False, "billing status=None", id="billing-untouched" + ), + pytest.param( + {"billing-1": 1, "spam-1": -1}, _s4_ctx(billing=None), False, "billing status=None", id="seed-lost-the-id" + ), + ], +) +def test_s4_requires_the_opposite_decision_on_each_intake_row(statuses, ctx, want, expect): + plane = SimpleNamespace(intake=_S4Intake(statuses)) + ok, note = asyncio.run(verify_s4(plane, ctx, _run())) + assert ok is want, note + assert expect in note, note + + +def test_s4_falls_back_to_the_intake_list_when_retrieve_is_unavailable(): + """Retrieve is optional; the list is independently authoritative for the same rows.""" + plane = SimpleNamespace(intake=_S4Intake({"billing-1": 1, "spam-1": -1}, retrieve_works=False)) + ok, note = asyncio.run(verify_s4(plane, _s4_ctx(), _run())) + assert ok is True, note + + +def test_s4_surfaces_a_double_read_failure_as_infrastructure(): + """With neither endpoint readable, the state is unknown — not declined.""" + plane = SimpleNamespace( + intake=_S4Intake({}, retrieve_works=False, list_error=HttpError("boom", status_code=500, response={})) + ) + with pytest.raises(VerifierReadError): + asyncio.run(verify_s4(plane, _s4_ctx(), _run())) diff --git a/tests/evals/test_cli.py b/tests/evals/test_cli.py new file mode 100644 index 0000000..a27437f --- /dev/null +++ b/tests/evals/test_cli.py @@ -0,0 +1,145 @@ +"""Offline eval tests for cli.""" + +from __future__ import annotations + +import pytest + +from evals import cli as run_mod +from evals.cli import cmd_dry_run, cmd_list, parse_args, resolve_model_for_driver +from evals.cli import main as eval_main +from evals.tasks.catalog import TASKS + +DESIGN_IDS = { + "R1", + "R2", + "R3", + "R4", + "R5", + "R6", + "W1", + "W2", + "W3", + "W4", + "W5", + "W6", + "W7", + "W8", + "S1", + "S2", + "S3", + "S4", + "C1", + "C2", +} + +EXTRA_IDS = {"W9", "W10", "W11", "R7", "S5"} # bulk, pages, feature recovery, transitions, features +DEBIAS_IDS = {"I1", "I2", "I3", "I4", "I5", "L1", "L2", "L3", "L4", "L5"} + + +@pytest.mark.parametrize("case", ["list-task-ids", "dry-run-all"]) +def test_cmd_behaviours(case, capsys): + if case == "list-task-ids": + assert cmd_list() == 0 + out = capsys.readouterr().out + # Every registered task, not a subset: the listing is how a caller discovers the + # battery, and a task missing from it is invisible. + assert DESIGN_IDS | EXTRA_IDS | DEBIAS_IDS == {task["id"] for task in TASKS} + for task_id in DESIGN_IDS | EXTRA_IDS | DEBIAS_IDS: + assert task_id in out + else: + assert cmd_dry_run(list(TASKS)) == 0 + out = capsys.readouterr().out + assert "Seed plan:" in out + for task_id in ("R1", "W9", "S4", "C2", "R7"): + assert f"=== {task_id} ===" in out + + +@pytest.mark.parametrize("case", ["list", "driver", "resume-and-canary"]) +def test_parse_args_behaviours(case): + if case == "list": + args = parse_args(["--list", "--label", "candidate-build"]) + assert args.list is True + assert args.label == "candidate-build" + assert parse_args(["--list"]).label == "local" + elif case == "driver": + assert parse_args(["--driver", "claude-cli", "--dry-run"]).driver == "claude-cli" + defaults = parse_args(["--dry-run"]) + assert (defaults.driver, defaults.model, defaults.provider) == ("api", "standard", "anthropic") + assert defaults.record_result_payloads is False + recorded = parse_args(["--driver", "claude-cli", "--record-result-payloads", "--dry-run"]) + assert recorded.record_result_payloads is True + else: + assert run_mod.parse_args(["--resume", "evals/output/x.jsonl", "--dry-run"]).resume == "evals/output/x.jsonl" + assert run_mod.parse_args(["--canary", "--tasks", "R1"]).canary is True + assert run_mod.parse_args(["--canary", "--canary-strict", "R1,R2"]).canary_strict == "R1,R2" + + +def test_canary_strict_cli_passes_explicit_required_ids(monkeypatch): + seen: dict = {} + + async def fake_canary(tasks, *, label, required_task_ids): + seen.update(task_ids=[task["id"] for task in tasks], label=label, required=required_task_ids) + return 0 + + monkeypatch.setattr(run_mod, "run_canary", fake_canary) + rc = eval_main(["--canary", "--tasks", "R1,R2", "--canary-strict", "R1", "--label", "ci"]) + assert rc == 0 + assert seen == {"task_ids": ["R1", "R2"], "label": "ci", "required": {"R1"}} + + +def test_canary_strict_cli_rejects_invalid_usage(capsys): + assert eval_main(["--canary-strict", "R1"]) == 2 + assert "requires --canary" in capsys.readouterr().err + assert eval_main(["--canary", "--canary-strict", "NOPE"]) == 2 + assert "unknown --canary-strict" in capsys.readouterr().err + + +def test_model_tiers_resolve_per_driver_and_provider(): + assert resolve_model_for_driver("api", "standard", provider="anthropic") == "claude-sonnet-5" + assert resolve_model_for_driver("api", "fast", provider="anthropic") == "claude-haiku-4-5" + assert resolve_model_for_driver("api", "standard", provider="openai") == "gpt-5.6-sol" + assert resolve_model_for_driver("api", "fast", provider="openai") == "gpt-5.6-luna" + assert resolve_model_for_driver("claude-cli", "standard") == "sonnet" + assert resolve_model_for_driver("claude-cli", "fast") == "haiku" + assert resolve_model_for_driver("codex-cli", "standard") == "gpt-5.6-sol" + assert resolve_model_for_driver("codex-cli", "fast") == "gpt-5.6-luna" + assert resolve_model_for_driver("antigravity-cli", "standard") == "gemini-3.6-flash-high" + assert resolve_model_for_driver("antigravity-cli", "fast") == "gemini-3.6-flash-low" + + +@pytest.mark.parametrize( + ("driver", "model"), + [ + ("api", "claude-opus-5"), + ("api", "sonnet"), + ("claude-cli", "sonnet"), + ("codex-cli", "sonnet"), + ("antigravity-cli", "gemini-3.1-pro-high"), + ("opencode-cli", "haiku"), + ], +) +def test_non_tier_model_strings_pass_through_unchanged(driver, model): + assert resolve_model_for_driver(driver, model) == model + + +@pytest.mark.parametrize("case", ["direct-guidance", "cli-error"]) +def test_unmapped_behaviours(case, tmp_path, capsys): + if case == "direct-guidance": + with pytest.raises(ValueError, match=r"opencode models"): + resolve_model_for_driver("opencode-cli", "standard") + return + + out = tmp_path / "must-not-exist.jsonl" + rc = eval_main(["--driver", "opencode-cli", "--model", "standard", "--tasks", "R1", "--out", str(out)]) + assert rc == 2 + assert "explicit provider/model ID" in capsys.readouterr().err + assert out.exists() is False + + +def test_tier_mapping_is_scoped_to_cli_provider(): + with pytest.raises(ValueError, match=r"codex-cli.*anthropic.*explicit model ID"): + resolve_model_for_driver("codex-cli", "standard", provider="anthropic") + + +def test_qualified_model_id_passes_through_unchanged(): + assert resolve_model_for_driver("opencode-cli", "openai/gpt-4o") == "openai/gpt-4o" diff --git a/tests/evals/test_docs.py b/tests/evals/test_docs.py new file mode 100644 index 0000000..f2e6af4 --- /dev/null +++ b/tests/evals/test_docs.py @@ -0,0 +1,41 @@ +"""The runbook must not re-acquire claims the code has stopped making. + +Documentation drifts silently. These are the two claims that cost the most when stale: a +prerequisite that is no longer required turns people away from running the harness at all, +and a fingerprint description that omits the revision hides why results stopped comparing. +""" + +from __future__ import annotations + +import pytest + +from evals import REPO_ROOT + +README = (REPO_ROOT / "evals" / "README.md").read_text() +DESIGN = (REPO_ROOT / "evals" / "DESIGN.md").read_text() + + +@pytest.mark.parametrize( + "case", + ["optional-flag-server", "plan-gate-reason", "catalog-revision", "exit-zero-contract"], +) +def test_the_behaviours(case): + if case == "optional-flag-server": + assert "FEATURE_FLAG_SERVER_BASE_URL" in README, "the option should still be documented" + index = README.index("FEATURE_FLAG_SERVER_BASE_URL") + paragraph = README[max(0, index - 200) : index + 500] + assert "not** required" in paragraph or "Optionally" in paragraph, ( + "the flag server stopped being a prerequisite when the seeders learned to skip; " + "the runbook must not tell people otherwise" + ) + elif case == "plan-gate-reason": + assert "env:plan-gated:" in README + elif case == "catalog-revision": + assert "CATALOG_REVISION" in README + else: + assert "exit 0 does **not** mean the agent passed" in README + assert "execution coverage" in README + + +def test_design_still_states_the_skip_contract_the_seeders_now_implement(): + assert "not rewritten as an agent task failure" in DESIGN diff --git a/tests/evals/test_error_class.py b/tests/evals/test_error_class.py new file mode 100644 index 0000000..3b9af4a --- /dev/null +++ b/tests/evals/test_error_class.py @@ -0,0 +1,320 @@ +"""Classifying what kind of "no" a tool call received, and what counts as friction. + +Every payload below is one that a real battery produced. One errored-call count +answered three unrelated questions at once: on an unpatched 28-tool surface, 31 of +40 errors were the schema correcting a malformed call, one was a genuine tool-design +defect, and one was a fair question fairly answered. Reading them as one number is +what made the defect invisible. +""" + +from __future__ import annotations + +import pytest + +from evals.core.error_class import ( + DENIED, + FAILED, + NOT_FOUND, + REFUSED, + REJECTED, + UNCLASSIFIED, + classify_error, + detect_refusal, +) +from evals.core.results import CallRecord +from evals.report.schema_friction import split_errors + +# --- the classifier, on payloads observed in real runs ----------------------- + +OBSERVED = [ + # This server's own refusals: no status, no pydantic shape, deliberate wording. + ("Error: project requires an action. It takes: archive, create, delete, list.", REFUSED), + ("Error: action 'create' does not take: points. It takes: description, name.", REFUSED), + # FastMCP rejecting a call against the tool signature, before the body runs. + ( + "1 validation error for call[project]\naction\n Missing required argument " + "[type=missing_argument, input_value={}, input_type=dict]", + REFUSED, + ), + ( + "1 validation error for call[workspace]\naction\n Input should be 'get_features' " + "or 'update_features' [type=literal_error, input_value='list', input_type=str]", + REFUSED, + ), + # The API answering a fair existence question. + ("Error calling tool 'project_estimate': HTTP 404: Not Found: Estimate not found", NOT_FOUND), + # The API refusing the meaning of a well-formed call -- the defect class. + ("HTTP 400: Bad Request: The old cycle is not completed yet", REJECTED), + ("HTTP 409: Conflict: name: The project name is already taken", REJECTED), + # Plan and permission gates say nothing about the tool surface. + ("HTTP 402: Payment Required: Upgrade your plan to access Initiatives", DENIED), + ("HTTP 403: Forbidden: Customer feature is not enabled for this workspace", DENIED), + ("HTTP 500: Internal Server Error", FAILED), +] + + +@pytest.mark.parametrize(("payload", "expected"), OBSERVED, ids=[e + ":" + p[:28] for p, e in OBSERVED]) +def test_an_observed_payload_lands_in_its_class(payload: str, expected: str): + assert classify_error(payload) == expected + + +def test_a_status_outranks_wording(): + """A 404 that happens to mention an argument is still an absent resource. + + Only a payload with no status at all can be a schema refusal, because that is + exactly the case where the call never reached the API. + """ + assert classify_error("HTTP 404: Not Found: missing required argument foo") == NOT_FOUND + + +def test_an_unrecognised_payload_is_never_guessed_into_a_class(): + """Silently sorting the unknown into `refused` would inflate the one number + that is supposed to be attributable to our own schema.""" + assert classify_error("something went sideways") == UNCLASSIFIED + assert classify_error("") == UNCLASSIFIED + assert classify_error(None) == UNCLASSIFIED + + +def test_a_foreign_surface_still_classifies_by_status(): + """The battery scores servers it has never seen -- a 177-tool build, a future v2. + + Those emit neither this server's refusal wording nor its tool names, so status + has to carry them. Coupling the classifier to `ACTIONS` would end that. + """ + assert classify_error("HTTP 400: Bad Request: whatever a foreign server says") == REJECTED + assert classify_error("HTTP 404: Not Found") == NOT_FOUND + + +# --- the split, including the rule that keeps a fair question out of friction --- + + +def err(tool: str, action: str | None, kind: str) -> CallRecord: + return CallRecord(tool=tool, action=action, is_error=True, error_class=kind) + + +def test_an_unclassified_error_is_not_filed_beside_ones_we_chose_not_to_charge(): + """`other` means classified and deliberately not charged to tool design. + `unclassified` means we do not know, which a reader must be able to tell apart. + """ + counts = split_errors([err("project", "list", DENIED), err("project", "list", UNCLASSIFIED)]) + assert counts["other"] == 1 + assert counts["unclassified"] == 1 + + +def test_each_kind_lands_in_its_own_column(): + counts = split_errors( + [ + err("project", None, REFUSED), + err("cycle", "transfer_workitems", REJECTED), + err("initiative", "list", DENIED), + CallRecord(tool="project", action="list"), # a success is not counted anywhere + ] + ) + assert counts == { + "navigation": 1, + "surface": 1, + "answered": 0, + "other": 1, + "unclassified": 0, + "unflagged": 0, + } + + +def test_a_first_absent_read_is_an_answer_not_friction(): + """`project_estimate retrieve` -> 404 before creating one is the correct move. + + Three separate models made this exact call and each was charged for it. There is + no cheaper way to ask whether something exists than to ask. + """ + counts = split_errors([err("project_estimate", "retrieve", NOT_FOUND)]) + assert counts["answered"] == 1 + assert counts["surface"] == 0 + + +def test_a_repeated_absent_read_is_friction(): + """Asking twice means the first answer did not land, which is the surface's problem.""" + counts = split_errors( + [ + err("project_estimate", "retrieve", NOT_FOUND), + err("project_estimate", "retrieve", NOT_FOUND), + err("project_estimate", "retrieve", NOT_FOUND), + ] + ) + assert counts["answered"] == 1 + assert counts["surface"] == 2 + + +def test_absent_reads_of_different_things_are_each_their_own_question(): + counts = split_errors( + [ + err("project_estimate", "retrieve", NOT_FOUND), + err("cycle", "retrieve", NOT_FOUND), + err("project_estimate", "list_points", NOT_FOUND), + ] + ) + assert counts["answered"] == 3 + assert counts["surface"] == 0 + + +def test_an_unclassified_error_is_never_counted_as_surface_friction(): + """Surface friction is the number someone will act on, so it may only hold + calls we can actually attribute to tool design.""" + counts = split_errors([err("project", "list", UNCLASSIFIED), err("project", "list", FAILED)]) + assert counts["surface"] == 0 + assert counts["unclassified"] == 1 + assert counts["other"] == 1 + + +def test_a_row_from_before_this_field_existed_does_not_crash_or_inflate(): + """Older result files carry is_error with no error_class.""" + counts = split_errors([CallRecord(tool="project", action="list", is_error=True)]) + assert counts["unclassified"] == 1 + assert counts["surface"] == 0 + assert counts["other"] == 0 + + +# --- the whole path, because a key dropped anywhere on it reaches no report ---- + + +def test_a_class_survives_every_hop_from_proxy_to_report(tmp_path): + """The classifier ran and the report still said "unclassified", because the + sidecar reader rebuilds each call from an explicit key list and did not copy the + field. Every hop is asserted here: proxy row -> sidecar reader -> AgentRun -> + TaskResult -> serialized row -> reloaded row. + """ + import json + + from evals.core.results import AgentRun, TaskResult, Usage, agent_run_to_task_result + from evals.drivers.cli.sidecar import load_proxy_sidecar + + sidecar = tmp_path / "proxy-sidecar.jsonl" + rows = [ + { + "tool": "cycle", + "args": {"action": "transfer_workitems"}, + "is_error": True, + "error_class": REJECTED, + "result_chars": 145, + "duration_ms": 115, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + }, + ] + sidecar.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + calls, _status = load_proxy_sidecar(sidecar) + assert calls and calls[0].get("error_class") == REJECTED, "the sidecar reader dropped it" + + agent = agent_run_to_task_result(AgentRun(calls=calls, final_text="done", usage=Usage(), stopped_reason="end_turn")) + assert agent.calls[0].error_class == REJECTED, "agent_run_to_task_result dropped it" + + serialized = json.loads(json.dumps(agent.to_row())) + assert serialized["calls"][0]["error_class"] == REJECTED, "serialization dropped it" + + reloaded = TaskResult.from_row(serialized) + assert reloaded.calls[0].error_class == REJECTED, "reload dropped it" + assert split_errors(reloaded.calls)["surface"] == 1, "the report did not see it" + + +def test_request_args_survive_the_hop_chain_on_every_driver(tmp_path): + """A recorded refusal that cannot be attributed to a target answers half a question. + + W7 failed reproducibly with workitem_link.create reporting success while the link was + absent, and the two candidate explanations -- wrong target, or a create that does not + persist -- were indistinguishable because only args_chars was kept. + + Args used to ride along with a recorded payload and stay out otherwise. That coupling + was to result_text, which only the recording proxy sets, so the api driver -- which + calls tools directly -- recorded args on none of its 484 calls while a CLI arm recorded + them on 330 of 331. Since the arguments are in hand on both paths and `action` is + already kept unconditionally, they are now recorded either way. This still guards the + hop chain, which is where a field of this kind gets silently dropped. + """ + import json + + from evals.core.results import AgentRun, TaskResult, Usage, agent_run_to_task_result + from evals.drivers.cli.sidecar import load_proxy_sidecar + + args = {"action": "create", "workitem_id": "wi-42", "url": "https://example.com/eval/runbook-w7"} + + def row_for(*, with_payload: bool) -> dict: + row = { + "tool": "workitem_link", + "args": args, + "is_error": False, + "result_chars": 88, + "duration_ms": 12, + "seq": 1, + } + if with_payload: + row["result_text"] = '{"content":[{"type":"text","text":"link created"}]}' + return row + + def roundtrip(*, with_payload: bool) -> TaskResult: + sidecar = tmp_path / f"sidecar-{with_payload}.jsonl" + sidecar.write_text(json.dumps(row_for(with_payload=with_payload)) + "\n", encoding="utf-8") + calls, _status = load_proxy_sidecar(sidecar) + assert calls, "the sidecar reader produced no calls" + result = agent_run_to_task_result( + AgentRun(calls=calls, final_text="done", usage=Usage(), stopped_reason="end_turn") + ) + return TaskResult.from_row(json.loads(json.dumps(result.to_row()))) + + recorded = roundtrip(with_payload=True) + assert recorded.calls[0].args_json is not None, "args were dropped somewhere in the chain" + assert json.loads(recorded.calls[0].args_json) == args + # The target is the point: without it the record cannot say which item was linked. + assert "wi-42" in recorded.calls[0].args_json + + # A call with no recorded payload keeps its args too -- that is the whole point of + # decoupling them, since the api driver never produces a payload to ride along with. + plain = roundtrip(with_payload=False) + assert plain.calls[0].args_json is not None, "args must not depend on payload recording" + assert json.loads(plain.calls[0].args_json) == args + assert plain.calls[0].args_chars > 0 + assert plain.calls[0].action == "create", "action is kept regardless — it is half the tool choice" + + +def test_a_refusal_the_server_called_a_success_is_still_counted(): + """The server answers a malformed call with a plain result whose text says + "Error", so the protocol reports success. Roughly 47 per battery arrived that + way and were counted as successes. They are counted here and named apart, + because they are absent from the errored-call total every earlier run used. + """ + counts = split_errors( + [ + CallRecord(tool="project", action=None, is_error=False, error_class=REFUSED), + CallRecord(tool="project", action="list", is_error=True, error_class=REFUSED), + ] + ) + assert counts["navigation"] == 2 + assert counts["unflagged"] == 1 + + +def test_a_plain_successful_call_is_still_not_counted(): + counts = split_errors([CallRecord(tool="project", action="list")]) + assert sum(counts.values()) == 0 + + +REFUSAL_TEXTS = [ + ('{"content":[{"type":"text","text":"Error: project requires an action. It takes: list."}]}', REFUSED), + ('{"content":[{"type":"text","text":"Error: action \'create\' does not take: points. It takes: name."}]}', REFUSED), + # An ordinary result quoting one half of the stray-argument wording is not a refusal. + ('{"content":[{"type":"text","text":"the docs say the endpoint does not take: a body"}]}', None), + ('{"content":[{"type":"text","text":"[{"id":"abc","name":"Delivery Planning"}]"}]}', None), +] + + +@pytest.mark.parametrize( + ("payload", "expected"), REFUSAL_TEXTS, ids=["missing-action", "stray-arg", "quotes-one-half", "ordinary-result"] +) +def test_only_this_servers_own_refusal_wording_is_detected(payload: str, expected: str | None): + assert detect_refusal(payload) == expected diff --git a/tests/evals/test_evidence.py b/tests/evals/test_evidence.py new file mode 100644 index 0000000..6074663 --- /dev/null +++ b/tests/evals/test_evidence.py @@ -0,0 +1,17 @@ +def test_aggregate_evidence_alone_counts_as_registered_target_bound_evidence(): + """A count is evidence: the proxy matches an exact total_count for a targeted request. + + The live runner's seed gate asked only for sentinels and targets, so a read task whose + answer *is* a count could register its evidence and still be rejected as having registered + none. L2 failed that way on every repetition of the first full-catalog battery, after the + seeding bug that had hidden it was fixed. + """ + from evals.core.evidence import TARGET_ENTITY_EVIDENCE, configured_evidence_labels + + targets = {TARGET_ENTITY_EVIDENCE: ("wi-1",)} + aggregates = {TARGET_ENTITY_EVIDENCE: ({"kind": "total_count", "value": 3},)} + + assert configured_evidence_labels(None, targets, aggregates) == (TARGET_ENTITY_EVIDENCE,) + # Targets alone are still not evidence, and neither is an aggregate with nothing to bind to. + assert configured_evidence_labels(None, targets, None) == () + assert configured_evidence_labels(None, None, aggregates) == () diff --git a/tests/evals/test_failure_kind.py b/tests/evals/test_failure_kind.py new file mode 100644 index 0000000..cfc8429 --- /dev/null +++ b/tests/evals/test_failure_kind.py @@ -0,0 +1,120 @@ +"""The observed notes are the case table. + +Every string here was emitted by a real verifier in a recorded run; the scan behind +this file covered 100 distinct notes across every result file on disk. +""" + +from __future__ import annotations + +import pytest + +from evals.core.failure_kind import ( + ABANDONED, + ENVIRONMENT, + FAILURE_KINDS, + MISSING_WRITE, + PARTIAL_WRITE, + UNCLASSIFIED, + UNPROVEN, + WRONG_VALUE, + classify_failure, +) + + +def test_the_three_defects_that_motivated_this_are_distinct(): + """W7, I1 and S2 were three different defects reported identically as 'failed'.""" + w7 = classify_failure("blocking relation present; link 'https://example.com/eval/runbook-w7' missing; have []") + i1 = classify_failure("work_item 0cf779b6 priority='urgent' (want high)") + s2 = classify_failure("estimate points missing fib subset; have []; item estimate_point=None (want 5)") + assert w7 == PARTIAL_WRITE + assert i1 == WRONG_VALUE + assert s2 == MISSING_WRITE + assert len({w7, i1, s2}) == 3 + + +def test_a_correct_answer_the_harness_could_not_prove_is_not_an_agent_defect(): + """The largest family in the corpus. Counting these as defects would be wrong. + + The agent answered correctly; the run could not evidence it. That is a harness + property, and lumping it with wrong answers would misattribute the biggest + single group of failures to the model. + """ + for note in ( + "answer_correct=true (final text reports exactly 1 seeded comments); provenance=trace incomplete " + "(source=proxy; proxy sidecar was not authoritative)", + "answer_correct=true (final text reports activity count 1 via contract); provenance=missing " + "(0 evidence-bearing of 4 successful Plane calls; 5 total)", + ): + assert classify_failure(note) == UNPROVEN + + +def test_a_wrong_answer_is_a_wrong_value_even_when_provenance_is_named(): + note = ( + "answer_correct=false (logged-minutes values=['90', '90']; want ['90']); " + "provenance=observed seeded-value response evidence (source=proxy)" + ) + assert classify_failure(note) == WRONG_VALUE + + +@pytest.mark.parametrize( + "note", + [ + "customer 'Acme Corp' not found", + "Severity property not found on Bug type", + "project estimate not found; requested Fibonacci scale was not created", + ], +) +def test_absent_entities_are_missing_writes(note): + assert classify_failure(note) == MISSING_WRITE + + +def test_a_half_landed_write_is_partial_not_missing(): + assert classify_failure("names 1.2.0; missing changelog content") == PARTIAL_WRITE + + +@pytest.mark.parametrize( + "note", + [ + "state='Backlog' (want exact 'Done')", + "Sprint 12 not closed: end_date='2026-08-20T23:59:00Z' (want end_date='2026-08-19' or archived_at)", + ], +) +def test_value_mismatches_are_wrong_value(note): + assert classify_failure(note) == WRONG_VALUE + + +def test_environment_skips_are_not_defects(): + assert classify_failure("env:no-activity-worker") == ENVIRONMENT + + +def test_running_out_of_iterations_beats_whatever_the_note_says(): + """A capped run's note describes the unfinished state, not why it stopped.""" + assert classify_failure("estimate points missing fib subset", hit_max_iterations=True) == ABANDONED + assert classify_failure("anything at all", stop_reason="max_tokens") == ABANDONED + + +def test_s2_is_not_abandoned_despite_giving_up(): + """S2 spent 43 calls and stopped voluntarily with end_turn. + + The plan assumed stop_reason would classify this directly. It does not -- the + run ended normally, so only the note carries the defect. + """ + assert ( + classify_failure( + "estimate points missing fib subset; have []; item estimate_point=None (want 5)", + stop_reason="end_turn", + hit_max_iterations=False, + ) + == MISSING_WRITE + ) + + +def test_an_unrecognised_note_is_unclassified_never_silently_bucketed(): + """A zero in some kind must mean zero, not 'the classifier did not recognise it'.""" + assert classify_failure("3 module completed items archived") == UNCLASSIFIED + assert classify_failure("") == UNCLASSIFIED + assert classify_failure(None) == UNCLASSIFIED + + +def test_unclassified_is_a_first_class_member(): + assert UNCLASSIFIED in FAILURE_KINDS diff --git a/tests/evals/test_import_compat.py b/tests/evals/test_import_compat.py new file mode 100644 index 0000000..2862fb6 --- /dev/null +++ b/tests/evals/test_import_compat.py @@ -0,0 +1,33 @@ +"""Public import compatibility after neutral fixture extraction.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def test_seed_and_task_packages_import_in_either_order_with_legacy_reexports(): + root = Path(__file__).parents[2] + assertions = """ +from evals.core.errors import TaskSkipped as NeutralTaskSkipped +from evals.core.fixtures import CUSTOMER_NAME as NeutralCustomerName +from evals.seed import CUSTOMER_NAME, R1_TITLE +from evals.seed.customers import is_evaluation_customer_name +from evals.seed.releases import EVALUATION_RELEASE_TAG_VERSION +from evals.tasks.skip import TaskSkipped +assert CUSTOMER_NAME == NeutralCustomerName == 'Acme Corp' +assert R1_TITLE == 'Payment webhook drops retries' +assert EVALUATION_RELEASE_TAG_VERSION == 'eval-rc1' +assert is_evaluation_customer_name('Acme') +assert TaskSkipped is NeutralTaskSkipped +""" + for imports in ("import evals.tasks\nimport evals.seed\n", "import evals.seed\nimport evals.tasks\n"): + result = subprocess.run( + [sys.executable, "-c", imports + assertions], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/evals/test_listing.py b/tests/evals/test_listing.py new file mode 100644 index 0000000..6786ec4 --- /dev/null +++ b/tests/evals/test_listing.py @@ -0,0 +1,44 @@ +"""Offline eval tests for listing.""" + +from __future__ import annotations + +from evals.listing import count_tool_tokens, tool_payload_model_facing, tool_payload_wire + + +def test_count_tool_tokens_fake_list(): + class T: + def __init__(self, name, desc, inp, out=None): + self.name = name + self.description = desc + self.inputSchema = inp + self.outputSchema = out + + tools = [ + T("alpha", "short", {"type": "object"}), + T( + "beta", + "longer description here", + {"type": "object", "properties": {"x": {"type": "string"}}}, + out={"type": "object"}, + ), + ] + # Fake encode: 1 token per character (deterministic, no tiktoken needed). + encode = lambda s: list(s) # noqa: E731 + rows, total_wire, total_model = count_tool_tokens(tools, encode=encode) + assert len(rows) == 2 + assert total_wire == sum(r.wire_tokens for r in rows) + assert total_model == sum(r.model_facing_tokens for r in rows) + # Tool with outputSchema has wire > model-facing. + beta = next(r for r in rows if r.name == "beta") + assert beta.has_output_schema is True + assert beta.wire_tokens > beta.model_facing_tokens + alpha = next(r for r in rows if r.name == "alpha") + assert alpha.has_output_schema is False + assert alpha.wire_tokens == alpha.model_facing_tokens + # Sorted by wire desc + assert rows[0].wire_tokens >= rows[1].wire_tokens + + wire = tool_payload_wire(tools[1]) + assert "output_schema" in wire + model = tool_payload_model_facing(tools[1]) + assert "output_schema" not in model diff --git a/tests/evals/test_package_boundaries.py b/tests/evals/test_package_boundaries.py new file mode 100644 index 0000000..f4d1aba --- /dev/null +++ b/tests/evals/test_package_boundaries.py @@ -0,0 +1,130 @@ +"""Import-closure tests for the package's layering. + +The direction of these edges is the part of the structure worth defending: offline reporting +must be usable without live-run code, fixtures without agent backends, and the recording +proxy without any of it. Every invariant here held when the tests were written, so a failure +means a new import changed the shape of the package rather than a pre-existing violation. + +Each case runs in a fresh interpreter, because import closure is a property of a process. +""" + +from __future__ import annotations + +import pathlib +import subprocess +import sys + +import pytest + +# (module to import, package names it must not drag in) +BOUNDARIES = [ + ("evals.report.load", ("runner", "drivers", "seed", "proxy")), + ("evals.report.statistics", ("runner", "drivers", "seed")), + ("evals.seed.build", ("drivers", "report", "runner")), + ("evals.proxy", ("runner", "drivers", "seed", "tasks", "report")), + ("evals.tasks", ("runner", "drivers", "report", "proxy")), + # A pure token-counting helper once imported the live runner, so importing it loaded + # every driver, seeder, task and report module in the tree. + ("evals.listing", ("runner", "drivers", "seed", "tasks", "report")), +] + + +def loaded_subpackages(module: str) -> set[str]: + """Return the ``evals.*`` subpackages present in sys.modules after importing ``module``.""" + code = ( + f"import {module}, sys\n" + "print(' '.join(sorted({m.split('.')[1] for m in sys.modules " + "if m.startswith('evals.') and m.count('.') >= 1})))" + ) + completed = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + return set(completed.stdout.split()) + + +@pytest.mark.parametrize(("module", "forbidden"), BOUNDARIES, ids=[case[0] for case in BOUNDARIES]) +def test_import_does_not_cross_layer(module: str, forbidden: tuple[str, ...]): + leaked = loaded_subpackages(module) & set(forbidden) + assert not leaked, f"importing {module} loaded {sorted(leaked)}, which it must not depend on" + + +def test_the_probe_can_actually_observe_a_violation(): + """A boundary test that cannot fail is worse than none: prove the probe sees imports.""" + assert "runner" in loaded_subpackages("evals.runner.live") + + +# The two driver surfaces are independent: the API driver owns its loop and speaks to a +# provider, while a CLI driver supervises a subprocess and reads a recording proxy. Neither +# needs anything the other has. Depth-1 names cannot express this — both live under +# ``drivers`` — so these cases match module prefixes instead. +# +# (module to import, module prefixes it must not drag in) +SURFACE_BOUNDARIES = [ + # Reading the registry must load no surface at all, or get_driver's per-vendor imports + # are decoration: a flat re-export wall here once made every consumer load all five + # agent CLIs, because Python runs a package's __init__ before any submodule. + ("evals.drivers", ("evals.drivers.api.", "evals.drivers.cli.")), + ("evals.drivers.api.base", ("evals.drivers.cli.",)), + ("evals.drivers.api.driver", ("evals.drivers.cli.",)), + ("evals.drivers.cli.base", ("evals.drivers.api.",)), + ("evals.drivers.cli.claude", ("evals.drivers.api.",)), +] + + +def loaded_modules(module: str) -> set[str]: + """Return the full names of the ``evals.*`` modules present after importing ``module``.""" + code = f"import {module}, sys\nprint(' '.join(sorted(m for m in sys.modules if m.startswith('evals'))))" + completed = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + return set(completed.stdout.split()) + + +@pytest.mark.parametrize(("module", "forbidden"), SURFACE_BOUNDARIES, ids=[case[0] for case in SURFACE_BOUNDARIES]) +def test_import_does_not_cross_driver_surface(module: str, forbidden: tuple[str, ...]): + leaked = sorted(name for name in loaded_modules(module) if name.startswith(forbidden)) + assert not leaked, f"importing {module} loaded {leaked}, which it must not depend on" + + +def test_the_surface_probe_can_actually_observe_a_violation(): + """Same guard as above, for the prefix probe: prove it sees a real intra-surface import.""" + assert "evals.drivers.cli.base" in loaded_modules("evals.drivers.cli.claude") + + +# Shared floor: modules under evals.core may import only each other (plus stdlib / +# third-party). A flat dump of helpers into core would silently reintroduce the +# invisible shared vocabulary this package exists to make visible. +# +# Membership is discovered, not listed. Naming a package after its position in the graph +# only holds if the position is checked, and a hand-maintained list makes that opt-in: a +# module dropped into core/ and left out of the list would import whatever it liked. +CORE_MODULES = tuple( + f"evals.core.{path.stem}" + for path in sorted((pathlib.Path(__file__).parents[2] / "evals" / "core").glob("*.py")) + if path.stem != "__init__" +) + + +def test_core_is_not_empty(): + """Guard the discovery above: a bad glob would make every core case vanish silently.""" + assert len(CORE_MODULES) >= 11, CORE_MODULES + + +@pytest.mark.parametrize("module", CORE_MODULES, ids=list(CORE_MODULES)) +def test_core_imports_only_core(module: str): + """Importing any core module must load no evals module outside evals.core.""" + loaded = loaded_modules(module) + leaked = sorted( + name + for name in loaded + if name.startswith("evals.") and name != "evals.core" and not name.startswith("evals.core.") + ) + assert not leaked, f"importing {module} loaded non-core evals modules: {leaked}" + + +def test_the_exception_module_is_the_floor_of_the_floor(): + """``errors`` may depend on nothing of ours at all, not even its core siblings. + + ``BOUNDARIES`` used to assert this, but it matches depth-1 package names, and once + ``results`` moved under ``core`` its depth-1 name became ``core`` — so the entry + forbidding ``results`` could no longer match anything. Asserted here at the granularity + that survives the move. + """ + siblings = sorted(name for name in loaded_modules("evals.core.errors") if name != "evals.core.errors") + assert siblings == ["evals", "evals.core"], siblings diff --git a/tests/evals/test_pricing.py b/tests/evals/test_pricing.py new file mode 100644 index 0000000..0211a1c --- /dev/null +++ b/tests/evals/test_pricing.py @@ -0,0 +1,99 @@ +"""Pricing must be right, refuse, or say it has nothing -- never quietly zero.""" + +from __future__ import annotations + +from evals.core.pricing import ( + PRICED, + PRICES_AS_OF, + UNMEASURED, + UNPRICED, + price_usage, + resolve_model_id, +) + +OPENAI_ROW = { + "input_tokens": 97813, + "output_tokens": 1121, + "cache_read_input_tokens": 83460, + "cache_creation_input_tokens": 0, + "source": "iterations", +} +CLAUDE_CLI_ROW = { + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 0.0753878, + "modelUsage": {"claude-haiku-4-5-20251001": {}}, + "source": "modelUsage", +} + + +def test_prices_carry_an_as_of_date(): + assert PRICES_AS_OF + + +def test_a_known_model_is_priced(): + cost = price_usage(OPENAI_ROW, model="gpt-5.6-luna") + assert cost.outcome == PRICED + assert cost.usd is not None + # 1.27M fresh at $0.20, 83460 cached at $0.02, 1121 out at $1.20. + assert cost.usd == round((14353 * 0.20 + 83460 * 0.02 + 1121 * 1.20) / 1e6, 10) + + +def test_an_unknown_model_is_unpriced_not_free(): + """A silent zero reads as 'free'. The whole point is that it must read as 'unknown'.""" + cost = price_usage(OPENAI_ROW, model="some-model-nobody-has-heard-of") + assert cost.outcome == UNPRICED + assert cost.usd is None + + +def test_a_row_with_no_usage_at_all_is_unmeasured(): + """antigravity records usage_total on 0 of 70 rows; that is not the same as unknown.""" + for empty in (None, {}): + cost = price_usage(empty, model="gemini-3.6-flash-low") + assert cost.outcome == UNMEASURED + assert cost.usd is None + + +def test_unmeasured_and_unpriced_are_distinguishable(): + assert UNMEASURED != UNPRICED + assert price_usage(None, model="gpt-5.6-luna").outcome != price_usage(OPENAI_ROW, model="nope").outcome + + +def test_model_id_resolves_through_model_usage_before_the_row_alias(): + """claude-cli records model 'haiku' -- an alias, unpriceable as a key.""" + assert resolve_model_id(CLAUDE_CLI_ROW, model="haiku") == "claude-haiku-4-5-20251001" + assert resolve_model_id(OPENAI_ROW, model="gpt-5.6-luna") == "gpt-5.6-luna" + assert resolve_model_id(None, model="haiku") == "haiku" + + +def test_dated_model_ids_match_their_family(): + assert price_usage(CLAUDE_CLI_ROW, model="haiku").outcome == PRICED + + +def test_computed_cost_agrees_with_the_vendor_reported_cost(): + """The only mechanism that detects a stale price table. + + Published 5-minute cache-write rates give $2.46 against this arm's reported + $3.18; the 1-hour TTL multiplier reproduces it. Without this check the table + would have shipped 23% low and looked fine. + """ + cost = price_usage(CLAUDE_CLI_ROW, model="haiku") + assert cost.vendor_usd == 0.0753878 + assert cost.usd is not None + assert abs(cost.usd - cost.vendor_usd) / cost.vendor_usd < 0.01 + + +def test_vendor_cost_is_preferred_when_present(): + cost = price_usage(CLAUDE_CLI_ROW, model="haiku") + assert cost.billed_usd == cost.vendor_usd + # ...and falls back to the computed figure when the vendor reports nothing. + assert price_usage(OPENAI_ROW, model="gpt-5.6-luna").billed_usd == price_usage(OPENAI_ROW, model="gpt-5.6-luna").usd + + +def test_undecidable_cache_semantics_is_unpriced(): + """A cached row whose model family is unknown cannot be normalised, so it cannot be priced.""" + row = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 400} + assert price_usage(row, model="mystery-model").outcome == UNPRICED diff --git a/tests/evals/test_proxy.py b/tests/evals/test_proxy.py new file mode 100644 index 0000000..f72c32f --- /dev/null +++ b/tests/evals/test_proxy.py @@ -0,0 +1,2043 @@ +"""Offline eval tests for proxy.""" + +from __future__ import annotations + +import json +import os +import signal +import stat +import subprocess +import sys +import textwrap +import time +from pathlib import Path +from typing import Any + +import pytest + +from evals.core.evidence import ( + EVIDENCE_SENTINELS_ENV, + TARGET_ENTITY_EVIDENCE, + consume_evidence_config, + write_evidence_config, +) +from evals.core.results import AgentRun, TaskResult, agent_run_to_task_result +from evals.drivers.cli.sidecar import ( + _pumps_blocking, + apply_proxy_sidecar, + ensure_proxy_pythonpath, + load_proxy_sidecar, + load_proxy_sidecar_calls, + proxy_pid_path, + proxy_session_paths, +) +from evals.proxy import ( + SHUTDOWN_DEADLINE_S, + SidecarRecorder, + map_child_returncode, + process_buffer_lines, + reap_timeout, + scrub_child_pythonpath, + write_all_fd, +) +from evals.proxy import main as proxy_main +from evals.report.summary import summarize +from evals.runner.live import _record_trace_infra +from tests.evals.conftest import case_params + +REPO = Path(__file__).resolve().parents[2] + + +def _only_proxy_session(configured_path: Path) -> Path: + sessions = proxy_session_paths(configured_path) + assert len(sessions) == 1, sessions + return sessions[0] + + +def _session_file(configured_path: Path, session_id: str) -> Path: + return configured_path.with_name(f"{configured_path.name}.{session_id}.jsonl") + + +def _request(request_id: int, method: str, params: dict | None = None) -> dict: + message = {"jsonrpc": "2.0", "id": request_id, "method": method} + if params is not None: + message["params"] = params + return message + + +def _response(request_id: int, result: object) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def _complete_proxy_meta(last_seq: int, manifest: str = "manifest-a") -> dict: + return { + "row_type": "proxy_meta", + "pending_left": 0, + "non_tool_pending_left": 0, + "unmatched_responses": 0, + "unparsed_lines": 0, + "recorder_errors": 0, + "pumps_alive": False, + "last_seq": last_seq, + "tool_request_count": last_seq, + "tool_manifest_fingerprint": manifest, + } + + +def _proxy_call(tool: str, seq: int) -> dict: + return { + "tool": tool, + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": seq, + } + + +FAKE_SERVER = textwrap.dedent( + r""" + import json, sys + + def send(obj): + sys.stdout.write(json.dumps(obj) + "\n") + sys.stdout.flush() + + for raw in sys.stdin: + raw = raw.strip() + if not raw: + continue + try: + msg = json.loads(raw) + except json.JSONDecodeError: + sys.stdout.write(raw + "\n") + sys.stdout.flush() + continue + mid = msg.get("id") + method = msg.get("method") + if method == "initialize": + send({ + "jsonrpc": "2.0", + "id": mid, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "serverInfo": {"name": "fake"}, + }, + }) + elif method == "tools/list": + send({ + "jsonrpc": "2.0", + "id": mid, + "result": {"tools": [{"name": "list_work_items", "inputSchema": {}}]}, + }) + elif method == "tools/call": + params = msg.get("params") or {} + name = params.get("name") + args = params.get("arguments") or {} + if name == "boom": + send({ + "jsonrpc": "2.0", + "id": mid, + "result": { + "content": [{"type": "text", "text": "fail"}], + "isError": True, + }, + }) + else: + body = f"ok:{name}:{json.dumps(args, sort_keys=True)}" + send({ + "jsonrpc": "2.0", + "id": mid, + "result": { + "content": [{"type": "text", "text": body}], + "isError": False, + }, + }) + elif method and mid is not None: + send({"jsonrpc": "2.0", "id": mid, "result": {}}) + sys.exit(7) + """ +).lstrip() + + +def _write_fake_server(path: Path) -> Path: + path.write_text(FAKE_SERVER, encoding="utf-8") + return path + + +def _proxy_records_tools_call_and_exit_code(tmp_path): + server = _write_fake_server(tmp_path / "fake_server.py") + sidecar = tmp_path / "side.jsonl" + cmd = [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ] + # Drive the proxy: initialize, tools/call ok, tools/call error, unparsed, then close. + client_in = ( + json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "list_work_items", "arguments": {"project": "P"}}, + } + ) + + "\n" + + json.dumps( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "boom", "arguments": {}}, + } + ) + + "\n" + + "NOT_JSON_LINE\n" + ) + proc = subprocess.run( + cmd, + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 7 # child exit propagated + session_path = _only_proxy_session(sidecar) + assert int(proxy_pid_path(session_path).read_text(encoding="ascii")) > 0 + # Byte-faithful: unparsed line and JSON responses appear on stdout. + out = proc.stdout.decode("utf-8", errors="replace") + assert "NOT_JSON_LINE" in out + assert "list_work_items" in out or "ok:list_work_items" in out + + rows = [json.loads(ln) for ln in session_path.read_text(encoding="utf-8").splitlines() if ln.strip()] + call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] + meta = next(r for r in rows if r.get("row_type") == "proxy_meta") + assert len(call_rows) == 2 + assert call_rows[0]["tool"] == "list_work_items" + assert call_rows[0]["args"] == {"project": "P"} + assert call_rows[0]["is_error"] is False + assert call_rows[0]["result_chars"] > 0 + assert call_rows[0]["seq"] == 1 + assert call_rows[1]["tool"] == "boom" + assert call_rows[1]["is_error"] is True + assert meta["unparsed_lines"] >= 1 + assert meta["relayed_lines"] >= 3 + assert meta["finalization_reason"] in {"normal_eof", "child_exit"} + assert meta["finalization_signal"] is None + + +def _proxy_byte_faithful_child_receives_exact_bytes(tmp_path): + received = tmp_path / "received.bin" + echo_server = tmp_path / "echo_server.py" + echo_server.write_text( + textwrap.dedent( + f""" + import sys + data = sys.stdin.buffer.read() + open({str(received)!r}, "wb").write(data) + # Still answer initialize-ish so proxy drains cleanly + for line in data.splitlines(keepends=True): + if not line.strip(): + continue + try: + import json + msg = json.loads(line) + except Exception: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + continue + if msg.get("id") is not None: + sys.stdout.buffer.write( + (json.dumps({{"jsonrpc": "2.0", "id": msg["id"], "result": {{}}}}) + "\\n").encode() + ) + sys.stdout.buffer.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "s.jsonl" + # Deliberately non-canonical JSON spacing — re-serialization would change it. + payload = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ "x":1}}\n' + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(echo_server), + ], + input=payload, + capture_output=True, + cwd=str(REPO), + timeout=10, + ) + assert proc.returncode == 0 + assert received.read_bytes() == payload + + +def _proxy_main_requires_command(_tmp_path): + with pytest.raises(SystemExit): + proxy_main(["--log", "/tmp/x.jsonl"]) + + +def _proxy_exits_when_child_dies_first(tmp_path): + server = tmp_path / "die_soon.py" + server.write_text( + textwrap.dedent( + """ + import sys, time + # Emit nothing and exit quickly; leave proxy client stdin open. + time.sleep(0.15) + sys.exit(3) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + t0 = __import__("time").monotonic() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=str(REPO), + ) + try: + # Keep stdin open (do not close) so the stdin pump blocks on readline; + # the proxy must still notice child death and exit. + deadline = SHUTDOWN_DEADLINE_S + 5.0 + try: + rc = proc.wait(timeout=deadline) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + pytest.fail(f"proxy hung >{deadline}s after child exit") + elapsed = __import__("time").monotonic() - t0 + # Must finish well under the hang window (not wait the full drain). + assert elapsed < deadline + # Child's exit code (3) should propagate; tolerate signal map if the + # runtime reaps oddly, but meta must still be present. + assert rc in (3, 128 + 3) or rc == 3 + assert proxy_session_paths(sidecar) + text = _only_proxy_session(sidecar).read_text(encoding="utf-8") + assert "proxy_meta" in text + # Prefer exact child code when available + if rc not in (3, 128 + 3): + # At least ensure we did not hang; surface stderr for diagnosis. + err = (proc.stderr.read() if proc.stderr else b"").decode() + assert "proxy_meta" in text, f"rc={rc} stderr={err!r}" + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + if proc.stdin: + try: + proc.stdin.close() + except Exception: + pass + + +def _proxy_from_foreign_cwd_with_pythonpath(tmp_path): + server = tmp_path / "echo_once.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {"content": [], "isError": False}, + }) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign_cwd" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(**{k: v for k, v in __import__("os").environ.items()})) + # Drop any ambient PYTHONPATH pollution by putting repo first. + assert str(REPO) in env["PYTHONPATH"].split(__import__("os").pathsep) + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "ping", "arguments": {}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), # foreign cwd — must still import evals.proxy + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + calls = load_proxy_sidecar_calls(sidecar) + assert len(calls) == 1 + assert calls[0]["tool"] == "ping" + + +def _proxy_child_env_pythonpath_clean(tmp_path): + server = tmp_path / "check_env.py" + server.write_text( + textwrap.dedent( + f""" + import json, os, sys + root = {str(REPO)!r} + pp = os.environ.get("PYTHONPATH", "") + parts = [p for p in pp.split(os.pathsep) if p] + bad = root in parts + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({{ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {{"content": [{{"type": "text", "text": "bad=" + str(bad)}}], "isError": False}}, + }}) + "\\n") + sys.stdout.flush() + sys.exit(0 if not bad else 9) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + foreign = tmp_path / "foreign" + foreign.mkdir() + env = ensure_proxy_pythonpath(dict(__import__("os").environ)) + assert str(REPO) in env.get("PYTHONPATH", "") + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "envcheck", "arguments": {}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(foreign), + env=env, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode() + assert b"bad=False" in proc.stdout + + +def _proxy_survives_cli_group_kill_and_writes_meta(tmp_path): + import os + import signal + import time + + server = tmp_path / "echo_server.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: + continue + mid = msg.get("id") + if mid is not None: + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + leader_script = tmp_path / "cli_leader.py" + leader_script.write_text( + textwrap.dedent( + f""" + import os, subprocess, sys, time + from pathlib import Path + sidecar = Path({str(sidecar)!r}) + server = Path({str(server)!r}) + # Spawn proxy in our process group (no start_new_session on child). + # proxy main() will os.setsid() and detach. + proxy = subprocess.Popen( + [ + sys.executable, "-m", "evals.proxy", + "--log", str(sidecar), + "--", + sys.executable, str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + # Give setsid a moment, then write a tools/call and keep stdin open briefly. + time.sleep(0.4) + if proxy.stdin: + req = ( + '{{"jsonrpc":"2.0","id":1,"method":"tools/call",' + '"params":{{"name":"t","arguments":{{}}}}}}\\n' + ) + proxy.stdin.write(req.encode()) + proxy.stdin.flush() + # Stay alive as group leader until killed by the test harness. + time.sleep(9999) + """ + ), + encoding="utf-8", + ) + + # Leader is a process-group leader (like run_cli_subprocess). + env = {**os.environ, "PYTHONPATH": str(REPO) + os.pathsep + os.environ.get("PYTHONPATH", "")} + leader = subprocess.Popen( + [sys.executable, str(leader_script)], + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=str(REPO), + env=env, + ) + try: + # Wait until proxy has started (sidecar created) and setsid likely done. + boot = time.monotonic() + 5.0 + while time.monotonic() < boot: + if proxy_session_paths(sidecar): + break + time.sleep(0.05) + time.sleep(0.5) # allow setsid + optional tools/call + # SIGKILL the CLI process group — must NOT kill the detached proxy. + try: + os.killpg(leader.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + leader.wait(timeout=2.0) + except subprocess.TimeoutExpired: + leader.kill() + leader.wait(timeout=1.0) + + # Proxy should see stdin EOF (leader dead → pipe closed), finalize meta. + deadline = time.monotonic() + SHUTDOWN_DEADLINE_S + 5.0 + meta_seen = False + while time.monotonic() < deadline: + if proxy_session_paths(sidecar): + text = _only_proxy_session(sidecar).read_text(encoding="utf-8") + if "proxy_meta" in text: + meta_seen = True + break + time.sleep(0.1) + assert meta_seen, ( + f"proxy_meta missing after group kill; " + f"sidecar={_only_proxy_session(sidecar).read_text() if proxy_session_paths(sidecar) else None!r}" + ) + rows = [json.loads(ln) for ln in _only_proxy_session(sidecar).read_text().splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + finally: + if leader.poll() is None: + try: + os.killpg(leader.pid, signal.SIGKILL) + except Exception: + leader.kill() + try: + leader.wait(timeout=2.0) + except Exception: + pass + + +@pytest.mark.parametrize( + "case", + case_params( + _proxy_records_tools_call_and_exit_code, + _proxy_byte_faithful_child_receives_exact_bytes, + _proxy_main_requires_command, + _proxy_exits_when_child_dies_first, + _proxy_from_foreign_cwd_with_pythonpath, + _proxy_child_env_pythonpath_clean, + _proxy_survives_cli_group_kill_and_writes_meta, + ), +) +def test_proxy_behaviours(case, tmp_path): + case(tmp_path) + + +@pytest.mark.parametrize( + "signum", + [ + pytest.param(signal.SIGTERM, id="SIGTERM"), + pytest.param(signal.SIGHUP, id="SIGHUP"), + pytest.param(signal.SIGINT, id="SIGINT"), + ], +) +def test_proxy_signal_finalization_writes_meta_and_preserves_signal_exit(tmp_path: Path, signum: int): + server = _write_fake_server(tmp_path / "signal_server.py") + sidecar = tmp_path / "signal-sidecar.jsonl" + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + cwd=str(REPO), + ) + try: + assert proc.stdin is not None + messages = [ + _request(1, "tools/list"), + _request(2, "tools/call", {"name": "list_work_items", "arguments": {"project": "P"}}), + ] + proc.stdin.write(("\n".join(json.dumps(message) for message in messages) + "\n").encode("utf-8")) + proc.stdin.flush() + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + calls, _status = load_proxy_sidecar(sidecar) + if len(calls) == 1: + break + time.sleep(0.01) + else: + pytest.fail("proxy did not record the completed tool call before signal") + + os.kill(proc.pid, signum) + returncode = proc.wait(timeout=SHUTDOWN_DEADLINE_S + 5.0) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=2.0) + if proc.stdin is not None: + proc.stdin.close() + + rows = [ + json.loads(line) + for line in _only_proxy_session(sidecar).read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert rows[-1]["row_type"] == "proxy_meta" + meta = rows[-1] + assert meta["finalization_reason"] == "signal" + assert meta["finalization_signal"] == signal.Signals(signum).name + assert returncode == -signum + + # SIGTERM/SIGHUP use the controlled drain. A completed request remains a + # complete trace; a signal is diagnostic, not intrinsically recorder loss. + if signum in (signal.SIGTERM, signal.SIGHUP): + _calls, status = load_proxy_sidecar(sidecar) + assert status["state"] == "complete" + assert status["pending_left"] == 0 + assert status["pumps_alive"] is False + assert status["tool_manifest_fingerprint"] + + +def test_signal_finalization_does_not_hide_pending_tool_loss(tmp_path: Path): + sidecar = tmp_path / "signal-pending.jsonl" + recorder = SidecarRecorder(sidecar) + recorder.finalization_reason = "signal" + recorder.finalization_signal = "SIGTERM" + recorder.on_client_message(_request(1, "tools/call", {"name": "unfinished", "arguments": {}})) + recorder.write_meta() + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], sidecar, notes) + + assert result.status["finalization_reason"] == "signal" + assert result.status["finalization_signal"] == "SIGTERM" + assert result.status["pending_left"] == 1 + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert any(note.startswith("proxy_sidecar_incomplete:pending_left=1") for note in notes) + + +def _sidecar_recorder_unit(tmp_path): + sentinel = "hidden-target-fact-7b0a1f9c" + rec = SidecarRecorder( + tmp_path / "a.jsonl", + evidence_sentinels={TARGET_ENTITY_EVIDENCE: [sentinel]}, + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 8, + "method": "tools/call", + "params": {"name": "t", "arguments": {"work_item_id": "non-target"}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 8, + "result": {"content": [{"type": "text", "text": f"target={sentinel}"}], "isError": False}, + } + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + "params": {"name": "t", "arguments": {"work_item_id": "target-1"}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 9, + "result": {"content": [{"type": "text", "text": f"target={sentinel}"}], "isError": False}, + } + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 10, + "method": "tools/call", + "params": {"name": "t", "arguments": {"work_item_id": "target-1"}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 10, + "result": {"content": [{"type": "text", "text": "no seeded value here"}], "isError": False}, + } + ) + rec.write_meta() + calls = load_proxy_sidecar_calls(tmp_path / "a.jsonl") + raw_rows = [json.loads(line) for line in (tmp_path / "a.jsonl").read_text().splitlines()] + raw_calls = [row for row in raw_rows if row.get("row_type") != "proxy_meta"] + assert len(calls) == 3 + assert calls[0]["tool"] == "t" + assert calls[0]["args"] == {"work_item_id": "non-target"} + assert calls[1]["args"] == {"work_item_id": "target-1"} + assert calls[0]["origin"] == "plane" + assert "result_text" not in calls[0] + assert all("result_text" not in row for row in raw_calls) + # A sentinel is a per-run random string that exists only inside Plane, so its + # presence proves the response came from the surface whichever entity the request + # named. Call 0 named an unrelated entity and is still evidence; call 2 named the + # seeded one but never received the value, and is not. + assert calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert calls[2]["observed_sentinels"] == [] + assert raw_calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert raw_calls[1]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert raw_calls[2]["observed_sentinels"] == [] + assert sentinel not in (tmp_path / "a.jsonl").read_text(encoding="utf-8") + assert rec.finalized is True + + +def test_proxy_records_exact_target_bound_aggregate_evidence_without_payload(tmp_path): + path = tmp_path / "aggregate.jsonl" + rec = SidecarRecorder( + path, + evidence_targets={TARGET_ENTITY_EVIDENCE: ["project-1"]}, + evidence_aggregates={ + TARGET_ENTITY_EVIDENCE: [{"kind": "total_count"}], + }, + ) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "count_work_items", + "arguments": {"pql": 'project = "project-1"'}, + }, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 1, + "result": {"content": [{"type": "text", "text": '{"total_count": 4}'}]}, + } + ) + # A count is guessable, so it is only evidence from a request naming a seeded entity. + # Without this case, dropping the target check from proxy wiring left every CLI test green. + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "count_work_items", "arguments": {"pql": 'project = "project-other"'}}, + } + ) + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 2, + "result": {"content": [{"type": "text", "text": '{"total_count": 4}'}]}, + } + ) + rec.write_meta() + + calls = load_proxy_sidecar_calls(path) + assert calls[0]["observed_sentinels"] == [] + assert calls[0]["observed_aggregates"] == [{"label": TARGET_ENTITY_EVIDENCE, "kind": "total_count", "value": 4}] + assert calls[1]["observed_aggregates"] == [] + persisted = path.read_text(encoding="utf-8") + assert "result_text" not in persisted + assert '"content"' not in persisted + + +def _sidecar_result_payload_round_trips_only_when_enabled(tmp_path): + path = tmp_path / "payload.jsonl" + rec = SidecarRecorder(path, record_result_payloads=True) + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "find_work_items", "arguments": {}}, + } + ) + result = {"content": [{"type": "text", "text": "workspace result"}], "isError": False} + rec.on_server_message({"jsonrpc": "2.0", "id": 3, "result": result}) + rec.write_meta() + + expected_text = json.dumps(result, default=str, ensure_ascii=False) + raw_call = next( + row + for row in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) + if row.get("row_type") != "proxy_meta" + ) + assert raw_call["result_text"] == expected_text + calls = load_proxy_sidecar_calls(path) + assert calls[0]["result_text"] == expected_text + assert calls[0]["result_chars"] == len(expected_text) + + +@pytest.mark.parametrize( + "case", + case_params(_sidecar_recorder_unit, _sidecar_result_payload_round_trips_only_when_enabled), +) +def test_sidecar_behaviours(case, tmp_path): + case(tmp_path) + + +def test_append_after_finalize_is_dropped(tmp_path: Path): + """Once write_meta seals the sidecar, further row appends no-op (meta stays last).""" + rec = SidecarRecorder(tmp_path / "fin.jsonl") + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "before", "arguments": {}}, + } + ) + rec.on_server_message({"jsonrpc": "2.0", "id": 1, "result": {"ok": True}}) + rec.write_meta() + assert rec.finalized is True + assert rec.post_finalize_appends == 0 + + # Late pump activity after meta — must not write another call row. + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "after", "arguments": {}}, + } + ) + rec.on_server_message({"jsonrpc": "2.0", "id": 2, "result": {"ok": True}}) + rec._append({"tool": "ghost", "args": {}, "seq": 99}) # noqa: SLF001 + rec.write_meta() # second meta attempt also dropped + + assert rec.post_finalize_appends >= 2 + text = (tmp_path / "fin.jsonl").read_text(encoding="utf-8") + rows = [json.loads(ln) for ln in text.splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + call_tools = [r["tool"] for r in rows if r.get("row_type") != "proxy_meta"] + assert call_tools == ["before"] + assert "after" not in call_tools + assert "ghost" not in call_tools + assert text.count("proxy_meta") == 1 + + +def test_pumps_alive_meta_classified_incomplete(tmp_path: Path): + """proxy_meta with pumps_alive=true is incomplete (same as pending_left>0).""" + p = tmp_path / "s.jsonl" + rows = [ + { + "tool": "from_proxy", + "args": {}, + "is_error": False, + "result_chars": 1, + "duration_ms": 1, + "seq": 1, + }, + { + "row_type": "proxy_meta", + "relayed_lines": 1, + "unparsed_lines": 0, + "unmatched_responses": 0, + "notifications": 0, + "pending_left": 0, + "child_killed": False, + "pumps_alive": True, + }, + ] + p.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8") + calls, status = load_proxy_sidecar(p) + assert status["state"] == "incomplete" + assert status.get("pumps_alive") is True + assert len(calls) == 1 + + cli = [ + {"tool": "c1", "args": {}, "origin": "plane"}, + {"tool": "c2", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + out, _client, src = apply_proxy_sidecar(cli, [], p, notes) + assert src != "proxy" + assert [c["tool"] for c in out] == ["c1", "c2"] + assert any("proxy_sidecar_incomplete" in n and "pumps_alive" in n for n in notes) + assert any("deferred_to_cli" in n for n in notes) + + +def test_reap_timeout_floor_when_deadline_exhausted(): + """Kill/reap waits use remaining budget with a non-zero floor.""" + past = __import__("time").monotonic() - 10.0 + assert reap_timeout(past, floor=0.1) == 0.1 + assert reap_timeout(None, floor=0.1) == 0.1 + future = __import__("time").monotonic() + 5.0 + assert reap_timeout(future, floor=0.1) >= 4.0 + + +def test_server_initiated_request_does_not_pop_pending(tmp_path: Path): + """Server message with method+id must not complete a tools/call pending slot.""" + rec = SidecarRecorder(tmp_path / "s.jsonl") + rec.on_client_message( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "list_work_items", "arguments": {}}, + } + ) + # Server-initiated request reusing id=1 (roots/list style). + rec.on_server_message({"jsonrpc": "2.0", "id": 1, "method": "roots/list", "params": {}}) + assert rec.server_requests == 1 + # Real response for the tools/call should still match. + rec.on_server_message( + { + "jsonrpc": "2.0", + "id": 1, + "result": {"content": [{"type": "text", "text": "ok"}], "isError": False}, + } + ) + rec.write_meta() + calls = load_proxy_sidecar_calls(tmp_path / "s.jsonl") + assert len(calls) == 1 + assert calls[0]["tool"] == "list_work_items" + assert calls[0]["is_error"] is False + + +def test_client_response_to_server_request_ignored(tmp_path: Path): + rec = SidecarRecorder(tmp_path / "s.jsonl") + # Client answers a server request — no method, has id. + rec.on_client_message({"jsonrpc": "2.0", "id": 99, "result": {"roots": []}}) + assert rec._pending == {} # noqa: SLF001 — intentional: no pending opened + rec.write_meta() + assert load_proxy_sidecar_calls(tmp_path / "s.jsonl") == [] + + +def test_map_child_returncode_signal(): + assert map_child_returncode(0) == 0 + assert map_child_returncode(1) == 1 + assert map_child_returncode(-9) == 128 + 9 + assert map_child_returncode(-15) == 128 + 15 + assert map_child_returncode(None) == 1 + + +def test_write_all_fd_loops_on_short_writes(tmp_path: Path): + """write_all_fd must loop until all bytes are written (simulate via pipe).""" + import os + + r, w = os.pipe() + payload = b"abcdefghijklmnopqrstuvwxyz" * 100 + # Write in parent; read in same process after. + write_all_fd(w, payload) + os.close(w) + got = b"" + while True: + chunk = os.read(r, 64) + if not chunk: + break + got += chunk + os.close(r) + assert got == payload + + +def test_process_buffer_partial_line_and_multi_line_chunk(tmp_path: Path): + """Partial line stays buffered; two lines in one chunk both process.""" + import os + + rec = SidecarRecorder(tmp_path / "s.jsonl") + r, w = os.pipe() + buf = bytearray() + # Partial line without newline — stays buffered (no tools/call pending yet). + buf.extend(b'{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"a","arguments":{}}') + process_buffer_lines(buf, forward_fd=w, recorder=rec, is_client=True, record_jsonrpc=True) + assert len(buf) > 0 and b"\n" not in buf + assert rec._pending == {} # noqa: SLF001 + + # Complete first line + second full line in one extend (multi-line prefetch). + buf.extend(b'}\n{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"b","arguments":{}}}\n') + process_buffer_lines(buf, forward_fd=w, recorder=rec, is_client=True, record_jsonrpc=True) + assert len(buf) == 0 + assert 1 in rec._pending and 2 in rec._pending # noqa: SLF001 + os.close(w) + while os.read(r, 65536): + pass + os.close(r) + + +def test_child_exit_drains_final_response(tmp_path: Path): + """Child writes a final tools/call response then exits immediately — must record it.""" + server = tmp_path / "final_then_exit.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", + "id": msg["id"], + "result": {"content": [{"type": "text", "text": "final"}], "isError": False}, + }) + "\\n") + sys.stdout.flush() + # Exit immediately after writing final response. + sys.exit(0) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 7, + "method": "tools/call", + "params": {"name": "final_tool", "arguments": {"x": 1}}, + } + ) + + "\n" + ) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 0 + assert b"final" in proc.stdout + calls = load_proxy_sidecar_calls(sidecar) + assert len(calls) == 1 + assert calls[0]["tool"] == "final_tool" + assert calls[0]["is_error"] is False + + +def test_scrub_child_pythonpath_removes_repo(): + import os + + env = { + "PYTHONPATH": f"{REPO}{os.pathsep}/other/lib", + "FOO": "1", + EVIDENCE_SENTINELS_ENV: '{"target":["secret"]}', + } + scrubbed = scrub_child_pythonpath(env) + assert "/other/lib" in scrubbed["PYTHONPATH"] + assert str(REPO) not in scrubbed["PYTHONPATH"].split(os.pathsep) + assert EVIDENCE_SENTINELS_ENV not in scrubbed + # Only-repo entry drops the var entirely + only = scrub_child_pythonpath({"PYTHONPATH": str(REPO)}) + assert "PYTHONPATH" not in only + + +def test_proxy_loads_reusable_private_evidence_file_before_starting_mcp(tmp_path: Path, monkeypatch): + sentinel = "hidden-target-fact-7b0a1f9c" + total_count = 918273 + grouped_counts = {"project-1": 564738, "project-2": 102938} + evidence_file = tmp_path / "evidence.json" + write_evidence_config( + evidence_file, + {TARGET_ENTITY_EVIDENCE: [sentinel]}, + {TARGET_ENTITY_EVIDENCE: ["target-1", *grouped_counts]}, + { + TARGET_ENTITY_EVIDENCE: [ + {"kind": "total_count", "value": total_count}, + {"kind": "grouped_counts", "values": grouped_counts}, + ] + }, + ) + assert stat.S_IMODE(evidence_file.stat().st_mode) == 0o600 + evidence_payload = evidence_file.read_text(encoding="utf-8") + assert sentinel not in evidence_payload + assert str(total_count) not in evidence_payload + assert all(str(count) not in evidence_payload for count in grouped_counts.values()) + captured = {} + + def fake_run_proxy(command, log_path, **kwargs): + assert evidence_file.is_file(), "run-scoped evidence must remain for later proxy sessions" + captured.update({"command": command, "log_path": log_path, **kwargs}) + return 0 + + monkeypatch.setattr("evals.proxy.os.setsid", lambda: (_ for _ in ()).throw(OSError())) + monkeypatch.setattr("evals.proxy.run_proxy", fake_run_proxy) + rc = proxy_main( + [ + "--log", + str(tmp_path / "calls.jsonl"), + "--evidence-file", + str(evidence_file), + "--", + "fake-plane-mcp", + ] + ) + + assert rc == 0 + assert evidence_file.is_file() + evidence_payload = evidence_file.read_text(encoding="utf-8") + assert sentinel not in evidence_payload + assert str(total_count) not in evidence_payload + assert all(str(count) not in evidence_payload for count in grouped_counts.values()) + assert set(captured["evidence_fingerprints"]) == {TARGET_ENTITY_EVIDENCE} + assert captured["evidence_targets"] == {TARGET_ENTITY_EVIDENCE: ("target-1", "project-1", "project-2")} + assert captured["evidence_aggregates"] == { + TARGET_ENTITY_EVIDENCE: ({"kind": "total_count"}, {"kind": "grouped_counts"}) + } + + +def test_probe_then_session_reuses_evidence_config_and_records_provenance(tmp_path: Path): + sentinel = "hidden-target-fact-7b0a1f9c" + evidence_file = tmp_path / "evidence.json" + write_evidence_config( + evidence_file, + {TARGET_ENTITY_EVIDENCE: [sentinel]}, + {TARGET_ENTITY_EVIDENCE: ["target-1"]}, + ) + assert stat.S_IMODE(evidence_file.stat().st_mode) == 0o600 + assert sentinel not in evidence_file.read_text(encoding="utf-8") + server = tmp_path / "evidence_server.py" + server.write_text( + textwrap.dedent( + f""" + import json, sys + sentinel = {sentinel!r} + for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + if method == "tools/list": + result = {{"tools": [{{"name": "read_target", "inputSchema": {{"type": "object"}}}}]}} + elif method == "tools/call": + result = {{ + "content": [{{"type": "text", "text": f"target={{sentinel}}"}}], + "isError": False, + }} + else: + result = {{}} + sys.stdout.write(json.dumps({{"jsonrpc": "2.0", "id": message["id"], "result": result}}) + "\\n") + sys.stdout.flush() + """ + ), + encoding="utf-8", + ) + + def run_session(sidecar: Path, request: dict) -> tuple[list[dict], dict]: + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--evidence-file", + str(evidence_file), + "--", + sys.executable, + str(server), + ], + input=(json.dumps(request) + "\n").encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 0, proc.stderr.decode("utf-8", errors="replace") + return load_proxy_sidecar(sidecar) + + probe_calls, probe_status = run_session(tmp_path / "probe.jsonl", _request(1, "tools/list")) + assert probe_calls == [] + assert probe_status["state"] == "complete" + assert probe_status["meta"]["evidence_trace_available"] is True + assert evidence_file.is_file() + assert sentinel not in evidence_file.read_text(encoding="utf-8") + + session_calls, session_status = run_session( + tmp_path / "session.jsonl", + _request( + 2, + "tools/call", + {"name": "read_target", "arguments": {"work_item_id": "target-1"}}, + ), + ) + assert session_status["state"] == "complete" + assert session_status["meta"]["evidence_trace_available"] is True + assert len(session_calls) == 1 + assert session_calls[0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert evidence_file.is_file() + assert sentinel not in evidence_file.read_text(encoding="utf-8") + + +def test_two_real_proxies_sharing_configured_path_preserve_both_complete_sessions(tmp_path: Path): + server = _write_fake_server(tmp_path / "two_session_server.py") + configured_sidecar = tmp_path / "shared-sidecar.jsonl" + + for request_id, tool in ((1, "first_session_call"), (2, "second_session_call")): + request = _request(request_id, "tools/call", {"name": tool, "arguments": {"session": request_id}}) + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(configured_sidecar), + "--", + sys.executable, + str(server), + ], + input=(json.dumps(request) + "\n").encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=15, + ) + assert proc.returncode == 7, proc.stderr.decode("utf-8", errors="replace") + + calls, status = load_proxy_sidecar(configured_sidecar) + + assert status["state"] == "complete" + assert status["session_file_count"] == 2 + assert status["session_count"] == 2 + assert status["all_sessions_finalized"] is True + assert all(session["finalized"] is True for session in status["sessions"]) + assert all(session["state"] == "complete" for session in status["sessions"]) + assert {call["tool"] for call in calls} == {"first_session_call", "second_session_call"} + + +def test_missing_or_malformed_evidence_config_fails_closed(tmp_path: Path): + empty = ({}, {}, {}) + assert consume_evidence_config(tmp_path / "missing.json") == empty + + malformed = tmp_path / "malformed.json" + malformed.write_text("{not-json", encoding="utf-8") + assert consume_evidence_config(malformed) == empty + assert malformed.is_file() + + +def test_rapid_response_pairing(tmp_path: Path): + """Record-before-forward: fast child responses must pair with requests (no unmatched). + + Real subprocess child replies instantly; many iterations stress the race where + a response could land on the stdout pump before _pending[id] was registered. + """ + server = tmp_path / "instant_reply.py" + server.write_text( + textwrap.dedent( + """ + import json, sys + for line in sys.stdin.buffer: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except Exception: + continue + mid = msg.get("id") + if mid is None: + continue + # Instant reply — no sleep — maximize race window. + sys.stdout.buffer.write( + (json.dumps({ + "jsonrpc": "2.0", + "id": mid, + "result": {"content": [{"type": "text", "text": "ok"}], "isError": False}, + }) + "\\n").encode() + ) + sys.stdout.buffer.flush() + """ + ), + encoding="utf-8", + ) + n = 40 + lines = [] + for i in range(1, n + 1): + lines.append( + json.dumps( + { + "jsonrpc": "2.0", + "id": i, + "method": "tools/call", + "params": {"name": f"tool_{i}", "arguments": {"i": i}}, + } + ) + ) + client_in = "\n".join(lines) + "\n" + sidecar = tmp_path / "side.jsonl" + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=30, + ) + assert proc.returncode == 0, proc.stderr.decode() + rows = [ + json.loads(ln) for ln in _only_proxy_session(sidecar).read_text(encoding="utf-8").splitlines() if ln.strip() + ] + call_rows = [r for r in rows if r.get("row_type") != "proxy_meta"] + meta = next(r for r in rows if r.get("row_type") == "proxy_meta") + assert rows[-1].get("row_type") == "proxy_meta" + assert len(call_rows) == n, f"paired {len(call_rows)}/{n}; meta={meta}" + assert meta.get("unmatched_responses", 0) == 0 + assert meta.get("pending_left", 0) == 0 + tools = {r["tool"] for r in call_rows} + assert tools == {f"tool_{i}" for i in range(1, n + 1)} + + +def test_meta_is_last_row_after_forced_kill(tmp_path: Path): + """After forced child kill, proxy_meta is the last sidecar row.""" + import time as time_mod + + server = tmp_path / "hang.py" + server.write_text( + textwrap.dedent( + """ + import sys, time + # Read one line (so proxy has something to record) then hang forever. + line = sys.stdin.buffer.readline() + if line: + import json + try: + msg = json.loads(line) + mid = msg.get("id") + if mid is not None: + sys.stdout.buffer.write( + (json.dumps({"jsonrpc": "2.0", "id": mid, "result": {}}) + "\\n").encode() + ) + sys.stdout.buffer.flush() + except Exception: + pass + while True: + time.sleep(60) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "hang_tool", "arguments": {}}, + } + ) + + "\n" + ) + # Close stdin after one request so proxy enters shutdown while child hangs + # → kill path under SHUTDOWN_DEADLINE_S. + t0 = time_mod.monotonic() + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode("utf-8"), + capture_output=True, + cwd=str(REPO), + timeout=SHUTDOWN_DEADLINE_S + 15, + ) + elapsed = time_mod.monotonic() - t0 + assert proxy_session_paths(sidecar) + rows = [ + json.loads(ln) for ln in _only_proxy_session(sidecar).read_text(encoding="utf-8").splitlines() if ln.strip() + ] + assert rows, "sidecar empty" + assert rows[-1].get("row_type") == "proxy_meta" + meta = rows[-1] + # Child was hung; kill path should have fired (or child reaped after kill). + assert meta.get("child_killed") is True or proc.returncode != 0 + # Wall clock bounded by deadline (+ small slack for process startup). + assert elapsed < SHUTDOWN_DEADLINE_S + 5.0 + + +def test_bounded_shutdown_wall_clock(tmp_path: Path): + """Shutdown after child death stays within SHUTDOWN_DEADLINE_S (+ small slack).""" + import time as time_mod + + server = tmp_path / "die_after_reply.py" + server.write_text( + textwrap.dedent( + """ + import json, sys, time + line = sys.stdin.readline() + msg = json.loads(line) + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", "id": msg["id"], + "result": {"content": [], "isError": False}, + }) + "\\n") + sys.stdout.flush() + sys.exit(0) + """ + ), + encoding="utf-8", + ) + sidecar = tmp_path / "side.jsonl" + client_in = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + "params": {"name": "quick", "arguments": {}}, + } + ) + + "\n" + ) + t0 = time_mod.monotonic() + proc = subprocess.run( + [ + sys.executable, + "-m", + "evals.proxy", + "--log", + str(sidecar), + "--", + sys.executable, + str(server), + ], + input=client_in.encode(), + capture_output=True, + cwd=str(REPO), + timeout=SHUTDOWN_DEADLINE_S + 5, + ) + elapsed = time_mod.monotonic() - t0 + assert proc.returncode == 0 + assert elapsed < SHUTDOWN_DEADLINE_S + 2.0 + rows = [json.loads(ln) for ln in _only_proxy_session(sidecar).read_text().splitlines() if ln.strip()] + assert rows[-1].get("row_type") == "proxy_meta" + + +def test_clean_protocol_session_is_complete_and_has_no_unmatched_responses(tmp_path: Path): + path = tmp_path / "clean.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(1, "initialize")) + recorder.on_server_message(_response(1, {"protocolVersion": "2025-06-18"})) + recorder.on_client_message(_request(2, "tools/list")) + recorder.on_server_message(_response(2, {"tools": [{"name": "lookup", "inputSchema": {"type": "object"}}]})) + recorder.on_client_message(_request(3, "tools/call", {"name": "lookup", "arguments": {"query": "x"}})) + recorder.on_server_message(_response(3, {"content": [], "isError": False})) + recorder.write_meta() + + calls, status = load_proxy_sidecar(path) + notes: list[str] = [] + applied = apply_proxy_sidecar([], [], path, notes) + agent = agent_run_to_task_result( + AgentRun( + calls=applied.calls, + final_text="done", + usage=None, + stopped_reason="end_turn", + trace_integrity=applied.trace_integrity, + trace_integrity_reason=applied.trace_integrity_reason, + ) + ) + row = TaskResult(task_id="R1", expected_rows=1, success=True) + row.apply_agent_result(agent) + + assert status["state"] == "complete" + assert status["meta"]["unmatched_responses"] == 0 + assert status["meta"]["non_tool_responses"] == 2 + assert status["meta"]["non_tool_pending_left"] == 0 + assert status["tool_manifest_fingerprint"] + assert [call["tool"] for call in calls] == ["lookup"] + assert summarize([row], expected_rows=1).complete is True + + +def test_genuinely_lossy_trace_makes_summary_incomplete(tmp_path: Path, capsys): + path = tmp_path / "lossy.jsonl" + recorder = SidecarRecorder(path) + recorder.on_server_message(_response(404, {"content": []})) + recorder.write_meta() + notes: list[str] = [] + + applied = apply_proxy_sidecar([], [], path, notes) + agent = agent_run_to_task_result( + AgentRun( + calls=[], + final_text="done", + usage=None, + stopped_reason="end_turn", + trace_integrity=applied.trace_integrity, + trace_integrity_reason=applied.trace_integrity_reason, + notes=notes, + ) + ) + row = TaskResult(task_id="R1", expected_rows=1) + row.apply_agent_result(agent) + assert _record_trace_infra(row, agent, task={"id": "R1"}, repetition=0) is True + + summary = summarize([row], expected_rows=1) + assert row.error_class == "infra_trace" + assert summary.infra_errors == 1 + assert summary.complete is False + assert "unmatched_responses=1" in row.error + capsys.readouterr() + + +def test_lost_tools_list_response_is_non_tool_pending_loss(tmp_path: Path): + path = tmp_path / "lost-list.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(1, "tools/list")) + recorder.write_meta() + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["non_tool_pending_left"] == 1 + assert "non_tool_pending_left=1" in notes[0] + + +def test_deleting_final_call_row_is_caught_by_last_seq(tmp_path: Path): + path = tmp_path / "deleted-final.jsonl" + recorder = SidecarRecorder(path) + for request_id in (1, 2): + recorder.on_client_message(_request(request_id, "tools/call", {"name": f"tool-{request_id}", "arguments": {}})) + recorder.on_server_message(_response(request_id, {"content": []})) + recorder.write_meta() + rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + path.write_text( + "\n".join(json.dumps(row) for row in rows if row.get("seq") != 2) + "\n", + encoding="utf-8", + ) + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["missing_seq"] == 1 + assert "missing_seq=1" in notes[0] + + +@pytest.mark.parametrize( + ("sequences", "last_seq", "status_key"), + [ + pytest.param([0], 0, "invalid_seq", id="nonpositive"), + pytest.param([1, 1], 1, "duplicate_seq", id="duplicate"), + pytest.param([1, 3], 3, "missing_seq", id="gap"), + pytest.param([1, 2], 1, "unexpected_seq", id="past-last-seq"), + ], +) +def test_sidecar_rejects_invalid_duplicate_and_gapped_sequences( + tmp_path: Path, + sequences: list[int], + last_seq: int, + status_key: str, +): + path = tmp_path / f"{status_key}.jsonl" + rows = [{"tool": "t", "args": {}, "seq": seq} for seq in sequences] + rows.append( + { + "row_type": "proxy_meta", + "pending_left": 0, + "last_seq": last_seq, + "tool_request_count": last_seq, + } + ) + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status[status_key] > 0 + assert f"{status_key}={status[status_key]}" in notes[0] + + +def test_zero_call_probe_then_real_session_is_complete(tmp_path: Path): + path = tmp_path / "probe-then-real.jsonl" + manifest = "31c209e40544" + _session_file(path, "probe").write_text( + json.dumps(_complete_proxy_meta(0, manifest)) + "\n", + encoding="utf-8", + ) + rows = [ + _proxy_call("list_projects", 1), + _proxy_call("search_work_items", 2), + _proxy_call("create_work_log", 3), + _complete_proxy_meta(3, manifest), + ] + _session_file(path, "real").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes) + + assert result.status["state"] == "complete" + assert result.status["session_count"] == 2 + assert result.status["proxy_meta_count"] == 2 + assert result.status["duplicate_seq"] == 0 + assert [call["tool"] for call in result.calls] == [ + "list_projects", + "search_work_items", + "create_work_log", + ] + assert result.trace_integrity is True + assert result.tool_manifest_fingerprint == manifest + assert not any("incomplete" in note for note in notes) + + +def test_zero_proxy_session_files_is_recorder_loss(tmp_path: Path): + path = tmp_path / "never-created.jsonl" + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes, max_wait_s=0) + + assert result.status["state"] == "missing" + assert result.status["session_file_count"] == 0 + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + + +def test_single_session_file_rejects_multiple_final_metadata_rows(tmp_path: Path): + path = tmp_path / "duplicate-meta.jsonl" + session = _session_file(path, "one") + session.write_text( + "\n".join(json.dumps(_complete_proxy_meta(0)) for _ in range(2)) + "\n", + encoding="utf-8", + ) + + _, status = load_proxy_sidecar(path) + + assert status["state"] == "incomplete" + assert status["session_file_count"] == 1 + assert status["sessions"][0]["meta_count"] == 2 + assert status["sessions"][0]["finalized"] is False + + +def test_merged_evidence_requires_every_session_to_report_available(tmp_path: Path): + path = tmp_path / "mixed-evidence-availability.jsonl" + unavailable = _complete_proxy_meta(0) + unavailable["evidence_trace_available"] = False + available = _complete_proxy_meta(0) + available["evidence_trace_available"] = True + _session_file(path, "one").write_text(json.dumps(unavailable) + "\n", encoding="utf-8") + _session_file(path, "two").write_text(json.dumps(available) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + + assert status["state"] == "complete" + assert status["all_sessions_finalized"] is True + assert status["evidence_trace_available"] is False + + +def test_two_calling_sessions_validate_sequences_independently(tmp_path: Path): + path = tmp_path / "two-calling-sessions.jsonl" + first_rows = [ + _proxy_call("session-1-call-2", 2), + _proxy_call("session-1-call-1", 1), + _complete_proxy_meta(2), + ] + second_rows = [ + _proxy_call("session-2-call-3", 3), + _proxy_call("session-2-call-1", 1), + _proxy_call("session-2-call-2", 2), + _complete_proxy_meta(3), + ] + _session_file(path, "one").write_text( + "\n".join(json.dumps(row) for row in first_rows) + "\n", + encoding="utf-8", + ) + _session_file(path, "two").write_text( + "\n".join(json.dumps(row) for row in second_rows) + "\n", + encoding="utf-8", + ) + + calls, status = load_proxy_sidecar(path) + + assert status["state"] == "complete" + assert status["duplicate_seq"] == 0 + assert status["missing_seq"] == 0 + assert status["unexpected_seq"] == 0 + assert [call["tool"] for call in calls] == [ + "session-1-call-1", + "session-1-call-2", + "session-2-call-1", + "session-2-call-2", + "session-2-call-3", + ] + assert [call["seq"] for call in calls] == [1, 2, 1, 2, 3] + + +def test_trailing_unfinalized_proxy_session_stays_fatal(tmp_path: Path): + path = tmp_path / "trailing-unfinalized.jsonl" + rows = [_complete_proxy_meta(0), _proxy_call("late", 1)] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes) + + assert result.status["state"] == "incomplete" + assert result.status["proxy_meta_count"] == 1 + assert result.status["unfinalized_sessions"] == 1 + assert result.status["proxy_meta_not_final"] is True + assert result.trace_integrity is False + assert result.trace_integrity_reason == "recorder_loss" + assert any("unfinalized_sessions=1" in note for note in notes) + + +def test_disagreeing_session_manifests_are_reported_and_invalidated(tmp_path: Path): + path = tmp_path / "manifest-disagreement.jsonl" + _session_file(path, "one").write_text(json.dumps(_complete_proxy_meta(0, "manifest-a")) + "\n") + _session_file(path, "two").write_text(json.dumps(_complete_proxy_meta(0, "manifest-b")) + "\n") + + notes: list[str] = [] + result = apply_proxy_sidecar([], [], path, notes) + + assert result.status["state"] == "complete" + assert result.status["tool_manifest_disagreement"] is True + assert result.status["tool_manifest_fingerprints"] == ["manifest-a", "manifest-b"] + assert result.trace_integrity is True + assert result.tool_manifest_fingerprint is None + assert "proxy_tool_manifest_disagreement:manifest-a,manifest-b" in notes + + +@pytest.mark.parametrize("case", ["missing-last-seq", "request-count-mismatch"]) +def test_sidecar_requires_consistent_sequence_metadata(tmp_path: Path, case: str): + path = tmp_path / f"sequence-meta-{case}.jsonl" + meta = {"row_type": "proxy_meta", "pending_left": 0, "tool_request_count": 0} + if case == "request-count-mismatch": + meta.update({"last_seq": 0, "tool_request_count": 1}) + path.write_text(json.dumps(meta) + "\n", encoding="utf-8") + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["invalid_meta_fields"] > 0 + assert f"invalid_meta_fields={status['invalid_meta_fields']}" in notes[0] + + +def test_protocol_noise_and_malformed_jsonrpc_are_distinct_and_fatal(tmp_path: Path): + path = tmp_path / "protocol.jsonl" + recorder = SidecarRecorder(path) + read_fd, write_fd = os.pipe() + try: + buf = bytearray( + b' \nserver banner\n{"jsonrpc":"2.0"}\n{"jsonrpc":"2.0","method":"notifications/initialized"}\n' + ) + process_buffer_lines( + buf, + forward_fd=write_fd, + recorder=recorder, + is_client=False, + record_jsonrpc=True, + ) + finally: + os.close(write_fd) + os.close(read_fd) + recorder.write_meta() + notes: list[str] = [] + applied = apply_proxy_sidecar([], [], path, notes) + + assert applied.trace_integrity is False + assert applied.trace_integrity_reason == "protocol_violation" + assert applied.status["unparsed_lines"] == 2 + assert applied.status["non_json_lines"] == 1 + assert applied.status["malformed_jsonrpc"] == 1 + assert "unparsed_lines=2" in notes[0] + agent = agent_run_to_task_result( + AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + trace_integrity=applied.trace_integrity, + trace_integrity_reason=applied.trace_integrity_reason, + notes=notes, + ) + ) + row = TaskResult(task_id="R1") + row.apply_agent_result(agent) + assert _record_trace_infra(row, agent, task={"id": "R1"}, repetition=0) is True + assert row.error_class == "infra_protocol" + + +def test_recorder_callback_failure_is_counted_and_invalidates_trace(tmp_path: Path, monkeypatch): + path = tmp_path / "recorder-error.jsonl" + recorder = SidecarRecorder(path) + + def fail(_obj): + raise RuntimeError("append failed") + + monkeypatch.setattr(recorder, "on_client_message", fail) + read_fd, write_fd = os.pipe() + try: + process_buffer_lines( + bytearray(b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n'), + forward_fd=write_fd, + recorder=recorder, + is_client=True, + record_jsonrpc=True, + ) + finally: + os.close(write_fd) + os.close(read_fd) + recorder.write_meta() + + _, status = load_proxy_sidecar(path) + notes: list[str] = [] + apply_proxy_sidecar([], [], path, notes) + + assert status["state"] == "incomplete" + assert status["recorder_errors"] == 1 + assert "recorder_errors=1" in notes[0] + + +def test_tools_list_changed_invalidates_proxy_manifest_snapshot(tmp_path: Path): + path = tmp_path / "stale-manifest.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(1, "tools/list")) + recorder.on_server_message(_response(1, {"tools": [{"name": "lookup", "inputSchema": {"type": "object"}}]})) + recorder.on_server_message({"jsonrpc": "2.0", "method": "notifications/tools/list_changed"}) + recorder.write_meta() + + _, status = load_proxy_sidecar(path) + + assert status["state"] == "complete" + assert status["tool_manifest_fingerprint"] is None + + +def test_cli_fallback_does_not_restore_trace_integrity(tmp_path: Path): + path = tmp_path / "fallback.jsonl" + recorder = SidecarRecorder(path) + recorder.on_client_message(_request(0, "tools/list")) + recorder.on_server_message(_response(0, {"tools": [{"name": "proxy-only", "inputSchema": {}}]})) + recorder.on_client_message(_request(1, "tools/call", {"name": "proxy-only", "arguments": {}})) + recorder.write_meta() + cli_calls = [ + {"tool": "cli-one", "args": {}, "origin": "plane"}, + {"tool": "cli-two", "args": {}, "origin": "plane"}, + ] + notes: list[str] = [] + + applied = apply_proxy_sidecar(cli_calls, [], path, notes) + calls, _, source = applied + + assert source == "json" + assert calls == cli_calls + assert applied.trace_integrity is False + assert applied.trace_integrity_reason == "recorder_loss" + assert applied.tool_manifest_fingerprint is None + assert "proxy_sidecar_deferred_to_cli_trace" in notes + + +def test_a_response_the_agent_never_received_is_not_authoritative_evidence(tmp_path: Path): + """Recording happens before forwarding, so a broken pipe can match what nobody saw. + + The ordering is deliberate — a fast child must not race an unregistered pending id — but + it meant a sentinel or count from a response that failed to reach the agent stayed in a + sidecar still marked complete, proving surface use the agent never had. + """ + recorder = SidecarRecorder(tmp_path / "undelivered.jsonl") + read_fd, write_fd = os.pipe() + os.close(read_fd) # nothing is listening: the forward will fail + line = json.dumps({"jsonrpc": "2.0", "id": 1, "result": {"content": []}}).encode() + b"\n" + + with pytest.raises((BrokenPipeError, OSError)): + process_buffer_lines( + bytearray(line), + forward_fd=write_fd, + recorder=recorder, + is_client=False, + record_jsonrpc=True, + ) + os.close(write_fd) + recorder.write_meta() + + _calls, status = load_proxy_sidecar(tmp_path / "undelivered.jsonl") + assert status["undelivered_lines"] == 1 + assert status["state"] == "incomplete" + + +# --------------------------------------------------------------------------- +# Which pump is alive at shutdown decides whether a trace is short or merely +# interrupted. Claude Code signals its MCP servers on exit, so the proxy's stdin +# read is parked on a client that will never write again — every claude-cli row +# was charged to infrastructure for a pump that could not have lost anything. +# --------------------------------------------------------------------------- + + +def _meta(**kw: Any) -> dict[str, Any]: + base = {"pumps_alive": True, "pumps_alive_streams": ["stdin"], "finalization_reason": "signal"} + base.update(kw) + return base + + +@pytest.mark.parametrize( + ("meta", "blocking", "why"), + [ + pytest.param(None, False, "no meta row at all", id="no-meta"), + pytest.param( + _meta(pumps_alive=False, pumps_alive_streams=[]), False, "nothing was pumping", id="quiet-shutdown" + ), + # The case that made every claude-cli run an infra error. + pytest.param(_meta(), False, "client signalled away; stdin cannot deliver more", id="stdin-parked-on-signal"), + pytest.param( + _meta(finalization_reason="child_exit"), False, "server gone; same reasoning", id="stdin-on-child-exit" + ), + # A live stdin pump with no reason for the client to have stopped is still suspect. + pytest.param( + _meta(finalization_reason="normal_eof"), True, "EOF should have ended the read", id="stdin-after-clean-eof" + ), + # Output pumps carry the server's replies, so a live one may mean a lost response. + pytest.param(_meta(pumps_alive_streams=["stdout"]), True, "server may have been mid-reply", id="stdout-alive"), + pytest.param(_meta(pumps_alive_streams=["stderr"]), True, "same for stderr", id="stderr-alive"), + pytest.param( + _meta(pumps_alive_streams=["stdin", "stdout"]), True, "one bad stream is enough", id="mixed-streams" + ), + # Sidecars written before per-stream detail keep the old stricter reading rather + # than being silently reinterpreted in their favour. + pytest.param( + {"pumps_alive": True, "finalization_reason": "signal"}, True, "legacy file", id="pre-detail-sidecar" + ), + ], +) +def test_only_a_pump_that_could_have_lost_something_invalidates_the_trace(meta, blocking, why): + assert _pumps_blocking(meta) is blocking, why + + +@pytest.mark.parametrize( + ("metas", "fingerprint", "disagreement", "why"), + [ + pytest.param([{"tool_manifest_fingerprint": "fp1"}], "fp1", False, "one session, one listing", id="single"), + # Claude Code lists tools in one session and makes the calls in another. The quiet + # session has no opinion, and counting it as a dissenter discarded the fingerprint + # on nearly every row and left the reporter unable to compare the file. + pytest.param( + [{"tool_manifest_fingerprint": "fp1"}, {}], "fp1", False, "abstaining session", id="one-lists-one-calls" + ), + pytest.param([{}, {}], None, False, "nobody listed", id="no-listing-anywhere"), + # Real disagreement is two sessions naming different surfaces. + pytest.param( + [{"tool_manifest_fingerprint": "fp1"}, {"tool_manifest_fingerprint": "fp2"}], + None, + True, + "genuinely different surfaces", + id="conflicting", + ), + ], +) +def test_a_session_that_never_listed_tools_does_not_veto_the_manifest(tmp_path, metas, fingerprint, disagreement, why): + base = tmp_path / "sidecar.jsonl" + for index, meta in enumerate(metas): + row = {"row_type": "proxy_meta", "finalized": True, "last_seq": 0, "tool_request_count": 0, **meta} + (tmp_path / f"sidecar.jsonl.{index}.jsonl").write_text(json.dumps(row) + "\n") + _calls, status = load_proxy_sidecar(base) + assert status["tool_manifest_fingerprint"] == fingerprint, why + assert status["tool_manifest_disagreement"] is disagreement, why diff --git a/tests/evals/test_results.py b/tests/evals/test_results.py new file mode 100644 index 0000000..3354896 --- /dev/null +++ b/tests/evals/test_results.py @@ -0,0 +1,399 @@ +"""Offline eval tests for results.""" + +from __future__ import annotations + +from collections import deque +from contextlib import asynccontextmanager +from dataclasses import fields +from types import SimpleNamespace +from typing import Any + +import pytest + +from evals.core.evidence import TARGET_ENTITY_EVIDENCE +from evals.core.results import ( + AGENT_RESULT_COPY_FIELDS, + AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS, + RESULT_SCHEMA_VERSION, + TASK_RESULT_HARNESS_FIELDS, + AgentRun, + CallRecord, + TaskResult, + Usage, + agent_run_to_harness_dict, +) +from evals.core.token_counting import estimate_result_tokens +from evals.core.tool_names import ( + normalize_tool_call, +) +from evals.drivers.api import ( + StopReason, + ToolCall, + ToolResult, + ToolSpec, + Turn, +) +from evals.drivers.api.driver import ApiDriver +from tests.evals.conftest import case_params + + +class FakeBackend: + provider = "fake" + model = "fake-requested" + actual_model = "fake-actual" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = deque(turns) + self.started: tuple[str | None, str, list[ToolSpec]] | None = None + self.added_results: list[list[ToolResult]] = [] + self.num_turns = 0 + + def start(self, system: str | None, prompt: str, tools: list[ToolSpec]) -> None: + self.started = (system, prompt, tools) + + def next_turn(self) -> Turn: + self.num_turns += 1 + if not self.turns: + raise AssertionError("driver requested an unexpected backend turn") + return self.turns.popleft() + + def add_tool_results(self, results: list[ToolResult]) -> None: + self.added_results.append(results) + + +class FakeMcpSession: + def __init__(self, results: list[Any] | None = None) -> None: + self.results = deque(results or []) + self.initialized = False + self.called: list[tuple[str, dict[str, Any]]] = [] + + async def initialize(self) -> None: + self.initialized = True + + async def list_tools(self) -> Any: + return SimpleNamespace( + tools=[ + SimpleNamespace( + name="lookup", + description="Look something up", + inputSchema={"type": "object", "properties": {"q": {"type": "string"}}}, + ), + SimpleNamespace( + name="write", + description="Write something", + inputSchema={"type": "object"}, + ), + ] + ) + + async def call_tool(self, name: str, arguments: dict[str, Any]) -> Any: + self.called.append((name, arguments)) + if not self.results: + raise AssertionError(f"no fake result left for {name}") + return self.results.popleft() + + +def make_driver(backend: FakeBackend, session: FakeMcpSession) -> ApiDriver: + @asynccontextmanager + async def session_factory(_params): + yield session + + return ApiDriver( + provider="anthropic", + backend_factory=lambda _model, _max_tokens: backend, + mcp_session_factory=session_factory, + ) + + +def run_driver(driver: ApiDriver, *, max_turns: int = 5): + return driver.run_task( + "do it", + {"SAFE": "1"}, + "fake-requested", + max_turns, + system="system", + ) + + +def test_api_driver_maps_every_current_row_field(): + backend = FakeBackend( + [ + Turn( + text="", + tool_calls=[ToolCall("a", "lookup", {"q": "a"})], + usage=Usage(4, 1), + stop_reason=StopReason.TOOL_USE, + ), + Turn( + text="done", + tool_calls=[], + usage=None, + stop_reason=StopReason.END_TURN, + provider_stop_reason="fake_done", + ), + ] + ) + run = run_driver(make_driver(backend, FakeMcpSession([ToolResult(call_id="a", text="12345")]))) + row = agent_run_to_harness_dict(run) + + required = { + "final_text", + "calls", + "num_calls", + "errored_calls", + "total_result_tokens", + "usage_per_iteration", + "cum_input_tokens", + "wall_time_s", + "stop_reason", + "provider_stop_reason", + "hit_max_iterations", + "result_pair_mismatch", + "token_count_failures", + } + assert required <= row.keys() + assert { + "tool", + "args_chars", + "result_tokens", + "result_chars", + "result_kind", + "is_error", + } <= row["calls"][0].keys() + assert row["calls"][0]["result_chars"] == 5 + assert row["calls"][0]["result_tokens"] == estimate_result_tokens(5) == 2 + assert row["result_tokens_estimated"] is True + assert row["provider"] == "fake" + assert row["model"] == "fake-actual" + assert row["requested_model"] == "fake-requested" + assert row["provider_stop_reason"] == "fake_done" + assert row["tool_manifest_fingerprint"] + + +def _agent_run_dict_keeps_action_arg(): + run = AgentRun( + calls=[ + {"tool": "work_item", "args": {"action": "create", "name": "x"}, "origin": "plane"}, + {"tool": "get_pql_reference", "args": {}, "origin": "plane"}, + ], + final_text="done", + usage=None, + stopped_reason="end_turn", + ) + d = agent_run_to_harness_dict(run) + assert d["calls"][0]["action"] == "create" + assert "action" not in d["calls"][1] + + +def _agent_run_to_harness_dict_excludes_toolsearch_from_plane_calls(): + run = AgentRun( + calls=[ + normalize_tool_call("mcp__plane__find_work_items", {"project": "A"}), + ], + client_tool_calls=[ + normalize_tool_call("ToolSearch", {"query": "work items"}), + ], + final_text="done", + usage={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_cost_usd": 0.29, + "modelUsage": { + "claude-sonnet": { + "inputTokens": 10, + "outputTokens": 865, + "cacheReadInputTokens": 250433, + "cacheCreationInputTokens": 33838, + "costUSD": 0.29, + } + }, + }, + usage_total={ + "input_tokens": 10, + "output_tokens": 865, + "cache_read_input_tokens": 250433, + "cache_creation_input_tokens": 33838, + "total_input_tokens_including_cache": 10 + 250433 + 33838, + "total_cost_usd": 0.29, + "source": "modelUsage", + }, + stopped_reason="end_turn", + usage_scope="run", + call_source="transcript", + hit_max_turns=False, + wall_time_s=1.5, + ) + out = agent_run_to_harness_dict(run) + assert out["num_calls"] == 1 + assert out["client_tool_call_count"] == 1 + assert out["client_tool_calls"][0]["tool"] == "ToolSearch" + # F2: cum_input_tokens null — not the misleading uncached-only 10 + assert out["cum_input_tokens"] is None + assert out["cum_input_tokens_reason"] + assert out["usage_total"]["total_input_tokens_including_cache"] == 10 + 250433 + 33838 + assert out["usage_per_iteration"] == [] + assert out["calls"][0]["result_tokens"] == 0 + assert out["calls"][0]["result_tokens_estimated"] is True + assert out["result_tokens_estimated"] is True + assert "result_tokens_skipped_reason" not in out + + +def _agent_run_hit_max_maps_to_hit_max_iterations(): + run = AgentRun( + calls=[], + final_text="", + usage=None, + stopped_reason="end_turn", + hit_max_turns=True, + call_source="json", + ) + out = agent_run_to_harness_dict(run) + assert out["hit_max_iterations"] is True + assert out["stop_reason"] == "max_turns" + + +def _agent_run_to_harness_dict_does_not_guess_usage_total(): + run = AgentRun( + calls=[], + final_text="ok", + usage={ + "input_tokens": 5000, + "output_tokens": 200, + # Codex-ish shape — not Claude modelUsage. A Claude rebuild would + # silently produce a wrong / empty total if reintroduced. + "total_token_usage": {"input_tokens": 5000, "output_tokens": 200}, + }, + usage_total=None, + stopped_reason="completed", + usage_scope="run", + call_source="stream", + ) + out = agent_run_to_harness_dict(run) + assert out["usage"] == run.usage + assert out["usage_total"] is None + + +def _agent_run_to_harness_propagates_proxy_fields(): + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {"q": "a"}, + "origin": "plane", + "is_error": True, + "result_chars": 99, + "duration_ms": 42, + "observed_sentinels": [TARGET_ENTITY_EVIDENCE], + } + ], + final_text="x", + usage=None, + stopped_reason="end_turn", + call_source="proxy", + usage_scope="run", + evidence_trace_available=True, + ) + d = agent_run_to_harness_dict(run) + assert d["calls"][0]["is_error"] is True + assert d["calls"][0]["result_chars"] == 99 + assert d["calls"][0]["result_tokens"] == estimate_result_tokens(99) + assert d["calls"][0]["result_tokens_estimated"] is True + assert d["result_tokens_estimated"] is True + assert d["calls"][0]["duration_ms"] == 42 + assert d["calls"][0]["observed_sentinels"] == [TARGET_ENTITY_EVIDENCE] + assert d["evidence_trace_available"] is True + assert d["errored_calls"] == 1 + + +@pytest.mark.parametrize( + "case", + case_params( + _agent_run_dict_keeps_action_arg, + _agent_run_to_harness_dict_excludes_toolsearch_from_plane_calls, + _agent_run_hit_max_maps_to_hit_max_iterations, + _agent_run_to_harness_dict_does_not_guess_usage_total, + _agent_run_to_harness_propagates_proxy_fields, + ), +) +def test_agent_run_behaviours(case): + case() + + +def test_task_result_schema_round_trip_owns_usage_shape(): + result = TaskResult( + row_type="result", + run_id="run-1", + fixture_seed_id="fixture-seed-1", + task_id="R1", + task_fingerprint="taskhash0001", + label="local", + server="local", + expected_rows=35, + cleanup_error="RuntimeError: teardown failed", + seeded_entity_kinds=["project", "work_item"], + randomized_seed_namespaces=["R2.urgent_open_count"], + calls=[ + CallRecord( + tool="find_work_items", + result_tokens=3, + result_tokens_estimated=False, + result_token_count_method="backend", + observed_sentinels=[TARGET_ENTITY_EVIDENCE], + ) + ], + num_calls=1, + evidence_trace_available=True, + trace_integrity=False, + trace_integrity_reason="protocol_violation", + tool_manifest_fingerprint="manifest-sha256", + usage_per_iteration=[Usage(10, 2, 3, 4)], + ) + + row = result.to_row() + assert row["schema_version"] == RESULT_SCHEMA_VERSION + assert row["row_type"] == "result" + assert row["run_id"] == "run-1" + assert row["fixture_seed_id"] == "fixture-seed-1" + assert row["task_fingerprint"] == "taskhash0001" + assert row["label"] == "local" + assert row["server"] == "local" + assert row["expected_rows"] == 35 + assert row["cleanup_error"] == "RuntimeError: teardown failed" + assert row["seeded_entity_kinds"] == ["project", "work_item"] + assert row["randomized_seed_namespaces"] == ["R2.urgent_open_count"] + assert row["usage_per_iteration"] == [{"in": 10, "out": 2, "cache_read": 3, "cache_write": 4}] + loaded = TaskResult.from_row(row) + assert loaded.row_type == "result" + assert loaded.run_id == "run-1" + assert loaded.fixture_seed_id == "fixture-seed-1" + assert loaded.task_fingerprint == "taskhash0001" + assert loaded.calls[0].tool == "find_work_items" + assert loaded.calls[0].observed_sentinels == [TARGET_ENTITY_EVIDENCE] + assert loaded.evidence_trace_available is True + assert loaded.trace_integrity is False + assert loaded.trace_integrity_reason == "protocol_violation" + assert loaded.tool_manifest_fingerprint == "manifest-sha256" + assert loaded.expected_rows == 35 + assert loaded.cleanup_error == "RuntimeError: teardown failed" + assert loaded.seeded_entity_kinds == result.seeded_entity_kinds + assert loaded.randomized_seed_namespaces == result.randomized_seed_namespaces + assert loaded.usage_per_iteration == [Usage(10, 2, 3, 4)] + + +def test_apply_agent_result_reflection_parity_and_skipped_reason_copy(): + declared = {field.name for field in fields(TaskResult)} + copied = set(AGENT_RESULT_COPY_FIELDS) + optional_identity = set(AGENT_RESULT_OPTIONAL_IDENTITY_FIELDS) + harness_owned = set(TASK_RESULT_HARNESS_FIELDS) + + assert not (copied & optional_identity or copied & harness_owned or optional_identity & harness_owned) + assert declared == copied | optional_identity | harness_owned + + row = TaskResult(task_id="R1", result_tokens_skipped_reason=None) + agent = TaskResult(task_id="must-not-replace", result_tokens_skipped_reason="payload recording disabled") + row.apply_agent_result(agent) + + assert row.task_id == "R1" + assert row.result_tokens_skipped_reason == "payload recording disabled" diff --git a/tests/evals/test_skip_taxonomy.py b/tests/evals/test_skip_taxonomy.py new file mode 100644 index 0000000..24efbad --- /dev/null +++ b/tests/evals/test_skip_taxonomy.py @@ -0,0 +1,110 @@ +"""Offline tests for the explicit environment skip taxonomy.""" + +from __future__ import annotations + +import pytest + +from evals.skip_taxonomy import ( + PLAN_GATED_CAPABILITIES, + classify_skip_reason, + is_expected_environment_capability_skip, + skip_reason_family, +) + + +@pytest.mark.parametrize( + ("reason", "disposition", "family"), + [ + pytest.param("env:plan-gated:customers", "expected-capability", "plan-gated", id="plan-gated"), + pytest.param("env:no-activity-worker", "expected-capability", "no-activity-worker", id="activity-worker"), + pytest.param( + "env:no-activity-worker (ConnectionError: unavailable)", + "unexpected", + "env:no-activity-worker (ConnectionError: unavailable)", + id="activity-worker-detail-is-not-a-capability-skip", + ), + pytest.param( + "env:fixture-collision:customers:Acme", + "dirty-environment", + "fixture-collision", + id="fixture-collision", + ), + pytest.param("env:new-capability", "unexpected", "env:new-capability", id="unknown-env-reason"), + pytest.param( + "env:plan-gated:customerz", + "unexpected", + "env:plan-gated:customerz", + id="unknown-plan-gated-capability", + ), + pytest.param("env:plan-gated:", "unexpected", "env:plan-gated:", id="malformed-plan-gate"), + pytest.param("env:no-activity-worker-new", "unexpected", "env:no-activity-worker-new", id="near-miss"), + ], +) +def test_skip_reason_taxonomy_is_explicit_and_fail_closed(reason, disposition, family): + assert classify_skip_reason(reason) == disposition + assert is_expected_environment_capability_skip(reason) is (disposition == "expected-capability") + assert skip_reason_family(reason) == family + + +def test_plan_gated_capability_allowlist_matches_reviewed_seed_surfaces(): + assert PLAN_GATED_CAPABILITIES == frozenset( + {"customers", "releases", "work-item-types", "initiatives", "teamspaces"} + ) + for capability in PLAN_GATED_CAPABILITIES: + assert classify_skip_reason(f"env:plan-gated:{capability}") == "expected-capability", capability + + +def test_task_capability_pairs_are_derived_from_fixture_needs_and_fail_closed(): + assert classify_skip_reason("env:plan-gated:customers", task_id="L4") == "expected-capability" + assert classify_skip_reason("env:plan-gated:customers", task_id="W1") == "unexpected" + assert classify_skip_reason("env:plan-gated:releases", task_id="C2") == "expected-capability" + assert classify_skip_reason("env:plan-gated:releases", task_id="L3") == "unexpected" + assert classify_skip_reason("env:plan-gated:work-item-types", task_id="S1") == "expected-capability" + assert classify_skip_reason("env:no-activity-worker", task_id="L2") == "expected-capability" + assert classify_skip_reason("env:no-activity-worker", task_id="R1") == "unexpected" + + +def test_describe_exception_flattens_a_task_group(): + """The real failure must survive into the row, not the group's sub-exception count. + + An OpenAI 400 naming the exact unsupported parameter was recorded as "unhandled errors in + a TaskGroup (1 sub-exception)", and recovering it meant reproducing the call by hand. + + The helper reads only ``.exceptions``, so the contract is testable on every supported + Python; the genuine builtin is 3.11+, and this project still declares 3.10. + """ + import builtins + import sys + + from evals.core.errors import describe_exception + + class FakeGroup(Exception): + """Anything exposing .exceptions -- which is all the helper looks at.""" + + def __init__(self, message, exceptions): + super().__init__(message) + self.exceptions = tuple(exceptions) + + inner = ValueError("Function tools with reasoning_effort are not supported") + described = describe_exception(FakeGroup("unhandled errors in a TaskGroup", [inner])) + assert "reasoning_effort" in described, "the actual cause was dropped" + assert "ValueError" in described + + # Nested groups flatten to their leaves. + nested = FakeGroup("outer", [FakeGroup("inner", [RuntimeError("deep")])]) + assert "deep" in describe_exception(nested) + + # A plain exception is unchanged in substance. + assert describe_exception(RuntimeError("plain")) == "RuntimeError: plain" + + # A pathological fan-out is bounded rather than unbounded. + many = FakeGroup("many", [RuntimeError(f"e{i}") for i in range(20)]) + assert describe_exception(many, limit=3).count("RuntimeError") == 3 + + # And against the real builtin wherever it exists -- looked up dynamically so this stays + # importable on 3.10 and does not read as an undefined name to the linter. + real_group = getattr(builtins, "BaseExceptionGroup", None) + if real_group is not None and sys.version_info >= (3, 11): + described = describe_exception(real_group("unhandled errors in a TaskGroup", [inner])) + assert "reasoning_effort" in described + assert "sub-exception" not in described.split(" -> ")[-1] diff --git a/tests/evals/test_token_accounting.py b/tests/evals/test_token_accounting.py new file mode 100644 index 0000000..a90de83 --- /dev/null +++ b/tests/evals/test_token_accounting.py @@ -0,0 +1,134 @@ +"""Both cache semantics must normalise to the same meaning.""" + +from __future__ import annotations + +import pytest + +from evals.core.token_accounting import ( + EXCLUSIVE, + INCLUSIVE, + TokenAccounting, + cache_semantics_of, + normalize_usage, +) + +# Shapes taken verbatim from real arms (2026-08-24 battery eaf35e8019aa). +OPENAI_ROW = { + "input_tokens": 97813, + "output_tokens": 1121, + "cache_read_input_tokens": 83460, + "cache_creation_input_tokens": 0, + "source": "iterations", +} +CODEX_CLI_ROW = { + "input_tokens": 249983, + "output_tokens": 682, + "cache_read_input_tokens": 224768, + "cache_creation_input_tokens": 0, + "total_input_tokens_including_cache": 474751, + "source": "codex_token_count", +} +CLAUDE_CLI_ROW = { + "input_tokens": 971, + "output_tokens": 734, + "cache_read_input_tokens": 89488, + "cache_creation_input_tokens": 30899, + "total_input_tokens_including_cache": 121358, + "total_cost_usd": 0.0753878, + "source": "modelUsage", +} + + +def test_explicit_total_is_treated_as_exclusive(): + """A recorded total means input_tokens excludes cache; the total is authoritative.""" + accounting = normalize_usage(CODEX_CLI_ROW) + assert accounting == TokenAccounting( + uncached_input=249983, + cached_input=224768, + cache_creation=0, + output=682, + total_input=474751, + semantics=EXCLUSIVE, + semantics_source="explicit_total", + ) + + +def test_openai_input_tokens_are_inclusive_of_cache(): + """Responses counts cached reads inside input_tokens; uncached is the remainder.""" + accounting = normalize_usage(OPENAI_ROW, model="gpt-5.6-luna") + assert accounting is not None + assert accounting.semantics == INCLUSIVE + assert accounting.total_input == 97813 + assert accounting.uncached_input == 97813 - 83460 + + +def test_anthropic_api_input_tokens_are_exclusive_of_cache(): + """The Messages API reports input_tokens net of both cache fields. + + The same api driver produces this and the OpenAI shape above, so driver family + cannot decide the semantics -- this is the case the first plan draft got wrong. + """ + row = { + "input_tokens": 1200, + "output_tokens": 300, + "cache_read_input_tokens": 50000, + "cache_creation_input_tokens": 2000, + "source": "iterations", + } + accounting = normalize_usage(row, model="claude-haiku-4-5") + assert accounting is not None + assert accounting.semantics == EXCLUSIVE + assert accounting.uncached_input == 1200 + assert accounting.total_input == 1200 + 50000 + 2000 + + +def test_declared_semantics_beat_inference(): + """A driver that records what it means is trusted over any model-name guess.""" + row = dict(OPENAI_ROW, cache_semantics=EXCLUSIVE) + accounting = normalize_usage(row, model="gpt-5.6-luna") + assert accounting is not None + assert accounting.semantics == EXCLUSIVE + assert accounting.semantics_source == "declared" + assert accounting.total_input == 97813 + 83460 + + +def test_uncached_row_needs_no_semantics_at_all(): + """With no cache activity the two readings coincide, so an unknown model is fine.""" + row = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 0} + accounting = normalize_usage(row, model="some-model-nobody-has-heard-of") + assert accounting is not None + assert accounting.semantics_source == "no_cache" + assert accounting.total_input == 500 + assert accounting.uncached_input == 500 + + +def test_cached_row_with_unknown_model_refuses_to_guess(): + """Guessing here misprices by ~4x, which is how two conclusions reversed.""" + row = {"input_tokens": 500, "output_tokens": 10, "cache_read_input_tokens": 400} + assert normalize_usage(row, model="some-model-nobody-has-heard-of") is None + assert cache_semantics_of(row, model=None) is None + + +def test_identity_violation_is_not_silently_priced(): + """input + cache_read + cache_creation == total holds 70/70 on both CLI vendors. + + If a vendor changes shape the sum stops matching, and reporting the row as + unpriced is the loud outcome; using either number would misprice invisibly. + """ + broken = dict(CODEX_CLI_ROW, total_input_tokens_including_cache=999999) + assert normalize_usage(broken) is None + + +def test_absent_usage_is_none(): + assert normalize_usage(None) is None + assert normalize_usage({}) is None + + +@pytest.mark.parametrize("row", [OPENAI_ROW, CODEX_CLI_ROW, CLAUDE_CLI_ROW]) +def test_parts_never_exceed_the_total(row): + """Whatever the shape, the normalised parts must reconstruct the total input.""" + model = "gpt-5.6-luna" if row is OPENAI_ROW else None + accounting = normalize_usage(row, model=model) + assert accounting is not None + assert accounting.uncached_input + accounting.cached_input + accounting.cache_creation == accounting.total_input + assert accounting.uncached_input >= 0 diff --git a/tests/evals/test_token_counting.py b/tests/evals/test_token_counting.py new file mode 100644 index 0000000..f479f11 --- /dev/null +++ b/tests/evals/test_token_counting.py @@ -0,0 +1,60 @@ +"""Offline eval tests for token counting.""" + +from __future__ import annotations + +import sys + +import pytest + +from evals.core.results import AgentRun, agent_run_to_harness_dict +from evals.core.token_counting import estimate_result_tokens + + +@pytest.mark.parametrize("has_tokenizer", [True, False], ids=["importable-tokenizer", "estimator-fallback"]) +def test_agent_behaviours(monkeypatch, has_tokenizer): + text = "serialized workspace result" + + if has_tokenizer: + + class FakeEncoding: + def encode(self, encoded_text): + assert encoded_text == text + return [10, 20, 30] + + class FakeTiktoken: + @staticmethod + def get_encoding(name): + assert name == "cl100k_base" + return FakeEncoding() + + monkeypatch.setitem(sys.modules, "tiktoken", FakeTiktoken) + else: + monkeypatch.setitem(sys.modules, "tiktoken", None) + + run = AgentRun( + calls=[ + { + "tool": "find_work_items", + "args": {}, + "origin": "plane", + "result_chars": len(text), + "result_text": text, + } + ], + final_text="ok", + usage=None, + stopped_reason="completed", + usage_scope="run", + call_source="proxy", + ) + + out = agent_run_to_harness_dict(run) + + expected_tokens = 3 if has_tokenizer else estimate_result_tokens(len(text)) + assert out["calls"][0]["result_tokens"] == expected_tokens + assert out["calls"][0]["result_tokens_estimated"] is not has_tokenizer + assert out["result_tokens_estimated"] is not has_tokenizer + assert "result_text" not in out["calls"][0] + if has_tokenizer: + assert out["calls"][0]["result_token_count_method"] == "tiktoken:cl100k_base" + assert out["result_tokens_mode"] == "measured" diff --git a/tests/evals/test_tool_manifest.py b/tests/evals/test_tool_manifest.py new file mode 100644 index 0000000..35c1603 --- /dev/null +++ b/tests/evals/test_tool_manifest.py @@ -0,0 +1,71 @@ +"""Tool-manifest fingerprint regression tests.""" + +from __future__ import annotations + +from evals.core.tool_manifest import ToolManifestCapture, tool_manifest_fingerprint + + +def test_same_tool_names_with_different_schemas_have_different_manifest_fingerprints(): + short = [ + { + "name": "create_work_item", + "description": "Create an item", + "inputSchema": {"type": "object", "properties": {"title": {"type": "string"}}}, + } + ] + consolidated = [ + { + "name": "create_work_item", + "description": "Create an item", + "inputSchema": { + "type": "object", + "properties": { + "workspace_slug": {"type": "string"}, + "project_slug": {"type": "string"}, + "title": {"type": "string"}, + "type_id": {"type": "string"}, + }, + }, + } + ] + + assert tool_manifest_fingerprint(short) != tool_manifest_fingerprint(consolidated) + + +def test_paginated_tools_list_hashes_like_equivalent_single_page(): + first = [{"name": "alpha", "inputSchema": {"type": "object"}}] + second = [{"name": "beta", "inputSchema": {"type": "object"}}] + paginated = ToolManifestCapture() + paginated.observe_page({"tools": first, "nextCursor": "page-2"}, request_cursor=None) + assert paginated.fingerprint is None + paginated.observe_page({"tools": second}, request_cursor="page-2") + + single = ToolManifestCapture() + single.observe_page({"tools": [*second, *first]}, request_cursor=None) + + assert paginated.fingerprint == single.fingerprint + + +def test_manifest_fingerprint_recursively_canonicalizes_object_key_order(): + left = [ + { + "name": "lookup", + "inputSchema": { + "type": "object", + "properties": {"q": {"type": "string", "description": "query"}}, + }, + "annotations": {"readOnlyHint": True, "destructiveHint": False}, + } + ] + right = [ + { + "annotations": {"destructiveHint": False, "readOnlyHint": True}, + "inputSchema": { + "properties": {"q": {"description": "query", "type": "string"}}, + "type": "object", + }, + "name": "lookup", + } + ] + + assert tool_manifest_fingerprint(left) == tool_manifest_fingerprint(right) diff --git a/tests/evals/test_tool_names.py b/tests/evals/test_tool_names.py new file mode 100644 index 0000000..3cd33bc --- /dev/null +++ b/tests/evals/test_tool_names.py @@ -0,0 +1,24 @@ +"""Offline eval tests for tool names.""" + +from __future__ import annotations + +from evals.core.tool_names import ( + is_plane_mcp_tool, + strip_mcp_prefix, +) + + +def test_strip_mcp_prefix(): + assert strip_mcp_prefix("mcp__plane__list_work_items") == "list_work_items" + assert strip_mcp_prefix("mcp__plane-mcp-server__find_work_items") == "find_work_items" + assert strip_mcp_prefix("list_work_items") == "list_work_items" + assert strip_mcp_prefix("Bash") == "Bash" + + +def test_is_plane_mcp_tool(): + assert is_plane_mcp_tool("mcp__plane__find_work_items") + assert is_plane_mcp_tool("mcp__plane-foo__x") + assert not is_plane_mcp_tool("ToolSearch") + assert not is_plane_mcp_tool("Bash") + assert not is_plane_mcp_tool("mcp__other__tool") + assert not is_plane_mcp_tool("find_work_items") diff --git a/tests/fixtures/evals_schema_v0_rows.jsonl b/tests/fixtures/evals_schema_v0_rows.jsonl new file mode 100644 index 0000000..e05df63 --- /dev/null +++ b/tests/fixtures/evals_schema_v0_rows.jsonl @@ -0,0 +1,2 @@ +{"run_id":"7a637f5a54664d3eb2fff9ac5a53fb43","ts":"2026-08-12T17:59:05.498671+00:00","git_sha":"5da71142cab2d9fd7e8f95be8192ccb17ac3d826","battery":"6647676edc9e","label":"manish-v2","driver":"codex-cli","server":"external","model":"gpt-5.6-sol","task_id":"L3","author":"post-hoc-debias","rep":0,"success":true,"verify_note":"release tag 'eval-rc1' present","skipped":null,"error":null,"error_class":null,"stop_reason":"end_turn","hit_max_iterations":false,"calls":[{"tool":"release_tag","args_chars":43,"result_tokens":null,"result_chars":1016,"result_kind":"text","is_error":false,"duration_ms":91,"action":"create","result_tokens_skipped":"no API key / CLI driver has no count_tokens"}],"num_calls":1,"errored_calls":0,"total_result_tokens":0,"usage_per_iteration":[],"cum_input_tokens":null,"wall_time_s":25.381,"client_tool_calls":[{"tool":"release_tag","args_chars":43,"raw_tool":"release_tag"}],"client_tool_call_count":1,"cum_input_tokens_reason":"CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting","result_pair_mismatch":false,"token_count_failures":0,"usage_scope":"run","call_source":"proxy","driver_raw_ref":"session:019ff720-bf7b-75d2-9b23-eb0b635be673","driver_notes":["experimental:codex-cli","proxy_sidecar_incomplete:no_meta","calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-iivxzbkb/proxy-sidecar.jsonl"],"result_tokens_skipped_reason":"CLI driver: count_tokens requires Anthropic API key; skipped","usage":{"input_tokens":143874,"output_tokens":504,"cache_read_input_tokens":119552,"cache_creation_input_tokens":0,"total_tokens":null},"usage_total":{"input_tokens":143874,"output_tokens":504,"cache_read_input_tokens":119552,"cache_creation_input_tokens":0,"total_input_tokens_including_cache":263426,"source":"codex_token_count"}} +{"run_id":"625464c995c646429f7cfbcb1a9f5166","ts":"2026-08-13T03:37:23.554029+00:00","git_sha":"adf653458ed5788e58acc5e2e9751143df942a5d","battery":"6425dcc64404","label":"full","driver":"codex-cli","provider":null,"server":"local","model":"gpt-5.6-sol","requested_model":"gpt-5.6-sol","task_id":"R2","author":"claude","rep":0,"success":true,"verify_note":"final text names count 4","skipped":null,"error":null,"error_class":null,"final_text":"I\u2019m checking the project\u2019s current open work items and urgent priority filter.\n4","stop_reason":"end_turn","hit_max_iterations":false,"result_pair_mismatch":false,"token_count_failures":0,"result_tokens_estimated":true,"calls":[{"tool":"list_projects","args_chars":18,"result_tokens":315,"result_chars":1258,"result_kind":"text","is_error":false,"result_tokens_estimated":true,"result_token_count_method":"chars_div_4","duration_ms":159},{"tool":"count_work_items","args_chars":118,"result_tokens":64,"result_chars":253,"result_kind":"text","is_error":false,"result_tokens_estimated":true,"result_token_count_method":"chars_div_4","duration_ms":107}],"num_calls":2,"errored_calls":0,"total_result_tokens":379,"usage_per_iteration":[],"cum_input_tokens":null,"wall_time_s":29.679,"client_tool_calls":[{"tool":"list_projects","args_chars":18,"raw_tool":"list_projects"},{"tool":"count_work_items","args_chars":118,"raw_tool":"count_work_items"}],"client_tool_call_count":2,"cum_input_tokens_reason":"CLI driver: Claude usage.input_tokens is uncached-only; see usage_total (cache_read/cache_creation/output/cost) for run accounting","result_tokens_mode":"estimated","result_token_count_method":"chars_div_4","usage_scope":"run","call_source":"proxy","driver_raw_ref":"session:019ff932-5598-7d62-9ab9-30c6bf5fca15","driver_notes":["experimental:codex-cli","proxy_sidecar_incomplete:no_meta","calls_from_proxy:/var/folders/gl/0n9h9bk15sn80q_93jnc7py40000gn/T/plane-eval-codex-57f304ug/proxy-sidecar.jsonl"],"usage":{"input_tokens":157010,"output_tokens":501,"cache_read_input_tokens":135680,"cache_creation_input_tokens":0,"total_tokens":null},"usage_total":{"input_tokens":157010,"output_tokens":501,"cache_read_input_tokens":135680,"cache_creation_input_tokens":0,"total_input_tokens_including_cache":292690,"source":"codex_token_count"}} diff --git a/tests/tools/_spyclient.py b/tests/tools/_spyclient.py index 955803f..db60f30 100644 --- a/tests/tools/_spyclient.py +++ b/tests/tools/_spyclient.py @@ -20,6 +20,11 @@ from plane import PlaneClient from pydantic import BaseModel, TypeAdapter +try: # Python 3.14+ evaluates annotations inside inspect.signature; see _signature_of + from annotationlib import Format as _AnnotationFormat +except ImportError: # pragma: no cover - Python < 3.13 has no annotationlib + _AnnotationFormat = None + types_UnionType = type(int | str) # `X | Y` annotations are not typing.Union @@ -132,12 +137,39 @@ def _validate(method: str, param: inspect.Parameter, annotation: Any, value: Any raise TypeError(f"{method}(): argument {param.name}={value!r} does not satisfy {annotation}: {exc}") from exc +#: SDK methods whose annotations would not evaluate eagerly. Kept visible rather than +#: swallowed: a type-check that quietly checks nothing is worse than one that fails. +UNEVALUATED_ANNOTATIONS: set[str] = set() + + +def _signature_of(path: str, fn: Any) -> inspect.Signature: + """Bind-capable signature, even for a method whose annotations will not evaluate. + + Python 3.14 (PEP 649) made annotations lazy, and ``inspect.signature`` evaluates + them. plane-sdk declares twelve methods as ``def list(self, ...) -> list[...]``, + and under lazy evaluation the class namespace is in scope, so ``list`` resolves to + the method being defined rather than the builtin -- subscripting a function raises + TypeError. Before 3.14 the annotation was evaluated at ``def`` time, before that + name was bound, so this never came up. + + FORWARDREF still resolves what it can (``list[int]`` comes back intact) and leaves + the rest as ForwardRefs, so binding *and* type-checking survive the fallback. + """ + try: + return inspect.signature(fn) + except (TypeError, NameError): + if _AnnotationFormat is None: # pragma: no cover - Python < 3.14 never gets here + raise + UNEVALUATED_ANNOTATIONS.add(path) + return inspect.signature(fn, annotation_format=_AnnotationFormat.FORWARDREF) + + class _Method: def __init__(self, spy: SpyClient, path: str, fn: Any) -> None: self._spy = spy self._path = path self._fn = fn - self._signature = inspect.signature(fn) + self._signature = _signature_of(path, fn) try: self._hints = get_type_hints(fn) except Exception: diff --git a/tests/tools/test_governance.py b/tests/tools/test_governance.py index 3c59691..27972ba 100644 --- a/tests/tools/test_governance.py +++ b/tests/tools/test_governance.py @@ -14,13 +14,13 @@ from __future__ import annotations -import inspect from types import SimpleNamespace import pytest from plane.errors.errors import HttpError from plane_mcp.tools.workitem_type import _scope_of +from tests.tools._spyclient import _signature_of PROJECT = "project-1" TYPE_ID = "type-1" @@ -85,7 +85,7 @@ def test_the_resolver_matches_the_sdk(project_id): for verb in ("list", "retrieve", "create", "update", "delete"): method = getattr(namespace, verb, None) assert method is not None, f"the SDK namespace has no {verb}()" - takes = inspect.signature(method).parameters + takes = _signature_of(f"{verb}", method).parameters if verb in ("retrieve", "update", "delete"): assert id_kwarg in takes, f"{verb}() does not take {id_kwarg!r}" for name in scope: @@ -252,6 +252,7 @@ def test_the_feature_toggles_the_sdk_offers_are_all_reachable(): missing_flags = set(ProjectFeature.model_fields) - declared assert not missing_flags, f"ProjectFeature flags with no way to set them: {sorted(missing_flags)}" + PROPERTY_REFUSAL = HttpError( "Bad Request", status_code=400, response={"error": "This resource is managed at the workspace level"} ) diff --git a/tests/tools/test_spyclient_signatures.py b/tests/tools/test_spyclient_signatures.py new file mode 100644 index 0000000..72b3e26 --- /dev/null +++ b/tests/tools/test_spyclient_signatures.py @@ -0,0 +1,54 @@ +"""The spy must bind against an SDK method whose annotations will not evaluate. + +plane-sdk declares twelve methods as ``def list(self, ...) -> list[...]``. Under +Python 3.14's lazy annotations (PEP 649) the class namespace is in scope when the +annotation is evaluated, so ``list`` resolves to the method rather than the builtin +and subscripting it raises TypeError. That took out 17 tests across four files -- +none of which are about annotations -- because ``inspect.signature`` does the +evaluating. + +Nothing fails at runtime: the server advertises all 28 tools and a full eval battery +runs clean. The breakage is confined to test-time introspection. +""" + +import inspect + +from tests.tools._spyclient import UNEVALUATED_ANNOTATIONS, _signature_of + + +class ShadowingResource: + """The exact shape plane-sdk uses, reproduced so this test needs no SDK version.""" + + def list(self, workspace_slug: str) -> list[int]: + return [] + + +def test_the_shadowing_pattern_still_yields_a_bindable_signature(): + signature = _signature_of("shadow.list", ShadowingResource.list) + assert list(signature.parameters) == ["self", "workspace_slug"] + bound = signature.bind(ShadowingResource(), workspace_slug="acme") + assert bound.arguments["workspace_slug"] == "acme" + + +def test_a_degraded_signature_is_recorded_rather_than_silently_accepted(): + """A type-check that quietly checks nothing must not look like a passing one.""" + try: + inspect.signature(ShadowingResource.list) + except TypeError: + # This interpreter evaluates annotations eagerly here, so the fallback ran. + _signature_of("shadow.list", ShadowingResource.list) + assert "shadow.list" in UNEVALUATED_ANNOTATIONS + else: + # Pre-3.14: no fallback needed, so nothing should be recorded for it. + _signature_of("shadow.list", ShadowingResource.list) + assert "shadow.list" not in UNEVALUATED_ANNOTATIONS + + +def test_an_ordinary_signature_is_untouched(): + def plain(a: int, b: str = "x") -> bool: + return True + + signature = _signature_of("plain", plain) + assert list(signature.parameters) == ["a", "b"] + assert signature.parameters["a"].annotation is int + assert "plain" not in UNEVALUATED_ANNOTATIONS diff --git a/uv.lock b/uv.lock index 4605575..787279e 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anthropic" +version = "0.122.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/23/9987d70b74e3481d5bc5d2021d3e10fd5f60c1f7b54088ea86506d9b7f2b/anthropic-0.122.0.tar.gz", hash = "sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601", size = 1021535, upload-time = "2026-08-13T18:36:00.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/5b/f5c87e71097a9f89f1b414d1ef7ae8439051fae57d5e4ee90946082982b8/anthropic-0.122.0-py3-none-any.whl", hash = "sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67", size = 1041853, upload-time = "2026-08-13T18:36:01.831Z" }, +] + [[package]] name = "anyio" version = "4.12.1" @@ -429,6 +448,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/03/f906829bcfcbb945f19d6a64240ffb66a31d69ca5533e95882f0efc9c13c/cyclopts-4.5.2-py3-none-any.whl", hash = "sha256:ee56ee23c2c81abc34b66b5aa8fd2698ca699740054e84e534449ec3eb7f944d", size = 200165, upload-time = "2026-02-11T16:30:46.942Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -653,6 +681,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/d8/b959609e44012a42b1f3e5ba98ea3b33c7e41e6d4b77cd8f00fd19b1d3ad/jiter-0.16.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c", size = 310082, upload-time = "2026-06-29T13:02:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3d/4d7f5667ea0e0548534ba880b84bb3d12924fd133aa83ad6c6c80fca3d76/jiter-0.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b", size = 315643, upload-time = "2026-06-29T13:02:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/9b/83/bed2dcb5c9f3e1ccfcbc67dda48265fe7d5ad0c9cadda5fe95f6e3b87f94/jiter-0.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84", size = 341363, upload-time = "2026-06-29T13:02:34.853Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2f/6bb3c3dda668ebc0445689c81a2b0f26a82b10843d67ed9c9b2c3edc177f/jiter-0.16.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c", size = 365483, upload-time = "2026-06-29T13:02:36.295Z" }, + { url = "https://files.pythonhosted.org/packages/92/35/8a045ccb39164e70dcdae696413b661771f148b68b12b175c3a04d901937/jiter-0.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126", size = 461219, upload-time = "2026-06-29T13:02:38.116Z" }, + { url = "https://files.pythonhosted.org/packages/e7/99/22292dbbf0ed0c610cfe5ddc7f3bd67237a412f121318f865196e62a07bd/jiter-0.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c", size = 374905, upload-time = "2026-06-29T13:02:40.357Z" }, + { url = "https://files.pythonhosted.org/packages/29/ac/2f55ccb1f0eeafa6d89d24caf52f6f0944a59290ee199e9ade62177dca42/jiter-0.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de", size = 348320, upload-time = "2026-06-29T13:02:41.923Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/7d88b9174c40064fabc07c84a9b62e6b10f5644562ec0e0a29392edbe978/jiter-0.16.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244", size = 356519, upload-time = "2026-06-29T13:02:43.436Z" }, + { url = "https://files.pythonhosted.org/packages/27/57/c4a33aeef513a9d5e26e31534e0bcc752d6ea0e54c94ddb7b68bade669c2/jiter-0.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f", size = 394204, upload-time = "2026-06-29T13:02:44.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c6c23e76ebb3766b111bc399437bbc9f870a76e2a92e10b2a5f561d57372/jiter-0.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131", size = 521477, upload-time = "2026-06-29T13:02:46.724Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d3/0001c8c0c5976af2625bb1cfb1895e8ec693b6589fe4574b8e6fc2c85501/jiter-0.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b", size = 552187, upload-time = "2026-06-29T13:02:48.144Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/311b718e07e85740e48619c0632b36f7e0b8d113984499e436452ed13a9a/jiter-0.16.0-cp310-cp310-win32.whl", hash = "sha256:b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9", size = 206513, upload-time = "2026-06-29T13:02:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/db/7f/ac680eeb0777dc0eb7dc824800ba27880d7f6bc712e362d34ad8ee559f36/jiter-0.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26", size = 199505, upload-time = "2026-06-29T13:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3f/fae6cc967d120ec89e31c5418a51176d8278b3087fbb384a9176754f353c/jiter-0.16.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3", size = 309289, upload-time = "2026-06-29T13:02:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/97c6c3562c077f6247d6e6ce5c82562500b6316c0d928e97e106b7a1321a/jiter-0.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e", size = 315181, upload-time = "2026-06-29T13:02:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/7b/89/d8d073f8aa2667e46c6c0873f86fe4a512bba4293cc730f626a076211a62/jiter-0.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037", size = 340939, upload-time = "2026-06-29T13:02:55.412Z" }, + { url = "https://files.pythonhosted.org/packages/87/c9/db4fda3ed73fb864139305e935e5b8b38a5a24692a5a9dd356c22f1b9c8d/jiter-0.16.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e", size = 364932, upload-time = "2026-06-29T13:02:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/a2/74/52b5e86241057f52ddd7c9a580f90effb51f9d06239f6fc612279b91a838/jiter-0.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b", size = 461132, upload-time = "2026-06-29T13:02:58.994Z" }, + { url = "https://files.pythonhosted.org/packages/a9/87/544a700f7447c1f31c5d7833821a4daa5683165c2d5a094fbf5b5800c3dc/jiter-0.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c", size = 374857, upload-time = "2026-06-29T13:03:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/40/cd/0fcc3f7d39183674d5bfa9ec640faaeb506c60be7c8f94625dfba366e37c/jiter-0.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee", size = 347053, upload-time = "2026-06-29T13:03:02.045Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/c7e64e7932ad597fa395b61440b249ada6366716e25c6e08dd2afbd021e6/jiter-0.16.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03", size = 356153, upload-time = "2026-06-29T13:03:03.706Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/1c719044f14da814e1a060191ab19b96f3e99207bc5b4bfc6d6be34b3f80/jiter-0.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea", size = 393956, upload-time = "2026-06-29T13:03:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/7b2f303a2847207e265503853a2d964a55354cffd62a5f2936c155486798/jiter-0.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91", size = 521081, upload-time = "2026-06-29T13:03:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/501cf6e1e09caeb420195179ffc6f62aca603f1220ec53fd80d0d70b3e56/jiter-0.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3", size = 552085, upload-time = "2026-06-29T13:03:08.339Z" }, + { url = "https://files.pythonhosted.org/packages/79/54/aa5be86520113b79455c3877f3d1f07a348098df4083ba3688e9537e52dd/jiter-0.16.0-cp311-cp311-win32.whl", hash = "sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7", size = 206755, upload-time = "2026-06-29T13:03:09.653Z" }, + { url = "https://files.pythonhosted.org/packages/64/ec/2feb893eb330bd69b413866f4d5daada33c3962f1c6f270c91ca2d87fdf9/jiter-0.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1", size = 199155, upload-time = "2026-06-29T13:03:10.979Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9c/ca040d94415048a3666fc237774df8151c96f8d2b661cbe3b184acc95876/jiter-0.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056", size = 194403, upload-time = "2026-06-29T13:03:12.341Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/06/d3/8e278946d43eeca2585b4dd0834a887cd71136329b837f3a16ed86a8b4b0/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad", size = 304518, upload-time = "2026-06-29T13:05:00.172Z" }, + { url = "https://files.pythonhosted.org/packages/72/43/28d4ef495028bf0506a413d4db3f4eb3e7288a382e0f065f306a17bbeb5e/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f", size = 310207, upload-time = "2026-06-29T13:05:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ca/c366b1012da1d640de975d9683acd44e4d150d9068845d0ca2610435253f/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5", size = 342771, upload-time = "2026-06-29T13:05:03.55Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/50cc4056fc1ae02e7154704e7ecc89df0afb8300222cfe8a52d3f67e4730/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85", size = 346468, upload-time = "2026-06-29T13:05:05.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "jmespath" version = "1.1.0" @@ -912,9 +1039,13 @@ dev = [ { name = "pytest" }, { name = "ruff" }, ] +evals = [ + { name = "anthropic" }, +] [package.metadata] requires-dist = [ + { name = "anthropic", marker = "extra == 'evals'", specifier = ">=0.121.0" }, { name = "authlib", specifier = ">=1.6.9" }, { name = "boto3", specifier = ">=1.34.0" }, { name = "fakeredis", extras = ["lua"], specifier = ">=2.32.1,<2.35.0" }, @@ -926,7 +1057,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "evals"] [[package]] name = "plane-sdk" @@ -1574,6 +1705,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0"