diff --git a/.pr/evidence/analysis.md b/.pr/evidence/analysis.md new file mode 100644 index 0000000000..580f453743 --- /dev/null +++ b/.pr/evidence/analysis.md @@ -0,0 +1,61 @@ +# Evidence: per-call prompt composition on live agent runs + +> `.pr/` is PR-only reviewer context per repository policy (`.github/workflows/pr-artifacts.yml`); +> this directory is removed after PR approval. Nothing here is meant to merge to `main`. + +Per-call records from live runs of the default agent (`get_default_agent`, 19 tools, +browser-on) on tasks from the published harness-benchmark short suite +(). Each `calls.jsonl` row carries the +`PromptComposition` estimate and the provider-reported `TokenUsage` for one LLM call +(counts and response ids only; no message content, no credentials). + +## Lanes + +- `minimax-m3/` — `openai/MiniMax-M3` (chat completions; litellm has no tokenizer mapping for + this model, so estimates use litellm's fallback tokenizer). Tasks p09-task-01 (trivial + rename, 5 calls), p09-task-07 (medium refactor, 26 calls), p09-task-10 (hard cache + implementation, 10 calls). +- `gpt-4o-mini/` — `gpt-4o-mini` (chat completions; litellm maps this model to its real + `o200k_base` tokenizer — verified by comparing `token_counter` framing overhead against + `tiktoken.get_encoding("o200k_base")` vs `cl100k_base` on divergent inputs). Tasks + p09-task-01 (9 calls, verifier PASS) and p09-task-07 (28 calls, verifier FAIL — the model + left a syntax error in the repo; a completed run, labeled a model-quality failure). + +## gpt-4o-mini summary (mapped tokenizer) + +| task | calls | avg system | avg tools | avg history | avg latest | avg provider input | est/provider median | +|---|---:|---:|---:|---:|---:|---:|---:| +| p09-task-01 | 9 | 3,340 | 5,702 | 987 | 175 | 10,231 | 1.00 | +| p09-task-07 | 28 | 3,340 | 5,702 | 18,357 | 792 | 28,739 | 0.98 | + +- Per-call est/provider ratio band: 0.99–1.00 (task-01), 0.97–0.99 (task-07) — versus + 0.85–0.97 on the unmapped MiniMax-M3 lane. The mapped-tokenizer band tightens to ~1.0, so + the MiniMax underestimate was dominated by the fallback tokenizer; a residual ~1–3% + underestimate remains, consistent with uncounted request framing and litellm's tool + serialization convention. +- One notable event: between calls 9 and 10 of task-07, history dropped 30.8K → 10.0K tokens + with **no intervening LLM call** — view-property enforcement + (`View.enforce_properties`: batch/observation/atomicity properties drop events without a + summarization call), not an LLM-summarizing condensation. The composition tracked the + shrunken view exactly (ratio stayed 0.99 across the drop), which is itself evidence the + history bucket measures the payload actually sent. +- Total spend for this lane (litellm-computed `accumulated_cost`, PAYG key): $0.0081 + (task-01) + $0.0808 (task-07) ≈ **$0.089**; 87% of prompt tokens were cache reads. + + +## MiniMax-M3 summary + +| task | calls | avg system | avg tools | avg history | avg latest | avg provider input | est/provider median | +|---|---:|---:|---:|---:|---:|---:|---:| +| p09-task-01 | 5 | 3,331 | 5,749 | 533 | 205 | 11,478 | 0.85 | +| p09-task-07 | 26 | 3,331 | 5,749 | 10,462 | 381 | 21,176 | 0.94 | +| p09-task-10 | 10 | 3,331 | 5,749 | 2,605 | 215 | 13,506 | 0.86 | + +- The standing preamble (system + tool schemas ≈ 9,080 estimated tokens) is constant per call: + ~79% of the average call on the trivial task (avg provider input 11,478), ~84% of first calls. +- Re-sent tool schemas totaled 149,474 estimated tokens on the 26-call task — 27% of that run's + provider-reported input (550,568). +- Estimated component sum per call runs 0.85–0.97× the provider-reported `prompt_tokens`, + rising as history grows. The residual gap is consistent with request framing and tokenizer + mapping differences; the component split is the finding, not the absolute sum. +- MiniMax's automatic prefix caching served 79–94% of prompt tokens as cache reads. diff --git a/.pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl b/.pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl new file mode 100644 index 0000000000..de1c8904fe --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl @@ -0,0 +1,9 @@ +{"seq": 0, "elapsed_s": 3.12, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9172, "completion_tokens": 38, "cache_read_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9210, "response_id": "chatcmpl-EGVEG8NSZIEywDIyNQnqt99X5R4tJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 0, "latest_message_tokens": 38, "is_estimate": true, "response_id": "chatcmpl-EGVEG8NSZIEywDIyNQnqt99X5R4tJ"}, "latency_s": 2.5493357181549072} +{"seq": 1, "elapsed_s": 4.67, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9675, "completion_tokens": 63, "cache_read_tokens": 9088, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9738, "response_id": "chatcmpl-EGVEICEloV8BjwzBVpfYkPDKSNhVb"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 73, "latest_message_tokens": 486, "is_estimate": true, "response_id": "chatcmpl-EGVEICEloV8BjwzBVpfYkPDKSNhVb"}, "latency_s": 1.5301110744476318} +{"seq": 2, "elapsed_s": 7.12, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9797, "completion_tokens": 61, "cache_read_tokens": 9600, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9858, "response_id": "chatcmpl-EGVEJHP3YvqiBxIXWUbhpgqhTOhE7"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 620, "latest_message_tokens": 77, "is_estimate": true, "response_id": "chatcmpl-EGVEJHP3YvqiBxIXWUbhpgqhTOhE7"}, "latency_s": 2.4455978870391846} +{"seq": 3, "elapsed_s": 8.73, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10054, "completion_tokens": 63, "cache_read_tokens": 9728, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10117, "response_id": "chatcmpl-EGVEMHTRF4V3DUDvosBEZHNMSya0m"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 755, "latest_message_tokens": 213, "is_estimate": true, "response_id": "chatcmpl-EGVEMHTRF4V3DUDvosBEZHNMSya0m"}, "latency_s": 1.593902826309204} +{"seq": 4, "elapsed_s": 10.05, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10309, "completion_tokens": 63, "cache_read_tokens": 9984, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10372, "response_id": "chatcmpl-EGVENFynBkjyDGNnJZvpU8I5yjIcr"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1028, "latest_message_tokens": 212, "is_estimate": true, "response_id": "chatcmpl-EGVENFynBkjyDGNnJZvpU8I5yjIcr"}, "latency_s": 1.288510799407959} +{"seq": 5, "elapsed_s": 10.95, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10539, "completion_tokens": 32, "cache_read_tokens": 10240, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10571, "response_id": "chatcmpl-EGVEO5r1bOirNFiH5qiMMXW33T1og"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1301, "latest_message_tokens": 187, "is_estimate": true, "response_id": "chatcmpl-EGVEO5r1bOirNFiH5qiMMXW33T1og"}, "latency_s": 0.8807761669158936} +{"seq": 6, "elapsed_s": 13.23, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10669, "completion_tokens": 29, "cache_read_tokens": 10496, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10698, "response_id": "chatcmpl-EGVEQBtKdhnHab1gSiwzkourerokO"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1517, "latest_message_tokens": 118, "is_estimate": true, "response_id": "chatcmpl-EGVEQBtKdhnHab1gSiwzkourerokO"}, "latency_s": 1.3414945602416992} +{"seq": 7, "elapsed_s": 15.53, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10787, "completion_tokens": 97, "cache_read_tokens": 10624, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10884, "response_id": "chatcmpl-EGVESk7f9F0Vx4trTv6Cr3B7g2hM8"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1661, "latest_message_tokens": 110, "is_estimate": true, "response_id": "chatcmpl-EGVESk7f9F0Vx4trTv6Cr3B7g2hM8"}, "latency_s": 1.62056303024292} +{"seq": 8, "elapsed_s": 18.22, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 11074, "completion_tokens": 57, "cache_read_tokens": 10752, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 11131, "response_id": "chatcmpl-EGVEVrPlyWeD92ZjuW892YFNWzZvj"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1929, "latest_message_tokens": 135, "is_estimate": true, "response_id": "chatcmpl-EGVEVrPlyWeD92ZjuW892YFNWzZvj"}, "latency_s": 1.2971453666687012} diff --git a/.pr/evidence/gpt-4o-mini/p09-task-01-summary.json b/.pr/evidence/gpt-4o-mini/p09-task-01-summary.json new file mode 100644 index 0000000000..514d3a50db --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-01-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-01", + "prompt": "Rename the local variable `raw_items` to `source_items` in `toyapp/pagination.py`. Keep behavior unchanged and do not edit unrelated files.", + "model": "gpt-4o-mini", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 18.22, + "max_iterations_cap": 30, + "calls_recorded": 9, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0080748, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 92076, + "completion_tokens": 503, + "cache_read_tokens": 80512, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 11131, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl b/.pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl new file mode 100644 index 0000000000..6616c63d8c --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl @@ -0,0 +1,28 @@ +{"seq": 0, "elapsed_s": 2.19, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9229, "completion_tokens": 54, "cache_read_tokens": 3968, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9283, "response_id": "chatcmpl-EGVEaBHtEAXFwTfJuqnKMyvx8qt2z"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 0, "latest_message_tokens": 95, "is_estimate": true, "response_id": "chatcmpl-EGVEaBHtEAXFwTfJuqnKMyvx8qt2z"}, "latency_s": 1.6030583381652832} +{"seq": 1, "elapsed_s": 65.76, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 11615, "completion_tokens": 6265, "cache_read_tokens": 9216, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 17880, "response_id": "chatcmpl-EGVEbVMTDkqWJ4ho6EWwfTYTYLX7V"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 150, "latest_message_tokens": 2352, "is_estimate": true, "response_id": "chatcmpl-EGVEbVMTDkqWJ4ho6EWwfTYTYLX7V"}, "latency_s": 63.55660820007324} +{"seq": 2, "elapsed_s": 68.08, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 20060, "completion_tokens": 39, "cache_read_tokens": 17792, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 20099, "response_id": "chatcmpl-EGVFdiOXjmtDk4kIMmRdtl4UGj3jt"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10561, "latest_message_tokens": 216, "is_estimate": true, "response_id": "chatcmpl-EGVFdiOXjmtDk4kIMmRdtl4UGj3jt"}, "latency_s": 2.2881739139556885} +{"seq": 3, "elapsed_s": 91.85, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 22458, "completion_tokens": 2420, "cache_read_tokens": 19968, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 24878, "response_id": "chatcmpl-EGVFffVj591GMksiyQGFGh5v17bTa"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10810, "latest_message_tokens": 2380, "is_estimate": true, "response_id": "chatcmpl-EGVFffVj591GMksiyQGFGh5v17bTa"}, "latency_s": 23.761120080947876} +{"seq": 4, "elapsed_s": 114.08, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 24943, "completion_tokens": 2464, "cache_read_tokens": 22656, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 27407, "response_id": "chatcmpl-EGVG3sUfYwhR45fL1Ih0CxqVB21Rg"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 15589, "latest_message_tokens": 97, "is_estimate": true, "response_id": "chatcmpl-EGVG3sUfYwhR45fL1Ih0CxqVB21Rg"}, "latency_s": 22.207499265670776} +{"seq": 5, "elapsed_s": 115.49, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 32129, "completion_tokens": 58, "cache_read_tokens": 27264, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 32187, "response_id": "chatcmpl-EGVGPFc4ebzbYSDpp2rBoOlsTSvqF"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 21761, "latest_message_tokens": 509, "is_estimate": true, "response_id": "chatcmpl-EGVGPFc4ebzbYSDpp2rBoOlsTSvqF"}, "latency_s": 1.3639793395996094} +{"seq": 6, "elapsed_s": 116.7, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 36633, "completion_tokens": 34, "cache_read_tokens": 32128, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 36667, "response_id": "chatcmpl-EGVGRGyXamE9eBbbF4Qj3Gznwa1fo"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 22326, "latest_message_tokens": 4467, "is_estimate": true, "response_id": "chatcmpl-EGVGRGyXamE9eBbbF4Qj3Gznwa1fo"}, "latency_s": 1.177990436553955} +{"seq": 7, "elapsed_s": 128.0, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 37133, "completion_tokens": 956, "cache_read_tokens": 36608, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 38089, "response_id": "chatcmpl-EGVGTJYxwCVP3edBFNELpYeZsXWjJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 26824, "latest_message_tokens": 484, "is_estimate": true, "response_id": "chatcmpl-EGVGTJYxwCVP3edBFNELpYeZsXWjJ"}, "latency_s": 10.288436889648438} +{"seq": 8, "elapsed_s": 129.14, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 40656, "completion_tokens": 34, "cache_read_tokens": 37888, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 40690, "response_id": "chatcmpl-EGVGdViETiz1nvcAvkIdeyzAsfvWX"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 30436, "latest_message_tokens": 322, "is_estimate": true, "response_id": "chatcmpl-EGVGdViETiz1nvcAvkIdeyzAsfvWX"}, "latency_s": 1.0821290016174316} +{"seq": 9, "elapsed_s": 139.51, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 41156, "completion_tokens": 961, "cache_read_tokens": 40576, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 42117, "response_id": "chatcmpl-EGVGfqL1j1yIm0mHUoprQQMM8JsYq"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 30789, "latest_message_tokens": 485, "is_estimate": true, "response_id": "chatcmpl-EGVGfqL1j1yIm0mHUoprQQMM8JsYq"}, "latency_s": 9.654026985168457} +{"seq": 10, "elapsed_s": 145.03, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 19579, "completion_tokens": 35, "cache_read_tokens": 11008, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 19614, "response_id": "chatcmpl-EGVGteyULujx1MocC9a9i12g22BMd"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10028, "latest_message_tokens": 323, "is_estimate": true, "response_id": "chatcmpl-EGVGteyULujx1MocC9a9i12g22BMd"}, "latency_s": 1.008913278579712} +{"seq": 11, "elapsed_s": 168.16, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 20067, "completion_tokens": 943, "cache_read_tokens": 19584, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 21010, "response_id": "chatcmpl-EGVGvliksgwWU5TTpeW0QiW5yRQXa"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10383, "latest_message_tokens": 475, "is_estimate": true, "response_id": "chatcmpl-EGVGvliksgwWU5TTpeW0QiW5yRQXa"}, "latency_s": 22.469878435134888} +{"seq": 12, "elapsed_s": 169.26, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 23599, "completion_tokens": 39, "cache_read_tokens": 20864, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 23638, "response_id": "chatcmpl-EGVHIKWrVQFCZN8GphndiPs28RtVQ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 14000, "latest_message_tokens": 323, "is_estimate": true, "response_id": "chatcmpl-EGVHIKWrVQFCZN8GphndiPs28RtVQ"}, "latency_s": 1.0678575038909912} +{"seq": 13, "elapsed_s": 173.16, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 24104, "completion_tokens": 193, "cache_read_tokens": 23552, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 24297, "response_id": "chatcmpl-EGVHJfDADcTSqSwejg976KyrLZfrg"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 14359, "latest_message_tokens": 486, "is_estimate": true, "response_id": "chatcmpl-EGVHJfDADcTSqSwejg976KyrLZfrg"}, "latency_s": 3.2294814586639404} +{"seq": 14, "elapsed_s": 179.02, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 28754, "completion_tokens": 418, "cache_read_tokens": 24192, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 29172, "response_id": "chatcmpl-EGVHNAjkt5ocTthcGC8GJEX3YiHBp"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 15035, "latest_message_tokens": 4474, "is_estimate": true, "response_id": "chatcmpl-EGVHNAjkt5ocTthcGC8GJEX3YiHBp"}, "latency_s": 5.823587894439697} +{"seq": 15, "elapsed_s": 201.98, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 29587, "completion_tokens": 1273, "cache_read_tokens": 29056, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 30860, "response_id": "chatcmpl-EGVHSp7sw6cWsbwWqzVkuZ61XGxEJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 19924, "latest_message_tokens": 431, "is_estimate": true, "response_id": "chatcmpl-EGVHSp7sw6cWsbwWqzVkuZ61XGxEJ"}, "latency_s": 22.9333074092865} +{"seq": 16, "elapsed_s": 210.44, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 33078, "completion_tokens": 801, "cache_read_tokens": 30720, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 33879, "response_id": "chatcmpl-EGVHpcFMO3Xuvsg0wYFUXbZdpIknq"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 22879, "latest_message_tokens": 626, "is_estimate": true, "response_id": "chatcmpl-EGVHpcFMO3Xuvsg0wYFUXbZdpIknq"}, "latency_s": 8.41625690460205} +{"seq": 17, "elapsed_s": 211.58, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 35244, "completion_tokens": 39, "cache_read_tokens": 33792, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 35283, "response_id": "chatcmpl-EGVHyhnq2Dh8HwsuKrH64dRsymNst"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 25093, "latest_message_tokens": 429, "is_estimate": true, "response_id": "chatcmpl-EGVHyhnq2Dh8HwsuKrH64dRsymNst"}, "latency_s": 1.1085383892059326} +{"seq": 18, "elapsed_s": 216.29, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 35749, "completion_tokens": 349, "cache_read_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 36098, "response_id": "chatcmpl-EGVI0Nk3hyb6cbfgNTT7Dvla2P2lr"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 25558, "latest_message_tokens": 484, "is_estimate": true, "response_id": "chatcmpl-EGVI0Nk3hyb6cbfgNTT7Dvla2P2lr"}, "latency_s": 3.9800806045532227} +{"seq": 19, "elapsed_s": 230.63, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 36457, "completion_tokens": 1555, "cache_read_tokens": 35200, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 38012, "response_id": "chatcmpl-EGVI4tcM0letNC8zc3HFDqMPejIFm"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 26388, "latest_message_tokens": 378, "is_estimate": true, "response_id": "chatcmpl-EGVI4tcM0letNC8zc3HFDqMPejIFm"}, "latency_s": 14.30324649810791} +{"seq": 20, "elapsed_s": 235.18, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 28700, "completion_tokens": 39, "cache_read_tokens": 11136, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 28739, "response_id": "chatcmpl-EGVIL4IKSW9ZUQG1Vp5utKESF602P"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 18234, "latest_message_tokens": 539, "is_estimate": true, "response_id": "chatcmpl-EGVIL4IKSW9ZUQG1Vp5utKESF602P"}, "latency_s": 1.4693710803985596} +{"seq": 21, "elapsed_s": 238.2, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 29171, "completion_tokens": 177, "cache_read_tokens": 28672, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 29348, "response_id": "chatcmpl-EGVINa4VImNOx3jKptTsWgIJGXWEJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 18809, "latest_message_tokens": 452, "is_estimate": true, "response_id": "chatcmpl-EGVINa4VImNOx3jKptTsWgIJGXWEJ"}, "latency_s": 2.2953388690948486} +{"seq": 22, "elapsed_s": 241.14, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 29507, "completion_tokens": 272, "cache_read_tokens": 29312, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 29779, "response_id": "chatcmpl-EGVIQUULx7H0HkPwvSwPBFtBVO30I"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 19438, "latest_message_tokens": 174, "is_estimate": true, "response_id": "chatcmpl-EGVIQUULx7H0HkPwvSwPBFtBVO30I"}, "latency_s": 2.912266254425049} +{"seq": 23, "elapsed_s": 243.73, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 30032, "completion_tokens": 194, "cache_read_tokens": 29696, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 30226, "response_id": "chatcmpl-EGVITdkLHH8ywZBoOXP2kSLLWkRQ3"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 19881, "latest_message_tokens": 267, "is_estimate": true, "response_id": "chatcmpl-EGVITdkLHH8ywZBoOXP2kSLLWkRQ3"}, "latency_s": 2.567747116088867} +{"seq": 24, "elapsed_s": 248.38, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 30290, "completion_tokens": 457, "cache_read_tokens": 30208, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 30747, "response_id": "chatcmpl-EGVIVTZSaF7UByQ3SZYHlLMsk5pys"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 20338, "latest_message_tokens": 84, "is_estimate": true, "response_id": "chatcmpl-EGVIVTZSaF7UByQ3SZYHlLMsk5pys"}, "latency_s": 4.613501787185669} +{"seq": 25, "elapsed_s": 252.03, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 31089, "completion_tokens": 358, "cache_read_tokens": 30720, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 31447, "response_id": "chatcmpl-EGVIa2nGNiupew3LwWAn7M5KZeEhT"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 20876, "latest_message_tokens": 359, "is_estimate": true, "response_id": "chatcmpl-EGVIa2nGNiupew3LwWAn7M5KZeEhT"}, "latency_s": 3.6165292263031006} +{"seq": 26, "elapsed_s": 254.49, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 31538, "completion_tokens": 246, "cache_read_tokens": 31360, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 31784, "response_id": "chatcmpl-EGVIdfFtb5K0KEW7cjgnPbJWL7S2w"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 21588, "latest_message_tokens": 110, "is_estimate": true, "response_id": "chatcmpl-EGVIdfFtb5K0KEW7cjgnPbJWL7S2w"}, "latency_s": 2.427025079727173} +{"seq": 27, "elapsed_s": 256.43, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 32123, "completion_tokens": 132, "cache_read_tokens": 31744, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 32255, "response_id": "chatcmpl-EGVIgAl8Pz7WD9Vi7Nr6KLIBTiC3T"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 21942, "latest_message_tokens": 359, "is_estimate": true, "response_id": "chatcmpl-EGVIgAl8Pz7WD9Vi7Nr6KLIBTiC3T"}, "latency_s": 1.9118013381958008} diff --git a/.pr/evidence/gpt-4o-mini/p09-task-07-summary.json b/.pr/evidence/gpt-4o-mini/p09-task-07-summary.json new file mode 100644 index 0000000000..5bfff7af48 --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-07-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-07", + "prompt": "Refactor the long `build_monthly_report` function in `toyapp/reports.py` into smaller helpers or modules. Preserve behavior and keep the tests passing. Preserve this public contract:\n\n```python\nfrom toyapp.reports import build_monthly_report\nreport = build_monthly_report(rows, \"2026-05\")\nassert report[\"totals\"][\"revenue\"].startswith(\"$\")\nassert \"narrative\" in report\n```", + "model": "gpt-4o-mini", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.ERROR", + "wall_clock_s": 256.44, + "max_iterations_cap": 30, + "calls_recorded": 28, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.080769, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 804680, + "completion_tokens": 20805, + "cache_read_tokens": 698880, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 32255, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/minimax-m3/p09-task-01-calls.jsonl b/.pr/evidence/minimax-m3/p09-task-01-calls.jsonl new file mode 100644 index 0000000000..c0cb064222 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-01-calls.jsonl @@ -0,0 +1,5 @@ +{"seq": 0, "elapsed_s": 2.52, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10753, "completion_tokens": 106, "cache_read_tokens": 128, "cache_write_tokens": 0, "reasoning_tokens": 44, "context_window": 0, "per_turn_token": 10859, "response_id": "06dbb5db9493db12fb8930727ddce1c6"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 0, "latest_message_tokens": 38, "is_estimate": true, "response_id": "06dbb5db9493db12fb8930727ddce1c6"}, "latency_s": 2.0401830673217773} +{"seq": 1, "elapsed_s": 4.79, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10906, "completion_tokens": 80, "cache_read_tokens": 10752, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 10986, "response_id": "06dbb5de6e4f3bd67135995a5c4ddb3b"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 84, "latest_message_tokens": 100, "is_estimate": true, "response_id": "06dbb5de6e4f3bd67135995a5c4ddb3b"}, "latency_s": 1.3546624183654785} +{"seq": 2, "elapsed_s": 7.44, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11463, "completion_tokens": 298, "cache_read_tokens": 10905, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11761, "response_id": "06dbb5df31a5efed186e061448cc88c1"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 232, "latest_message_tokens": 487, "is_estimate": true, "response_id": "06dbb5df31a5efed186e061448cc88c1"}, "latency_s": 2.629474401473999} +{"seq": 3, "elapsed_s": 8.99, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12060, "completion_tokens": 105, "cache_read_tokens": 11462, "cache_write_tokens": 0, "reasoning_tokens": 34, "context_window": 0, "per_turn_token": 12165, "response_id": "06dbb5e27ea6bc960f36a696934697eb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 993, "latest_message_tokens": 313, "is_estimate": true, "response_id": "06dbb5e27ea6bc960f36a696934697eb"}, "latency_s": 1.5360901355743408} +{"seq": 4, "elapsed_s": 11.64, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12209, "completion_tokens": 146, "cache_read_tokens": 12059, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 12355, "response_id": "06dbb5e4e0e9b171812304c45290e354"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1357, "latest_message_tokens": 87, "is_estimate": true, "response_id": "06dbb5e4e0e9b171812304c45290e354"}, "latency_s": 1.942288875579834} diff --git a/.pr/evidence/minimax-m3/p09-task-01-summary.json b/.pr/evidence/minimax-m3/p09-task-01-summary.json new file mode 100644 index 0000000000..2ffd693954 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-01-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-01", + "prompt": "Rename the local variable `raw_items` to `source_items` in `toyapp/pagination.py`. Keep behavior unchanged and do not edit unrelated files.", + "model": "openai/MiniMax-M3", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 11.64, + "max_iterations_cap": 30, + "calls_recorded": 5, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "openai/MiniMax-M3", + "accumulated_cost": 0.0, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "openai/MiniMax-M3", + "prompt_tokens": 57391, + "completion_tokens": 735, + "cache_read_tokens": 45306, + "cache_write_tokens": 0, + "reasoning_tokens": 78, + "context_window": 0, + "per_turn_token": 12355, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/minimax-m3/p09-task-07-calls.jsonl b/.pr/evidence/minimax-m3/p09-task-07-calls.jsonl new file mode 100644 index 0000000000..29f876eef5 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-07-calls.jsonl @@ -0,0 +1,26 @@ +{"seq": 0, "elapsed_s": 2.54, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10811, "completion_tokens": 120, "cache_read_tokens": 128, "cache_write_tokens": 0, "reasoning_tokens": 23, "context_window": 0, "per_turn_token": 10931, "response_id": "06dbb621630bdaa73aa241d88930084e"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 0, "latest_message_tokens": 94, "is_estimate": true, "response_id": "06dbb621630bdaa73aa241d88930084e"}, "latency_s": 2.0828120708465576} +{"seq": 1, "elapsed_s": 4.85, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11323, "completion_tokens": 72, "cache_read_tokens": 10810, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11395, "response_id": "06dbb624284bcacf32032a62c2ce95a3"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 177, "latest_message_tokens": 425, "is_estimate": true, "response_id": "06dbb624284bcacf32032a62c2ce95a3"}, "latency_s": 1.36482572555542} +{"seq": 2, "elapsed_s": 7.08, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 13203, "completion_tokens": 72, "cache_read_tokens": 11322, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 13275, "response_id": "06dbb6262a0fbc14c7b77bd2476bca6b"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 646, "latest_message_tokens": 1755, "is_estimate": true, "response_id": "06dbb6262a0fbc14c7b77bd2476bca6b"}, "latency_s": 1.5264146327972412} +{"seq": 3, "elapsed_s": 9.27, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 13693, "completion_tokens": 86, "cache_read_tokens": 13202, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 13779, "response_id": "06dbb628602d744e87d44e34e71e2b45"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 2444, "latest_message_tokens": 426, "is_estimate": true, "response_id": "06dbb628602d744e87d44e34e71e2b45"}, "latency_s": 1.4912850856781006} +{"seq": 4, "elapsed_s": 11.31, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 13865, "completion_tokens": 82, "cache_read_tokens": 13692, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 13947, "response_id": "06dbb62ae649c708d544b9045b0acab6"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 2927, "latest_message_tokens": 101, "is_estimate": true, "response_id": "06dbb62ae649c708d544b9045b0acab6"}, "latency_s": 1.3661975860595703} +{"seq": 5, "elapsed_s": 54.25, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 14256, "completion_tokens": 6604, "cache_read_tokens": 13864, "cache_write_tokens": 0, "reasoning_tokens": 5978, "context_window": 0, "per_turn_token": 20860, "response_id": "06dbb62cd628b2fd953fc292e33aea2a"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 3082, "latest_message_tokens": 313, "is_estimate": true, "response_id": "06dbb62cd628b2fd953fc292e33aea2a"}, "latency_s": 42.22588348388672} +{"seq": 6, "elapsed_s": 56.91, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 15050, "completion_tokens": 245, "cache_read_tokens": 14255, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 15295, "response_id": "06dbb656b03ec23c38b033f0791abb70"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 4264, "latest_message_tokens": 33, "is_estimate": true, "response_id": "06dbb656b03ec23c38b033f0791abb70"}, "latency_s": 2.644223690032959} +{"seq": 7, "elapsed_s": 69.15, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 15320, "completion_tokens": 1888, "cache_read_tokens": 15049, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17208, "response_id": "06dbb659c94dbd6d07e428b2c47db4d5"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 4453, "latest_message_tokens": 40, "is_estimate": true, "response_id": "06dbb659c94dbd6d07e428b2c47db4d5"}, "latency_s": 12.213894844055176} +{"seq": 8, "elapsed_s": 71.36, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17250, "completion_tokens": 197, "cache_read_tokens": 15319, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17447, "response_id": "06dbb665f57c5f9f601120f350742337"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 6690, "latest_message_tokens": 58, "is_estimate": true, "response_id": "06dbb665f57c5f9f601120f350742337"}, "latency_s": 2.185944080352783} +{"seq": 9, "elapsed_s": 72.75, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17467, "completion_tokens": 73, "cache_read_tokens": 17249, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17540, "response_id": "06dbb668f2eba64bea7757fb744a7267"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 6933, "latest_message_tokens": 35, "is_estimate": true, "response_id": "06dbb668f2eba64bea7757fb744a7267"}, "latency_s": 1.374401569366455} +{"seq": 10, "elapsed_s": 84.08, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17612, "completion_tokens": 1640, "cache_read_tokens": 17466, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19252, "response_id": "06dbb66a29faddfe4528fc2feaa8992a"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 7013, "latest_message_tokens": 87, "is_estimate": true, "response_id": "06dbb66a29faddfe4528fc2feaa8992a"}, "latency_s": 10.641063451766968} +{"seq": 11, "elapsed_s": 85.59, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 19294, "completion_tokens": 94, "cache_read_tokens": 17611, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19388, "response_id": "06dbb6747d9a518fc781addbba4765c9"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9002, "latest_message_tokens": 57, "is_estimate": true, "response_id": "06dbb6747d9a518fc781addbba4765c9"}, "latency_s": 1.4767844676971436} +{"seq": 12, "elapsed_s": 87.79, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 19460, "completion_tokens": 89, "cache_read_tokens": 19293, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19549, "response_id": "06dbb67752070cf87aa20a0e988038a6"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9125, "latest_message_tokens": 89, "is_estimate": true, "response_id": "06dbb67752070cf87aa20a0e988038a6"}, "latency_s": 1.4858717918395996} +{"seq": 13, "elapsed_s": 90.05, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 19833, "completion_tokens": 83, "cache_read_tokens": 19459, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19916, "response_id": "06dbb6798dc63f0bd8b424e2ee26afea"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9273, "latest_message_tokens": 299, "is_estimate": true, "response_id": "06dbb6798dc63f0bd8b424e2ee26afea"}, "latency_s": 1.5481047630310059} +{"seq": 14, "elapsed_s": 92.88, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 20188, "completion_tokens": 193, "cache_read_tokens": 19832, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 20381, "response_id": "06dbb67bcf1f0244ac14f1baddbfcd3e"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9626, "latest_message_tokens": 276, "is_estimate": true, "response_id": "06dbb67bcf1f0244ac14f1baddbfcd3e"}, "latency_s": 2.143434524536133} +{"seq": 15, "elapsed_s": 110.16, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 20469, "completion_tokens": 2444, "cache_read_tokens": 20187, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 22913, "response_id": "06dbb67e3a44c9d91f0fc06f5b31b731"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 10071, "latest_message_tokens": 105, "is_estimate": true, "response_id": "06dbb67e3a44c9d91f0fc06f5b31b731"}, "latency_s": 16.57303738594055} +{"seq": 16, "elapsed_s": 117.4, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 25395, "completion_tokens": 746, "cache_read_tokens": 20468, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 26141, "response_id": "06dbb68f3d7b0f4f2468198ae9202206"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 12870, "latest_message_tokens": 2410, "is_estimate": true, "response_id": "06dbb68f3d7b0f4f2468198ae9202206"}, "latency_s": 6.5370707511901855} +{"seq": 17, "elapsed_s": 120.53, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 26161, "completion_tokens": 355, "cache_read_tokens": 25394, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 26516, "response_id": "06dbb69693143fae5edaca25846e90cb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 16078, "latest_message_tokens": 33, "is_estimate": true, "response_id": "06dbb69693143fae5edaca25846e90cb"}, "latency_s": 3.1108596324920654} +{"seq": 18, "elapsed_s": 134.56, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 26785, "completion_tokens": 2134, "cache_read_tokens": 26160, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 28919, "response_id": "06dbb6993da646d7823b2a8885d1edcb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 16439, "latest_message_tokens": 288, "is_estimate": true, "response_id": "06dbb6993da646d7823b2a8885d1edcb"}, "latency_s": 13.99971318244934} +{"seq": 19, "elapsed_s": 137.24, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29001, "completion_tokens": 136, "cache_read_tokens": 26784, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29137, "response_id": "06dbb6a82af4672636c66a1084e0fde5"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19076, "latest_message_tokens": 97, "is_estimate": true, "response_id": "06dbb6a82af4672636c66a1084e0fde5"}, "latency_s": 1.9628024101257324} +{"seq": 20, "elapsed_s": 139.43, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29370, "completion_tokens": 81, "cache_read_tokens": 29000, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29451, "response_id": "06dbb6aa6a4d3226286d24c4219bfbbb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19280, "latest_message_tokens": 248, "is_estimate": true, "response_id": "06dbb6aa6a4d3226286d24c4219bfbbb"}, "latency_s": 1.5076961517333984} +{"seq": 21, "elapsed_s": 142.08, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29656, "completion_tokens": 151, "cache_read_tokens": 29369, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29807, "response_id": "06dbb6ac589e63f83385ab634cb477d3"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19581, "latest_message_tokens": 213, "is_estimate": true, "response_id": "06dbb6ac589e63f83385ab634cb477d3"}, "latency_s": 1.9515719413757324} +{"seq": 22, "elapsed_s": 144.38, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29900, "completion_tokens": 98, "cache_read_tokens": 29655, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29998, "response_id": "06dbb6afe525c1edc3a6dc75b999a0e4"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19921, "latest_message_tokens": 109, "is_estimate": true, "response_id": "06dbb6afe525c1edc3a6dc75b999a0e4"}, "latency_s": 1.608078956604004} +{"seq": 23, "elapsed_s": 146.67, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 30077, "completion_tokens": 85, "cache_read_tokens": 29899, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 30162, "response_id": "06dbb6b10c6b5e003ed4301422782c43"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 20100, "latest_message_tokens": 97, "is_estimate": true, "response_id": "06dbb6b10c6b5e003ed4301422782c43"}, "latency_s": 1.5816996097564697} +{"seq": 24, "elapsed_s": 150.05, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 32388, "completion_tokens": 328, "cache_read_tokens": 30076, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 32716, "response_id": "06dbb6b35357b2e3833200731df62b36"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 20250, "latest_message_tokens": 2182, "is_estimate": true, "response_id": "06dbb6b35357b2e3833200731df62b36"}, "latency_s": 3.3420867919921875} +{"seq": 25, "elapsed_s": 156.88, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 32741, "completion_tokens": 644, "cache_read_tokens": 32387, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 33385, "response_id": "06dbb6b67ec8afa4f8aa435b7ad5ffa5"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 22679, "latest_message_tokens": 38, "is_estimate": true, "response_id": "06dbb6b67ec8afa4f8aa435b7ad5ffa5"}, "latency_s": 6.802237033843994} diff --git a/.pr/evidence/minimax-m3/p09-task-07-summary.json b/.pr/evidence/minimax-m3/p09-task-07-summary.json new file mode 100644 index 0000000000..51f1c4ced7 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-07-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-07", + "prompt": "Refactor the long `build_monthly_report` function in `toyapp/reports.py` into smaller helpers or modules. Preserve behavior and keep the tests passing. Preserve this public contract:\n\n```python\nfrom toyapp.reports import build_monthly_report\nreport = build_monthly_report(rows, \"2026-05\")\nassert report[\"totals\"][\"revenue\"].startswith(\"$\")\nassert \"narrative\" in report\n```", + "model": "openai/MiniMax-M3", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 156.89, + "max_iterations_cap": 30, + "calls_recorded": 26, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "openai/MiniMax-M3", + "accumulated_cost": 0.0, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "openai/MiniMax-M3", + "prompt_tokens": 550568, + "completion_tokens": 18740, + "cache_read_tokens": 517930, + "cache_write_tokens": 0, + "reasoning_tokens": 6001, + "context_window": 0, + "per_turn_token": 33385, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/minimax-m3/p09-task-10-calls.jsonl b/.pr/evidence/minimax-m3/p09-task-10-calls.jsonl new file mode 100644 index 0000000000..5372fbb199 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-10-calls.jsonl @@ -0,0 +1,10 @@ +{"seq": 0, "elapsed_s": 3.71, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10979, "completion_tokens": 113, "cache_read_tokens": 2304, "cache_write_tokens": 0, "reasoning_tokens": 13, "context_window": 0, "per_turn_token": 11092, "response_id": "06dbb6c02df6125128fa2f99d51206bd"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 0, "latest_message_tokens": 255, "is_estimate": true, "response_id": "06dbb6c02df6125128fa2f99d51206bd"}, "latency_s": 3.247368335723877} +{"seq": 1, "elapsed_s": 6.05, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11453, "completion_tokens": 78, "cache_read_tokens": 10880, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11531, "response_id": "06dbb6c51037d633135dfb26db1fe400"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 335, "latest_message_tokens": 381, "is_estimate": true, "response_id": "06dbb6c51037d633135dfb26db1fe400"}, "latency_s": 1.3597357273101807} +{"seq": 2, "elapsed_s": 9.54, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11829, "completion_tokens": 67, "cache_read_tokens": 11520, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11896, "response_id": "06dbb6c6c2e331e83ee04dd2cd1679c9"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 763, "latest_message_tokens": 302, "is_estimate": true, "response_id": "06dbb6c6c2e331e83ee04dd2cd1679c9"}, "latency_s": 3.458706855773926} +{"seq": 3, "elapsed_s": 23.35, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11998, "completion_tokens": 69, "cache_read_tokens": 11776, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 12067, "response_id": "06dbb6caf4498837aeb03583240e0d6f"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1105, "latest_message_tokens": 109, "is_estimate": true, "response_id": "06dbb6caf4498837aeb03583240e0d6f"}, "latency_s": 13.118934154510498} +{"seq": 4, "elapsed_s": 73.01, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12213, "completion_tokens": 5043, "cache_read_tokens": 12032, "cache_write_tokens": 0, "reasoning_tokens": 4902, "context_window": 0, "per_turn_token": 17256, "response_id": "06dbb6d8a85fc161b8d0ebd3d5c03545"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1258, "latest_message_tokens": 159, "is_estimate": true, "response_id": "06dbb6d8a85fc161b8d0ebd3d5c03545"}, "latency_s": 48.94923162460327} +{"seq": 5, "elapsed_s": 250.2, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12491, "completion_tokens": 20022, "cache_read_tokens": 12160, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 32513, "response_id": "06dbb70a42dc5e7ea00df3555590e2a1"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1535, "latest_message_tokens": 145, "is_estimate": true, "response_id": "06dbb70a42dc5e7ea00df3555590e2a1"}, "latency_s": 176.5030059814453} +{"seq": 6, "elapsed_s": 326.49, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12613, "completion_tokens": 3445, "cache_read_tokens": 12416, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 16058, "response_id": "06dbb7ba0939ce1368334e3af96e69dc"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1727, "latest_message_tokens": 54, "is_estimate": true, "response_id": "06dbb7ba0939ce1368334e3af96e69dc"}, "latency_s": 76.26301431655884} +{"seq": 7, "elapsed_s": 333.87, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 16079, "completion_tokens": 1051, "cache_read_tokens": 16000, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17130, "response_id": "06dbb8069722fbc7cd274869cf0db873"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 5519, "latest_message_tokens": 34, "is_estimate": true, "response_id": "06dbb8069722fbc7cd274869cf0db873"}, "latency_s": 7.361048221588135} +{"seq": 8, "elapsed_s": 338.86, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17171, "completion_tokens": 395, "cache_read_tokens": 17024, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17566, "response_id": "06dbb80eaf94f0fec2c19a86ce37214b"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 6676, "latest_message_tokens": 51, "is_estimate": true, "response_id": "06dbb80eaf94f0fec2c19a86ce37214b"}, "latency_s": 4.976585626602173} +{"seq": 9, "elapsed_s": 348.48, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 18232, "completion_tokens": 742, "cache_read_tokens": 17536, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 18974, "response_id": "06dbb813844ff4664a61e2ab25135957"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 7135, "latest_message_tokens": 664, "is_estimate": true, "response_id": "06dbb813844ff4664a61e2ab25135957"}, "latency_s": 8.94175910949707} diff --git a/.pr/evidence/minimax-m3/p09-task-10-summary.json b/.pr/evidence/minimax-m3/p09-task-10-summary.json new file mode 100644 index 0000000000..d6157fd23c --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-10-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-10", + "prompt": "Implement an in-memory caching layer in `toyapp/cache.py` with `CachingAPIClient`. Calling `CachingAPIClient()` with no client should wrap `toyapp.api.APIClient`. The constructor should also accept an optional backend client, `clock` callable, `user_ttl`, and `account_summary_ttl`. Preserve `fetch_user(user_id)` and `fetch_account_summary(user_id)`, use separate read-through TTL caches for users and account summaries, expose `invalidate_user` and `invalidate_account_summary`, and make `invalidate_user` clear both the user cache and that user's dependent account summary cache. Coalesce concurrent requests for the same key without serializing different keys behind one long backend lock. Return stale cached data on backend failure when available, but do not mark that stale value fresh; the next call should retry the backend. Propagate the backend error to all waiters when a failed fill has no stale value, clear failed fills so a later retry can succeed, and expose `get_metrics()` with `cache_hit_total`, `cache_miss_total`, `cache_fill_seconds`, and `cache_invalidation_total`. Count `cache_miss_total` once per backend fill attempt; coalesced waiters must not inflate it.", + "model": "openai/MiniMax-M3", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 348.49, + "max_iterations_cap": 30, + "calls_recorded": 10, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "openai/MiniMax-M3", + "accumulated_cost": 0.0, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "openai/MiniMax-M3", + "prompt_tokens": 135058, + "completion_tokens": 31025, + "cache_read_tokens": 123648, + "cache_write_tokens": 0, + "reasoning_tokens": 4915, + "context_window": 0, + "per_turn_token": 18974, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 85d39409a0..32e563a896 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -1318,7 +1318,9 @@ def _finalize_completion_params( telemetry_ctx.update( { "messages": formatted_messages[:], # already simple dicts - "tools": tools, + # OpenAI-format schemas as sent, so offline analysis can + # reconstruct the request without the Tool classes. + "tools": cc_tools or None, "kwargs": {k: v for k, v in call_kwargs.items()}, } ) @@ -1456,7 +1458,8 @@ def _finalize_responses_params( "llm_path": "responses", "instructions": instructions, "input": input_items[:], - "tools": tools, + # Responses-format schemas as sent (see above). + "tools": resp_tools, "kwargs": {k: v for k, v in call_kwargs.items()}, } ) diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py new file mode 100644 index 0000000000..16c03eb626 --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -0,0 +1,226 @@ +"""Client-side estimation of per-call prompt token composition. + +Offline analysis utility: it tokenizes logged request payloads after a run +(see ``scripts/prompt_composition_report.py``); nothing here runs on the +LLM call path. +""" + +from typing import Any + +from litellm import ChatCompletionToolParam +from litellm.utils import token_counter +from pydantic import BaseModel, Field + +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) + +# token_counter requires a messages argument when tools are passed, so tool +# schema tokens are measured as the marginal cost over an empty probe message +# (messages=[] would also work in litellm 1.84.1; the probe keeps the call +# shape explicit either way). +_TOOLS_PROBE_MESSAGES: list[dict[str, Any]] = [{"role": "user", "content": ""}] + + +class PromptComposition(BaseModel): + """Per-call decomposition of prompt tokens by component. + + Counts are client-side estimates computed offline from the logged + request payload; the provider-reported ``TokenUsage`` remains + authoritative. Each component is counted independently, so per-message + framing overhead is included in every bucket and the components do not + necessarily sum exactly to the provider-reported ``prompt_tokens``. + Tool schema counts follow litellm's ``token_counter`` serialization + convention for tools, which can differ from the provider's wire-format + tokenization. The composition always uses litellm's generic counter so + numbers stay comparable across models. + """ + + model: str = Field(default="") + system_prompt_tokens: int = Field( + default=0, ge=0, description="Estimated tokens in system messages" + ) + tool_schema_tokens: int = Field( + default=0, ge=0, description="Estimated tokens in tool schemas" + ) + history_tokens: int = Field( + default=0, + ge=0, + description="Estimated tokens in conversation history (all non-system " + "messages except the latest one)", + ) + latest_message_tokens: int = Field( + default=0, + ge=0, + description="Estimated tokens in the latest observation/user message", + ) + is_estimate: bool = Field( + default=True, + description="True when counts are client-side estimates rather than " + "provider-reported usage", + ) + response_id: str = Field(default="") + + +def compute_prompt_composition( + *, + model: str, + messages: list[dict[str, Any]], + tools: list[ChatCompletionToolParam] | None = None, + custom_tokenizer: Any = None, +) -> PromptComposition | None: + """Estimate prompt tokens per component for a single logged LLM call. + + Args: + model: Model name used to pick the tokenizer. + messages: Final OpenAI chat-format messages for the call. + tools: Final OpenAI-format tool schemas for the call, if sent as tools + (when tool schemas are rendered into the prompt text instead, pass + None so they are not double-counted). + custom_tokenizer: Optional LiteLLM custom tokenizer override. + + Returns: + A PromptComposition snapshot, or None when counting fails or returns + no tokens at all (e.g. ``litellm.disable_token_counter``), since + composition analysis is best-effort. + + Cost scales linearly with prompt size and stays small in absolute terms — + measured ~31 ms for a ~100K-token prompt and ~61 ms for ~190K tokens + (gpt-4o tokenizer, 19 tools), versus ~10-20 ms on typical agent-step + payloads. + """ + + def count( + msgs: list[dict[str, Any]], tool_params: list[ChatCompletionToolParam] | None + ) -> int: + return int( + token_counter( + model=model, + messages=msgs, + tools=tool_params, + custom_tokenizer=custom_tokenizer, + # Avoid a GET request per http(s) image URL while counting. + use_default_image_token_count=True, + ) + ) + + system_messages = [m for m in messages if m.get("role") == "system"] + conversation = [m for m in messages if m.get("role") != "system"] + + try: + tool_schema_tokens = 0 + if tools: + tool_schema_tokens = count(_TOOLS_PROBE_MESSAGES, tools) - count( + _TOOLS_PROBE_MESSAGES, None + ) + composition = PromptComposition( + model=model, + system_prompt_tokens=count(system_messages, None) if system_messages else 0, + tool_schema_tokens=tool_schema_tokens, + history_tokens=count(conversation[:-1], None) + if len(conversation) > 1 + else 0, + latest_message_tokens=count(conversation[-1:], None) if conversation else 0, + ) + except Exception: + logger.debug("Prompt composition counting failed for %s", model, exc_info=True) + return None + + if (messages or tools) and not ( + composition.system_prompt_tokens + + composition.tool_schema_tokens + + composition.history_tokens + + composition.latest_message_tokens + ): + logger.debug( + "Prompt composition counting returned zero tokens for %s; skipping record", + model, + ) + return None + return composition + + +def responses_payload_to_chat_messages( + instructions: str | None, input_items: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Convert a finalized Responses API payload into chat-format dicts. + + Used so offline prompt composition on the Responses path counts what the + provider actually received (instructions + input items) while sharing the + chat path's counting convention. Raises ValueError on unrecognized item + types; callers treat any failure as "skip the composition record". + + Buckets follow the wire, not the logical prompt: in subscription mode the + system prompt is folded into the first user message by the auth-layer + transform, so those tokens land in ``history_tokens`` or + ``latest_message_tokens`` rather than ``system_prompt_tokens``. + """ + messages: list[dict[str, Any]] = [] + if instructions: + messages.append({"role": "system", "content": instructions}) + for item in input_items: + messages.extend(_responses_item_to_chat(item)) + return messages + + +def _responses_item_to_chat(item: dict[str, Any]) -> list[dict[str, Any]]: + item_type = item.get("type") + if item_type is None and "role" in item and "content" in item: + # Subscription mode normalizes message items to {"role", "content"} + # without a "type" key (see transform_for_subscription). + item_type = "message" + if item_type == "message": + content = item.get("content", "") + if isinstance(content, str): + return [{"role": item["role"], "content": content}] + return [ + { + "role": item["role"], + "content": [_responses_content_part_to_chat(part) for part in content], + } + ] + if item_type == "function_call": + return [ + { + "role": "assistant", + "tool_calls": [ + { + "id": item.get("call_id", ""), + "type": "function", + "function": { + "name": item.get("name", ""), + "arguments": item.get("arguments", ""), + }, + } + ], + } + ] + if item_type == "function_call_output": + output = item.get("output", "") + if isinstance(output, list): + output = [_responses_content_part_to_chat(part) for part in output] + return [ + { + "role": "tool", + "tool_call_id": item.get("call_id", ""), + "content": output, + } + ] + if item_type == "reasoning": + texts = [part.get("text", "") for part in item.get("summary", [])] + texts += [part.get("text", "") for part in item.get("content", [])] + encrypted = item.get("encrypted_content") + if encrypted: + texts.append(encrypted) + return [{"role": "assistant", "content": "\n".join(texts)}] + raise ValueError(f"Unrecognized Responses input item type: {item_type!r}") + + +def _responses_content_part_to_chat(part: dict[str, Any]) -> dict[str, Any]: + part_type = part.get("type") + if part_type in ("input_text", "output_text"): + return {"type": "text", "text": part.get("text", "")} + if part_type == "input_image": + return {"type": "image_url", "image_url": {"url": part.get("image_url", "")}} + raise ValueError(f"Unrecognized Responses content part type: {part_type!r}") diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py new file mode 100644 index 0000000000..66692865a3 --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py @@ -0,0 +1,352 @@ +"""Offline prompt-composition report over OpenHands completion logs. + +Ingests the JSON logs written by ``LLM(log_completions=True)`` (see +``scripts/completion_logs_viewer.py`` for the directory layout) and rebuilds +the per-call prompt token composition from the logged request payload, joined +with the provider-reported usage. Nothing here runs on the LLM call path. +""" + +import json +import statistics +from pathlib import Path +from typing import Any + +from litellm import ChatCompletionToolParam + +from openhands.sdk.llm.utils.prompt_composition import ( + compute_prompt_composition, + responses_payload_to_chat_messages, +) +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) + +BUCKETS = ( + "system_prompt_tokens", + "tool_schema_tokens", + "history_tokens", + "latest_message_tokens", +) + +_CHART_WIDTH = 50 +_BUCKET_CHARS = { + "system_prompt_tokens": "S", + "tool_schema_tokens": "T", + "history_tokens": "H", + "latest_message_tokens": "L", +} + + +def iter_log_files(root: Path) -> list[Path]: + """List log files under a run folder, or under a root of run folders.""" + root = Path(root) + if not root.is_dir(): + return [] + direct = sorted(root.glob("*.json")) + if direct: + return direct + return sorted(root.glob("*/*.json")) + + +def _normalize_logged_tools(logged_tools: list[Any]) -> list[Any] | None: + """Normalize logged tool schemas to OpenAI chat format for token counting. + + Completion logs carry the finalized schemas as sent: OpenAI chat format + (``{"type": "function", "function": {...}}``) on the chat path, Responses + ToolParam (schema fields at the top level) on the Responses path. Returns + None when the entries are not recognizable tool schemas — logs written + before the schemas were logged in finalized form serialize tools as + ToolDefinition dumps (name/description only, no parameter schemas). + """ + normalized: list[Any] = [] + for tool in logged_tools: + if not isinstance(tool, dict) or tool.get("type") != "function": + return None + if isinstance(tool.get("function"), dict): + normalized.append(tool) + elif "name" in tool: + normalized.append( + { + "type": "function", + "function": {k: v for k, v in tool.items() if k != "type"}, + } + ) + else: + return None + return normalized + + +def _log_model(data: dict[str, Any], source: str) -> str: + response = data.get("response") + if isinstance(response, dict) and isinstance(response.get("model"), str): + return response["model"] + # Log filenames are "{model with '/'->'__'}-{timestamp}-{uuid4}.json". + stem = Path(source).stem + parts = stem.rsplit("-", 2) + if len(parts) == 3 and parts[0]: + return parts[0].replace("__", "/") + return "" + + +def call_record_from_log(data: Any, source: str) -> dict[str, Any] | None: + """Build one report row from a parsed log payload, or None to skip it. + + Returns None for anything that is not a countable LLM call log: unreadable + shapes, error logs, and payloads whose composition cannot be computed. + """ + if not isinstance(data, dict) or "error" in data: + return None + + if data.get("llm_path") == "responses": + input_items = data.get("input") + if not isinstance(input_items, list) or not input_items: + return None + try: + messages = responses_payload_to_chat_messages( + data.get("instructions"), input_items + ) + except ValueError: + return None + schemas_in_prompt = False + else: + messages = data.get("messages") + if not isinstance(messages, list) or not messages: + return None + # Non-native/mock-tools path: schemas are rendered into the prompt + # text, so counting them as tools too would double-count. + schemas_in_prompt = "raw_messages" in data + + tools: list[ChatCompletionToolParam] | None = None + tool_schema_counted = True + logged_tools = data.get("tools") + if not schemas_in_prompt and isinstance(logged_tools, list) and logged_tools: + normalized_tools = _normalize_logged_tools(logged_tools) + if normalized_tools is not None: + tools = normalized_tools + else: + # ToolDefinition dumps carry no parameter schemas, so the tool + # bucket cannot be reconstructed from them; mark the row rather + # than reporting a silently wrong count. + tool_schema_counted = False + + composition = compute_prompt_composition( + model=_log_model(data, source), messages=messages, tools=tools + ) + if composition is None: + return None + + response = data.get("response") + response_id = "" + if isinstance(response, dict) and isinstance(response.get("id"), str): + response_id = response["id"] + composition = composition.model_copy(update={"response_id": response_id}) + + usage_summary = data.get("usage_summary") + if not isinstance(usage_summary, dict): + usage_summary = {} + usage = { + "model": composition.model, + "prompt_tokens": int(usage_summary.get("prompt_tokens") or 0), + "completion_tokens": int(usage_summary.get("completion_tokens") or 0), + "cache_read_tokens": int(usage_summary.get("cache_read_tokens") or 0), + "reasoning_tokens": int(usage_summary.get("reasoning_tokens") or 0), + "context_window": int(data.get("context_window") or 0), + "response_id": response_id, + } + + return { + "timestamp": float(data.get("timestamp") or 0.0), + "source": source, + "usage": usage, + "composition": composition.model_dump(), + "latency_s": data.get("latency_sec"), + "tool_schema_counted": tool_schema_counted, + } + + +def estimated_total(composition: dict[str, Any]) -> int: + return int(sum(int(composition.get(bucket) or 0) for bucket in BUCKETS)) + + +def build_report(root: Path) -> dict[str, Any]: + """Ingest every log under ``root`` into rows plus a summary.""" + rows: list[dict[str, Any]] = [] + skipped: list[dict[str, str]] = [] + root = Path(root) + for path in iter_log_files(root): + source = str(path.relative_to(root)) if path.is_relative_to(root) else path.name + try: + data = json.loads(path.read_text()) + except (OSError, ValueError): + skipped.append({"source": source, "reason": "unreadable or invalid JSON"}) + continue + record = call_record_from_log(data, source) + if record is None: + skipped.append({"source": source, "reason": "not a countable call log"}) + continue + rows.append(record) + + rows.sort(key=lambda r: (r["timestamp"], r["source"])) + sequenced = [{"seq": seq, **row} for seq, row in enumerate(rows)] + return { + "root": str(root), + "rows": sequenced, + "skipped": skipped, + "summary": _summarize(root, sequenced, skipped), + } + + +def _est_provider_ratios(rows: list[dict[str, Any]]) -> list[float]: + ratios = [] + for row in rows: + if not row["tool_schema_counted"]: + # The estimate is missing a bucket, so the ratio would understate. + continue + provider = int(row["usage"].get("prompt_tokens") or 0) + if provider > 0: + ratios.append(estimated_total(row["composition"]) / provider) + return ratios + + +def _summarize( + root: Path, rows: list[dict[str, Any]], skipped: list[dict[str, str]] +) -> dict[str, Any]: + averages = {} + for bucket in BUCKETS: + values = [int(row["composition"].get(bucket) or 0) for row in rows] + averages[bucket] = round(statistics.fmean(values), 1) if values else 0.0 + est_totals = [estimated_total(row["composition"]) for row in rows] + provider_prompt = [int(row["usage"].get("prompt_tokens") or 0) for row in rows] + ratios = _est_provider_ratios(rows) + return { + "root": str(root), + "calls": len(rows), + "skipped_files": len(skipped), + "calls_with_uncountable_tool_schemas": sum( + 1 for row in rows if not row["tool_schema_counted"] + ), + "avg": averages, + "avg_est_total": round(statistics.fmean(est_totals), 1) if est_totals else 0.0, + "avg_provider_prompt_tokens": ( + round(statistics.fmean(provider_prompt), 1) if provider_prompt else 0.0 + ), + "est_provider_median_ratio": ( + round(statistics.median(ratios), 3) if ratios else None + ), + "est_provider_ratio_calls": len(ratios), + } + + +def _chart_bar(composition: dict[str, Any], tokens_per_char: float) -> str: + parts = [] + for bucket in BUCKETS: + chars = round(int(composition.get(bucket) or 0) / tokens_per_char) + parts.append(_BUCKET_CHARS[bucket] * chars) + return "".join(parts).ljust(_CHART_WIDTH) + + +def render_text_report(report: dict[str, Any]) -> str: + """Render the per-call stacked bars, the trend table, and the summary.""" + rows: list[dict[str, Any]] = report["rows"] + summary: dict[str, Any] = report["summary"] + lines = [ + "Prompt composition per call " + "(S=system T=tool_schema H=history L=latest, " + f"bar width ~{_CHART_WIDTH} chars)", + ] + if not rows: + lines.append(" no countable calls") + else: + tokens_per_char = max( + max(estimated_total(row["composition"]) for row in rows) / _CHART_WIDTH, + 1.0, + ) + lines.append( + f" {'seq':>4} {'composition':<{_CHART_WIDTH}} " + f"{'est_total':>9} {'provider':>9}" + ) + for row in rows: + est = estimated_total(row["composition"]) + provider = int(row["usage"].get("prompt_tokens") or 0) + bar = _chart_bar(row["composition"], tokens_per_char) + marker = "" if row["tool_schema_counted"] else " *" + lines.append(f" {row['seq']:>4} {bar} {est:>9} {provider:>9}{marker}") + if any(not row["tool_schema_counted"] for row in rows): + lines.append( + " * tool schemas not reconstructable from this log " + "(tool_schema bucket undercounted)" + ) + + lines += [ + "", + "Trend over seq:", + f" {'seq':>4} | {'system':>8} | {'tool_schema':>11} | {'history':>8} | " + f"{'latest':>8} | {'est_total':>9} | {'provider':>8} | {'ratio':>6} | " + f"{'latency_s':>9}", + ] + for row in rows: + composition = row["composition"] + est = estimated_total(composition) + provider = int(row["usage"].get("prompt_tokens") or 0) + if provider > 0 and row["tool_schema_counted"]: + ratio = f"{est / provider:.2f}" + else: + ratio = "-" + latency = row["latency_s"] + latency_s = f"{latency:.2f}" if isinstance(latency, int | float) else "-" + lines.append( + f" {row['seq']:>4} | {composition['system_prompt_tokens']:>8} | " + f"{composition['tool_schema_tokens']:>11} | " + f"{composition['history_tokens']:>8} | " + f"{composition['latest_message_tokens']:>8} | {est:>9} | " + f"{provider:>8} | {ratio:>6} | {latency_s:>9}" + ) + + avg = summary["avg"] + ratio = summary["est_provider_median_ratio"] + lines += [ + "", + "Summary:", + f" calls: {summary['calls']} " + f"(skipped files: {summary['skipped_files']}, " + f"uncountable tool schemas: " + f"{summary['calls_with_uncountable_tool_schemas']})", + f" avg system: {avg['system_prompt_tokens']}", + f" avg tool_schema: {avg['tool_schema_tokens']}", + f" avg history: {avg['history_tokens']}", + f" avg latest: {avg['latest_message_tokens']}", + f" avg est total: {summary['avg_est_total']} " + f"avg provider prompt: {summary['avg_provider_prompt_tokens']}", + f" est/provider median ratio: " + f"{ratio if ratio is not None else 'n/a'} " + f"(over {summary['est_provider_ratio_calls']} calls)", + ] + return "\n".join(lines) + + +def write_report(report: dict[str, Any], out_dir: Path) -> tuple[Path, Path]: + """Write the per-call JSONL rows and the summary JSON into ``out_dir``.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + calls_path = out_dir / "calls.jsonl" + with calls_path.open("w", encoding="utf-8") as f: + for row in report["rows"]: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + summary_path = out_dir / "summary.json" + summary_path.write_text( + json.dumps(report["summary"], indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return calls_path, summary_path + + +__all__ = [ + "BUCKETS", + "build_report", + "call_record_from_log", + "estimated_total", + "iter_log_files", + "render_text_report", + "write_report", +] diff --git a/scripts/prompt_composition_report.py b/scripts/prompt_composition_report.py new file mode 100644 index 0000000000..d423df67e8 --- /dev/null +++ b/scripts/prompt_composition_report.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Offline per-call prompt token composition report from completion logs. + +Usage: + uv run python scripts/prompt_composition_report.py --root LOGS [--out DIR] + [--no-chart] + +``--root`` follows the completion-log layout documented in +``scripts/completion_logs_viewer.py``: either a single run folder of ``*.json`` +logs, or a root directory containing such run folders. The logs are the files +written by ``LLM(log_completions=True)``. + +For each log the script rebuilds the prompt token composition (system prompt, +tool schemas, history, latest message) from the logged request payload, joins +it with the provider-reported usage, prints a text visualization, and +optionally writes the per-call rows (``calls.jsonl``) and aggregate +``summary.json`` into ``--out``. +""" + +import argparse +import sys +from pathlib import Path + +from openhands.sdk.llm.utils.prompt_composition_report import ( + build_report, + render_text_report, + write_report, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + required=True, + type=Path, + help="Run folder of *.json completion logs, or a root of run folders", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="Directory to write calls.jsonl and summary.json into", + ) + parser.add_argument( + "--no-chart", + action="store_true", + help="Skip the text visualization on stdout", + ) + args = parser.parse_args() + + report = build_report(args.root) + if not report["rows"]: + print(f"No countable LLM call logs found under {args.root}", file=sys.stderr) + return 1 + + if not args.no_chart: + print(render_text_report(report)) + if args.out: + calls_path, summary_path = write_report(report, args.out) + print(f"\nWrote {calls_path} and {summary_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/sdk/llm/test_llm_log_completions_integration.py b/tests/sdk/llm/test_llm_log_completions_integration.py index 7a68ce2b46..d1d6bee065 100644 --- a/tests/sdk/llm/test_llm_log_completions_integration.py +++ b/tests/sdk/llm/test_llm_log_completions_integration.py @@ -8,11 +8,18 @@ import os import tempfile import warnings +from collections.abc import Sequence +from typing import ClassVar from unittest.mock import patch +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from openai.types.responses.response_output_message import ResponseOutputMessage +from openai.types.responses.response_output_text import ResponseOutputText from pydantic import SecretStr from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.tool.schema import Action +from openhands.sdk.tool.tool import ToolDefinition # Import common test utilities from tests.conftest import create_mock_litellm_response @@ -187,3 +194,108 @@ def test_llm_log_completions_with_tool_calls(): assert "response" in log_data assert log_data["response"]["choices"][0]["message"]["tool_calls"] is not None + + +class _LogCompletionsArgs(Action): + param: str + + +class _LogCompletionsMockTool(ToolDefinition[_LogCompletionsArgs, None]): + name: ClassVar[str] = "test_tool" + + @classmethod + def create(cls, conv_state=None, **params) -> Sequence["_LogCompletionsMockTool"]: + return [cls(description="A test tool", action_type=_LogCompletionsArgs)] + + +def _read_single_log(temp_dir: str) -> dict: + log_files = os.listdir(temp_dir) + assert len(log_files) == 1, f"Expected 1 log file, got {len(log_files)}" + with open(os.path.join(temp_dir, log_files[0])) as f: + return json.loads(f.read()) + + +def test_log_completions_logs_openai_format_tool_schemas(): + """The chat-path log must carry the OpenAI-format tool schemas actually + sent (full parameter properties), so offline analysis can reconstruct the + request without the Tool classes.""" + with tempfile.TemporaryDirectory() as temp_dir: + llm = LLM( + model="gpt-4o", + api_key=SecretStr("test-key"), + usage_id="test-log-tools-llm", + log_completions=True, + log_completions_folder=temp_dir, + num_retries=0, + ) + mock_response = create_mock_litellm_response( + content="ok", + response_id="chat-tools-id", + model="gpt-4o", + ) + + with patch( + "openhands.sdk.llm.llm.litellm_completion", return_value=mock_response + ): + llm.completion( + [Message(role="user", content=[TextContent(text="Call a tool")])], + tools=list(_LogCompletionsMockTool.create()), + ) + + log_data = _read_single_log(temp_dir) + + (log_tool,) = log_data["tools"] + assert log_tool["type"] == "function" + assert log_tool["function"]["name"] == "test_tool" + assert "param" in log_tool["function"]["parameters"]["properties"] + # The duplicate copy under kwargs is still stripped by Telemetry. + assert "tools" not in log_data["kwargs"] + + +def test_log_completions_logs_responses_format_tool_schemas(): + """The Responses-path log must carry the finalized Responses ToolParam + schemas (parameter properties at the top level).""" + output = ResponseOutputMessage.model_construct( + id="m1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + mock_response = ResponsesAPIResponse( + id="resp-tools-id", + created_at=0, + output=[output], + parallel_tool_calls=False, + tool_choice="auto", + top_p=None, + tools=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), + status="completed", + ) + + with tempfile.TemporaryDirectory() as temp_dir: + llm = LLM( + model="gpt-5-mini", + api_key=SecretStr("test-key"), + usage_id="test-log-resp-tools-llm", + log_completions=True, + log_completions_folder=temp_dir, + num_retries=0, + ) + + with patch( + "openhands.sdk.llm.llm.litellm_responses", return_value=mock_response + ): + llm.responses( + [Message(role="user", content=[TextContent(text="Call a tool")])], + tools=list(_LogCompletionsMockTool.create()), + ) + + log_data = _read_single_log(temp_dir) + + assert log_data["llm_path"] == "responses" + (log_tool,) = log_data["tools"] + assert log_tool["type"] == "function" + assert log_tool["name"] == "test_tool" + assert "param" in log_tool["parameters"]["properties"] diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py new file mode 100644 index 0000000000..6f4a63c80a --- /dev/null +++ b/tests/sdk/llm/test_prompt_composition.py @@ -0,0 +1,481 @@ +"""Tests for per-call prompt token composition estimates.""" + +import json +from collections.abc import Sequence +from typing import ClassVar +from unittest.mock import patch + +import pytest +from litellm.utils import token_counter +from pydantic import SecretStr + +from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.llm.utils.prompt_composition import ( + compute_prompt_composition, + responses_payload_to_chat_messages, +) +from openhands.sdk.llm.utils.prompt_composition_report import ( + build_report, + write_report, +) +from openhands.sdk.tool.schema import Action +from openhands.sdk.tool.tool import ToolDefinition + + +class _CompositionArgs(Action): + param: str + + +class _CompositionMockTool(ToolDefinition[_CompositionArgs, None]): + name: ClassVar[str] = "test_tool" + + @classmethod + def create(cls, conv_state=None, **params) -> Sequence["_CompositionMockTool"]: + return [cls(description="A test tool", action_type=_CompositionArgs)] + + +def _make_llm(model: str = "gpt-4o") -> LLM: + return LLM(model=model, api_key=SecretStr("test"), usage_id="test-llm") + + +def _sample_messages() -> list[Message]: + return [ + Message(role="system", content=[TextContent(text="You are an agent." * 50)]), + Message(role="user", content=[TextContent(text="please do the task" * 20)]), + Message(role="assistant", content=[TextContent(text="working on it" * 20)]), + Message( + role="user", content=[TextContent(text="observation: file written" * 20)] + ), + ] + + +def test_compute_prompt_composition_decomposes_into_components(): + formatted = _make_llm().format_messages_for_llm(_sample_messages()) + tools = [ + t.to_openai_tool(add_security_risk_prediction=True) + for t in _CompositionMockTool.create() + ] + + composition = compute_prompt_composition( + model="gpt-4o", messages=formatted, tools=tools + ) + + assert composition is not None + assert composition.is_estimate + assert composition.system_prompt_tokens > 0 + assert composition.tool_schema_tokens > 0 + assert composition.history_tokens > 0 + assert composition.latest_message_tokens > 0 + # Buckets are counted independently with per-message framing overhead, + # so their sum covers the message content of the whole prompt. + total_messages = token_counter(model="gpt-4o", messages=formatted) + component_sum = ( + composition.system_prompt_tokens + + composition.history_tokens + + composition.latest_message_tokens + ) + assert component_sum >= total_messages + + +def test_compute_prompt_composition_single_turn_has_no_history(): + formatted = _make_llm().format_messages_for_llm(_sample_messages()[:2]) + + composition = compute_prompt_composition(model="gpt-4o", messages=formatted) + + assert composition is not None + assert composition.tool_schema_tokens == 0 + assert composition.history_tokens == 0 + assert composition.system_prompt_tokens > 0 + assert composition.latest_message_tokens > 0 + + +def test_compute_prompt_composition_returns_none_when_counter_fails(): + with patch( + "openhands.sdk.llm.utils.prompt_composition.token_counter", + side_effect=ValueError("unknown model"), + ): + composition = compute_prompt_composition( + model="not-a-model", messages=[{"role": "user", "content": "hi"}] + ) + + assert composition is None + + +def test_compute_prompt_composition_skips_all_zero_records(): + """A disabled token counter (all-zero result) must skip the record.""" + with patch( + "openhands.sdk.llm.utils.prompt_composition.token_counter", + return_value=0, + ): + composition = compute_prompt_composition( + model="gpt-4o", messages=[{"role": "user", "content": "hi"}] + ) + + assert composition is None + + +def test_compute_prompt_composition_tool_schema_tokens_matches_controlled_delta(): + """tool_schema_tokens must equal the marginal cost of adding tools.""" + formatted = _make_llm().format_messages_for_llm(_sample_messages()) + tools = [ + t.to_openai_tool(add_security_risk_prediction=True) + for t in _CompositionMockTool.create() + ] + + with_tools = compute_prompt_composition( + model="gpt-4o", messages=formatted, tools=tools + ) + without_tools = compute_prompt_composition(model="gpt-4o", messages=formatted) + assert with_tools is not None and without_tools is not None + assert without_tools.tool_schema_tokens == 0 + + component_sum = ( + with_tools.system_prompt_tokens + + with_tools.tool_schema_tokens + + with_tools.history_tokens + + with_tools.latest_message_tokens + ) + other_components = ( + without_tools.system_prompt_tokens + + without_tools.history_tokens + + without_tools.latest_message_tokens + ) + # Deliberately pins the joint-counting assumption (tools never enter the + # message buckets); it is an identity by construction, kept as a tripwire. + assert component_sum - other_components == with_tools.tool_schema_tokens + + # And the estimate tracks the real marginal cost of the tools closely. + real_delta = token_counter( + model="gpt-4o", messages=formatted, tools=tools + ) - token_counter(model="gpt-4o", messages=formatted) + assert abs(with_tools.tool_schema_tokens - real_delta) <= 8 + + +def test_responses_payload_to_chat_messages_covers_sent_item_types(): + instructions, input_items = _make_llm("gpt-5-mini").format_messages_for_responses( + _sample_messages() + ) + assert instructions is not None + + chat = responses_payload_to_chat_messages(instructions, input_items) + + assert chat[0] == {"role": "system", "content": instructions} + roles = [m["role"] for m in chat[1:]] + assert roles == ["user", "assistant", "user"] + + reasoning_chat = responses_payload_to_chat_messages( + None, + [ + { + "type": "reasoning", + "id": "rid", + "summary": [{"type": "summary_text", "text": "thinking"}], + "encrypted_content": "blob", + }, + { + "type": "function_call", + "id": "fc_1", + "call_id": "c1", + "name": "do", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "c1", "output": "done"}, + ], + ) + assert reasoning_chat[0]["role"] == "assistant" + assert "blob" in reasoning_chat[0]["content"] + assert reasoning_chat[1]["tool_calls"][0]["function"]["name"] == "do" + assert reasoning_chat[2] == { + "role": "tool", + "tool_call_id": "c1", + "content": "done", + } + + with pytest.raises(ValueError, match="Unrecognized Responses input item"): + responses_payload_to_chat_messages(None, [{"type": "mystery"}]) + + +def test_responses_payload_to_chat_messages_subscription_shaped_items(): + """Subscription mode strips the "type" key from message items + (transform_for_subscription); they must still convert.""" + chat = responses_payload_to_chat_messages( + None, + [ + {"role": "user", "content": "system prompt folded in"}, + {"role": "assistant", "content": "working on it"}, + ], + ) + + assert chat == [ + {"role": "user", "content": "system prompt folded in"}, + {"role": "assistant", "content": "working on it"}, + ] + + +def test_responses_payload_to_chat_messages_image_parts(): + chat = responses_payload_to_chat_messages( + None, + [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA", + "detail": "auto", + }, + ], + } + ], + ) + + assert chat == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + }, + ], + } + ] + + +def _write_log(run_dir, name: str, payload: dict): + path = run_dir / name + path.write_text(json.dumps(payload)) + return path + + +_OPENAI_TOOL = { + "type": "function", + "function": { + "name": "test_tool", + "description": "A test tool", + "parameters": { + "type": "object", + "properties": {"param": {"type": "string"}}, + "required": ["param"], + }, + }, +} + +_TOOL_DEFINITION_DUMP = { + "description": "A test tool", + "action_type": "_CompositionArgs", + "kind": "_CompositionMockTool", + "title": "test_tool", +} + +# Responses ToolParam as logged on the Responses path: schema fields at the +# top level, no nested "function" key. +_RESPONSES_TOOL = { + "type": "function", + "name": "test_tool", + "description": "A test tool", + "parameters": { + "type": "object", + "properties": {"param": {"type": "string"}}, + "required": ["param"], + }, +} + + +def _chat_log( + response_id: str, + timestamp: float, + history: list[dict] | None = None, + tools: list | None = None, + prompt_tokens: int = 900, +) -> dict: + messages = [{"role": "system", "content": "You are an agent. " * 50}] + messages.extend(history or []) + messages.append({"role": "user", "content": "latest observation " * 10}) + return { + "context_window": 128000, + "messages": messages, + "tools": tools if tools is not None else [_OPENAI_TOOL], + "kwargs": {}, + "response": {"id": response_id, "model": "gpt-4o"}, + "usage_summary": { + "prompt_tokens": prompt_tokens, + "completion_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + }, + "cost": 0.001, + "timestamp": timestamp, + "latency_sec": 1.5, + } + + +def test_report_ingests_chat_logs_in_timestamp_order(tmp_path): + run = tmp_path / "run1" + run.mkdir() + _write_log(run, "gpt-4o-002.000-ab12.json", _chat_log("r2", timestamp=2.0)) + _write_log(run, "gpt-4o-001.000-cd34.json", _chat_log("r1", timestamp=1.0)) + + report = build_report(run) + + assert report["skipped"] == [] + assert [row["seq"] for row in report["rows"]] == [0, 1] + first, _second = report["rows"] + # seq follows the call timestamp, not the filename order. + assert first["composition"]["response_id"] == "r1" + assert first["tool_schema_counted"] is True + assert first["composition"]["system_prompt_tokens"] > 0 + assert first["composition"]["tool_schema_tokens"] > 0 + assert first["composition"]["latest_message_tokens"] > 0 + assert first["usage"]["prompt_tokens"] == 900 + assert first["latency_s"] == 1.5 + + summary = report["summary"] + assert summary["calls"] == 2 + assert summary["calls_with_uncountable_tool_schemas"] == 0 + assert summary["avg"]["system_prompt_tokens"] > 0 + assert summary["est_provider_ratio_calls"] == 2 + assert summary["est_provider_median_ratio"] is not None + + +def test_report_ingests_responses_log(tmp_path): + run = tmp_path / "run1" + run.mkdir() + _write_log( + run, + "gpt-5-mini-001.000-ab12.json", + { + "context_window": 200000, + "llm_path": "responses", + "instructions": "You are an agent. " * 50, + "input": [ + {"type": "message", "role": "user", "content": "do the task " * 10}, + { + "type": "function_call", + "id": "fc1", + "call_id": "c1", + "name": "test_tool", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": "done " * 30, + }, + ], + "tools": [_RESPONSES_TOOL], + "kwargs": {}, + "response": {"id": "resp-1", "model": "gpt-5-mini"}, + "usage_summary": { + "prompt_tokens": 800, + "completion_tokens": 8, + "reasoning_tokens": 3, + "cache_read_tokens": 0, + }, + "cost": 0.002, + "timestamp": 1.0, + "latency_sec": 2.1, + }, + ) + + report = build_report(run) + + assert report["skipped"] == [] + (row,) = report["rows"] + # instructions become the system bucket; input items fill history/latest. + assert row["composition"]["system_prompt_tokens"] > 0 + assert row["composition"]["tool_schema_tokens"] > 0 + assert row["composition"]["history_tokens"] > 0 + assert row["tool_schema_counted"] is True + assert row["usage"]["response_id"] == "resp-1" + + +def test_report_mock_tools_log_counts_schemas_once(tmp_path): + """With raw_messages present, schemas live in the prompt text: the tool + bucket stays 0 (no double count) while they inflate the system bucket.""" + run = tmp_path / "run1" + run.mkdir() + raw_messages = [ + {"role": "system", "content": "You are an agent. " * 50}, + {"role": "user", "content": "do the task"}, + ] + mocked_messages = [ + { + "role": "system", + "content": "You are an agent. " * 50 + + "TOOLS:\n" + + json.dumps(_OPENAI_TOOL) * 3, + }, + {"role": "user", "content": "do the task"}, + ] + payload = _chat_log("mock-1", timestamp=1.0, tools=[_TOOL_DEFINITION_DUMP]) + payload["messages"] = mocked_messages + payload["raw_messages"] = raw_messages + _write_log(run, "gpt-4o-001.000-ab12.json", payload) + + report = build_report(run) + + assert report["skipped"] == [] + (row,) = report["rows"] + assert row["tool_schema_counted"] is True + assert row["composition"]["tool_schema_tokens"] == 0 + raw_composition = compute_prompt_composition(model="gpt-4o", messages=raw_messages) + assert raw_composition is not None + assert ( + row["composition"]["system_prompt_tokens"] + > raw_composition.system_prompt_tokens + ) + + +def test_report_marks_uncountable_tool_schemas(tmp_path): + """Logs serialize tools as ToolDefinition dumps (no parameter schemas), so + the tool bucket is marked uncountable rather than silently wrong.""" + run = tmp_path / "run1" + run.mkdir() + payload = _chat_log("r1", timestamp=1.0, tools=[_TOOL_DEFINITION_DUMP]) + _write_log(run, "gpt-4o-001.000-ab12.json", payload) + + report = build_report(run) + + assert report["skipped"] == [] + (row,) = report["rows"] + assert row["tool_schema_counted"] is False + assert row["composition"]["tool_schema_tokens"] == 0 + assert row["composition"]["system_prompt_tokens"] > 0 + summary = report["summary"] + assert summary["calls_with_uncountable_tool_schemas"] == 1 + # A row missing a bucket is excluded from the est/provider ratio. + assert summary["est_provider_ratio_calls"] == 0 + assert summary["est_provider_median_ratio"] is None + + +def test_report_skips_garbage_files(tmp_path): + run = tmp_path / "run1" + run.mkdir() + (run / "broken.json").write_text("{not json") + _write_log(run, "not-a-log.json", {"hello": "world"}) + _write_log(run, "error-log.json", {**_chat_log("e1", 1.0), "error": {}}) + + report = build_report(run) + + assert report["rows"] == [] + assert len(report["skipped"]) == 3 + assert report["summary"]["calls"] == 0 + + +def test_write_report_emits_calls_jsonl_and_summary(tmp_path): + run = tmp_path / "run1" + run.mkdir() + _write_log(run, "gpt-4o-001.000-ab12.json", _chat_log("r1", timestamp=1.0)) + report = build_report(run) + + calls_path, summary_path = write_report(report, tmp_path / "out") + + (row,) = [json.loads(line) for line in calls_path.read_text().splitlines()] + assert row["seq"] == 0 + assert set(row) >= {"seq", "usage", "composition", "latency_s"} + summary = json.loads(summary_path.read_text()) + assert summary["calls"] == 1