From 7494eb41eeccac673c13fc5990c47261a58bca51 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 24 Aug 2026 12:45:45 -0400 Subject: [PATCH 01/15] feat(sdk): add per-call prompt token composition metrics Record a per-call decomposition of estimated prompt tokens (system prompt, tool schemas, conversation history, latest message) for every LLM completion. Computed at the LLM completion/responses boundary where messages and tools are final, counted with litellm's token_counter, and recorded into Metrics via Telemetry so consumers can attribute prompt size to components. Counts are client-side estimates (flagged via is_estimate); provider-reported usage remains authoritative. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/__init__.py | 2 + openhands-sdk/openhands/sdk/llm/__init__.py | 8 +- openhands-sdk/openhands/sdk/llm/llm.py | 31 ++- .../openhands/sdk/llm/utils/metrics.py | 62 +++++ .../sdk/llm/utils/prompt_composition.py | 75 ++++++ .../openhands/sdk/llm/utils/telemetry.py | 8 +- tests/sdk/llm/test_prompt_composition.py | 224 ++++++++++++++++++ 7 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py create mode 100644 tests/sdk/llm/test_prompt_composition.py diff --git a/openhands-sdk/openhands/sdk/__init__.py b/openhands-sdk/openhands/sdk/__init__.py index 445dca8f19..8e1c85e067 100644 --- a/openhands-sdk/openhands/sdk/__init__.py +++ b/openhands-sdk/openhands/sdk/__init__.py @@ -32,6 +32,7 @@ LLMRegistry, LLMStreamChunk, Message, + PromptComposition, RedactedThinkingBlock, RegistryEvent, TextContent, @@ -127,6 +128,7 @@ "FallbackStrategy", "TokenCallbackType", "TokenUsage", + "PromptComposition", "ConversationStats", "RegistryEvent", "Message", diff --git a/openhands-sdk/openhands/sdk/llm/__init__.py b/openhands-sdk/openhands/sdk/llm/__init__.py index 01048f885a..5c72a093ac 100644 --- a/openhands-sdk/openhands/sdk/llm/__init__.py +++ b/openhands-sdk/openhands/sdk/llm/__init__.py @@ -34,7 +34,12 @@ LLMStreamChunk, TokenCallbackType, ) -from openhands.sdk.llm.utils.metrics import Metrics, MetricsSnapshot, TokenUsage +from openhands.sdk.llm.utils.metrics import ( + Metrics, + MetricsSnapshot, + PromptComposition, + TokenUsage, +) from openhands.sdk.llm.utils.runtime_metadata import ModelRuntimeMetadata from openhands.sdk.llm.utils.unverified_models import ( UNVERIFIED_MODELS_EXCLUDING_BEDROCK, @@ -79,6 +84,7 @@ # Metrics "Metrics", "MetricsSnapshot", + "PromptComposition", "TokenUsage", # Runtime metadata "ModelRuntimeMetadata", diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 2dd49d7164..7d202fe5ce 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -124,6 +124,7 @@ canonicalize_openhands_llm_payload, litellm_call_kwargs, ) +from openhands.sdk.llm.utils.prompt_composition import compute_prompt_composition from openhands.sdk.llm.utils.retry_mixin import RetryMixin from openhands.sdk.llm.utils.telemetry import Telemetry from openhands.sdk.llm.utils.vertex_preflight import assert_vertex_sdk_available @@ -1302,7 +1303,15 @@ def _finalize_completion_params( # logging is disabled. telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { - "context_window": self.effective_max_input_tokens or 0 + "context_window": self.effective_max_input_tokens or 0, + # When tool schemas are mocked into the prompt text, they are + # already inside the message buckets — don't count them twice. + "prompt_composition": compute_prompt_composition( + model=self.model, + messages=formatted_messages, + tools=None if use_mock_tools else cc_tools or None, + custom_tokenizer=self._tokenizer, + ), } if telemetry.log_enabled: telemetry_ctx.update( @@ -1347,6 +1356,7 @@ def _prepare_responses_params( """ instructions, input_items = self.format_messages_for_responses(messages) return self._finalize_responses_params( + messages, instructions, input_items, tools, @@ -1380,6 +1390,7 @@ async def _aprepare_responses_params( """ instructions, input_items = await self.aformat_messages_for_responses(messages) return self._finalize_responses_params( + messages, instructions, input_items, tools, @@ -1392,6 +1403,7 @@ async def _aprepare_responses_params( def _finalize_responses_params( self, + messages: list[Message], instructions: str | None, input_items: list[dict[str, Any]], tools: Sequence[ToolDefinition] | None, @@ -1438,7 +1450,22 @@ def _finalize_responses_params( # logging is disabled. telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { - "context_window": self.effective_max_input_tokens or 0 + "context_window": self.effective_max_input_tokens or 0, + # Counted on the chat-format equivalent of the payload so the + # decomposition is comparable with the chat completion path. + "prompt_composition": compute_prompt_composition( + model=self.model, + messages=self._to_chat_dicts(messages), + tools=[ + t.to_openai_tool( + add_security_risk_prediction=add_security_risk_prediction, + ) + for t in tools + ] + if tools + else None, + custom_tokenizer=self._tokenizer, + ), } if telemetry.log_enabled: telemetry_ctx.update( diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 1a3a23f421..732234969f 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -73,6 +73,42 @@ def __add__(self, other: "TokenUsage") -> "TokenUsage": ) +class PromptComposition(BaseModel): + """Per-call decomposition of prompt tokens by component. + + Counts are client-side estimates computed before the request is sent; + the provider-reported ``TokenUsage`` remains authoritative. Each + component is counted independently, so per-message framing overhead is + included in every bucket and the components may sum to slightly more + than the provider-reported ``prompt_tokens``. + """ + + model: str = Field(default="") + system_prompt_tokens: int = Field( + default=0, ge=0, description="Estimated tokens in system messages" + ) + tool_tokens: int = Field( + default=0, ge=0, description="Estimated tokens in tool schemas" + ) + history_tokens: int = Field( + default=0, + ge=0, + description="Estimated tokens in conversation history (all non-system " + "messages except the latest one)", + ) + latest_message_tokens: int = Field( + default=0, + ge=0, + description="Estimated tokens in the latest observation/user message", + ) + is_estimate: bool = Field( + default=True, + description="True when counts are client-side estimates rather than " + "provider-reported usage", + ) + response_id: str = Field(default="") + + class MetricsSnapshot(BaseModel): """A snapshot of metrics at a point in time. @@ -128,6 +164,15 @@ class Metrics(MetricsSnapshot): token_usages: list[TokenUsage] = Field( default_factory=list, description="List of token usage records" ) + prompt_compositions: list[PromptComposition] = Field( + default_factory=list, + description="Per-call prompt token composition estimates, one per call", + ) + + @property + def latest_prompt_composition(self) -> PromptComposition | None: + """The most recent per-call prompt composition, if any.""" + return self.prompt_compositions[-1] if self.prompt_compositions else None @field_validator("accumulated_cost") @classmethod @@ -219,6 +264,14 @@ def add_token_usage( else: self.accumulated_token_usage = self.accumulated_token_usage + new_usage + def add_prompt_composition( + self, composition: PromptComposition, response_id: str = "" + ) -> None: + """Record the per-call prompt composition snapshot for one call.""" + if response_id: + composition = composition.model_copy(update={"response_id": response_id}) + self.prompt_compositions.append(composition) + def merge(self, other: "Metrics") -> None: """Merge 'other' metrics into this one.""" self.accumulated_cost += other.accumulated_cost @@ -230,6 +283,7 @@ def merge(self, other: "Metrics") -> None: self.costs += other.costs self.token_usages += other.token_usages self.response_latencies += other.response_latencies + self.prompt_compositions += other.prompt_compositions # Merge accumulated token usage using the __add__ operator if self.accumulated_token_usage is None: @@ -252,6 +306,9 @@ def get(self) -> dict: latency.model_dump() for latency in self.response_latencies ], "token_usages": [usage.model_dump() for usage in self.token_usages], + "prompt_compositions": [ + composition.model_dump() for composition in self.prompt_compositions + ], } def log(self) -> str: @@ -299,6 +356,11 @@ def diff(self, baseline: "Metrics") -> "Metrics": # Include only token usages that were added after the baseline result.token_usages = self.token_usages[len(baseline.token_usages) :] + # Include only compositions that were added after the baseline + result.prompt_compositions = self.prompt_compositions[ + len(baseline.prompt_compositions) : + ] + # Calculate accumulated token usage difference base_usage = baseline.accumulated_token_usage current_usage = self.accumulated_token_usage diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py new file mode 100644 index 0000000000..a42a965e1f --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -0,0 +1,75 @@ +"""Client-side estimation of per-call prompt token composition.""" + +from typing import Any + +from litellm import ChatCompletionToolParam +from litellm.utils import token_counter + +from openhands.sdk.llm.utils.metrics import PromptComposition +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) + +# token_counter requires at least one message when tools are passed, so tool +# schema tokens are measured as the marginal cost over an empty probe message. +_TOOLS_PROBE_MESSAGES: list[dict[str, Any]] = [{"role": "user", "content": ""}] + + +def compute_prompt_composition( + *, + model: str, + messages: list[dict[str, Any]], + tools: list[ChatCompletionToolParam] | None = None, + custom_tokenizer: Any = None, +) -> PromptComposition | None: + """Estimate prompt tokens per component for a single LLM call. + + Args: + model: Model name used to pick the tokenizer. + messages: Final OpenAI chat-format messages for the call. + tools: Final OpenAI-format tool schemas for the call, if sent as tools + (when tool schemas are rendered into the prompt text instead, pass + None so they are not double-counted). + custom_tokenizer: Optional LiteLLM custom tokenizer override. + + Returns: + A PromptComposition snapshot, or None when the model's tokenizer is + unavailable (composition recording is best-effort). + """ + + def count( + msgs: list[dict[str, Any]], tool_params: list[ChatCompletionToolParam] | None + ) -> int: + return int( + token_counter( + model=model, + messages=msgs, + tools=tool_params, + custom_tokenizer=custom_tokenizer, + # Avoid a GET request per http(s) image URL while counting. + use_default_image_token_count=True, + ) + ) + + system_messages = [m for m in messages if m.get("role") == "system"] + conversation = [m for m in messages if m.get("role") != "system"] + + try: + tool_tokens = 0 + if tools: + tool_tokens = count(_TOOLS_PROBE_MESSAGES, tools) - count( + _TOOLS_PROBE_MESSAGES, None + ) + return PromptComposition( + model=model, + system_prompt_tokens=count(system_messages, None) if system_messages else 0, + tool_tokens=tool_tokens, + history_tokens=count(conversation[:-1], None) + if len(conversation) > 1 + else 0, + latest_message_tokens=count(conversation[-1:], None) if conversation else 0, + ) + except Exception: + logger.debug("Prompt composition counting failed for %s", model, exc_info=True) + return None diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index 737c7c3b29..b1c1ee63dc 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from openhands.sdk.llm.utils.litellm_provider import LLMProvider -from openhands.sdk.llm.utils.metrics import Metrics +from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition from openhands.sdk.llm.utils.openhands_provider import litellm_call_kwargs from openhands.sdk.logger import get_logger @@ -111,6 +111,12 @@ def on_response( usage, response_id, self._req_ctx.get("context_window", 0) ) + # 3a) per-call prompt composition estimate (request-side, recorded even + # when the provider returned no usage) + composition = self._req_ctx.get("prompt_composition") + if isinstance(composition, PromptComposition): + self.metrics.add_prompt_composition(composition, response_id) + # 4) optional logging if self.log_enabled: self.log_llm_call(resp, cost, raw_resp=raw_resp) diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py new file mode 100644 index 0000000000..c5987e97f0 --- /dev/null +++ b/tests/sdk/llm/test_prompt_composition.py @@ -0,0 +1,224 @@ +"""Tests for per-call prompt token composition estimates.""" + +from collections.abc import Sequence +from typing import ClassVar +from unittest.mock import AsyncMock, patch + +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import ( + Choices, + Message as LiteLLMMessage, + ModelResponse, + Usage, +) +from openai.types.responses.response_output_message import ResponseOutputMessage +from openai.types.responses.response_output_text import ResponseOutputText +from pydantic import SecretStr + +from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition +from openhands.sdk.llm.utils.prompt_composition import compute_prompt_composition +from openhands.sdk.tool.schema import Action +from openhands.sdk.tool.tool import ToolDefinition + + +class _Args(Action): + param: str + + +class _MockTool(ToolDefinition[_Args, None]): + name: ClassVar[str] = "test_tool" + + @classmethod + def create(cls, conv_state=None, **params) -> Sequence["_MockTool"]: + return [cls(description="A test tool", action_type=_Args)] + + +def _chat_response(response_id: str = "resp-1") -> ModelResponse: + return ModelResponse( + id=response_id, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=LiteLLMMessage(content="ok", role="assistant"), + ) + ], + created=0, + model="gpt-4o", + object="chat.completion", + usage=Usage(prompt_tokens=100, completion_tokens=5, total_tokens=105), + ) + + +def _make_llm(model: str = "gpt-4o") -> LLM: + return LLM(model=model, api_key=SecretStr("test"), usage_id="test-llm") + + +def _sample_messages() -> list[Message]: + return [ + Message(role="system", content=[TextContent(text="You are an agent." * 50)]), + Message(role="user", content=[TextContent(text="please do the task" * 20)]), + Message(role="assistant", content=[TextContent(text="working on it" * 20)]), + Message( + role="user", content=[TextContent(text="observation: file written" * 20)] + ), + ] + + +def test_compute_prompt_composition_decomposes_into_components(): + formatted = _make_llm().format_messages_for_llm(_sample_messages()) + tools = [ + t.to_openai_tool(add_security_risk_prediction=True) for t in _MockTool.create() + ] + + composition = compute_prompt_composition( + model="gpt-4o", messages=formatted, tools=tools + ) + + assert composition is not None + assert composition.is_estimate + assert composition.system_prompt_tokens > 0 + assert composition.tool_tokens > 0 + assert composition.history_tokens > 0 + assert composition.latest_message_tokens > 0 + # Buckets are counted independently with per-message framing overhead, + # so their sum covers the message content of the whole prompt. + from litellm.utils import token_counter + + total_messages = token_counter(model="gpt-4o", messages=formatted) + component_sum = ( + composition.system_prompt_tokens + + composition.history_tokens + + composition.latest_message_tokens + ) + assert component_sum >= total_messages + + +def test_compute_prompt_composition_single_turn_has_no_history(): + formatted = _make_llm().format_messages_for_llm(_sample_messages()[:2]) + + composition = compute_prompt_composition(model="gpt-4o", messages=formatted) + + assert composition is not None + assert composition.tool_tokens == 0 + assert composition.history_tokens == 0 + assert composition.system_prompt_tokens > 0 + assert composition.latest_message_tokens > 0 + + +def test_compute_prompt_composition_returns_none_when_counter_fails(): + with patch( + "openhands.sdk.llm.utils.prompt_composition.token_counter", + side_effect=ValueError("unknown model"), + ): + composition = compute_prompt_composition( + model="not-a-model", messages=[{"role": "user", "content": "hi"}] + ) + + assert composition is None + + +def test_completion_records_prompt_composition(): + llm = _make_llm() + tools = list(_MockTool.create()) + + with patch( + "openhands.sdk.llm.llm.litellm_completion", return_value=_chat_response() + ): + llm.completion(messages=_sample_messages(), tools=tools) + + composition = llm.metrics.latest_prompt_composition + assert composition is not None + assert composition.response_id == "resp-1" + # Agent-step style call: tool schemas are part of the prompt. + assert composition.tool_tokens > 0 + assert composition.system_prompt_tokens > 0 + assert composition.history_tokens > 0 + assert composition.latest_message_tokens > 0 + + +def test_completion_without_tools_records_zero_tool_tokens(): + llm = _make_llm() + + with patch( + "openhands.sdk.llm.llm.litellm_completion", return_value=_chat_response() + ): + llm.completion(messages=_sample_messages()) + + composition = llm.metrics.latest_prompt_composition + assert composition is not None + assert composition.tool_tokens == 0 + + +async def test_acompletion_records_prompt_composition(): + llm = _make_llm() + + with patch( + "openhands.sdk.llm.llm.litellm_acompletion", + new_callable=AsyncMock, + return_value=_chat_response(), + ): + await llm.acompletion(messages=_sample_messages()) + + assert llm.metrics.latest_prompt_composition is not None + + +def test_responses_records_prompt_composition(): + llm = _make_llm("gpt-5-mini") + output = ResponseOutputMessage.model_construct( + id="m1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + resp = ResponsesAPIResponse( + id="r1", + created_at=0, + output=[output], + parallel_tool_calls=False, + tool_choice="auto", + top_p=None, + tools=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), + status="completed", + ) + + with patch("openhands.sdk.llm.llm.litellm_responses", return_value=resp): + llm.responses(_sample_messages(), tools=list(_MockTool.create())) + + composition = llm.metrics.latest_prompt_composition + assert composition is not None + assert composition.response_id == "r1" + assert composition.system_prompt_tokens > 0 + assert composition.tool_tokens > 0 + assert composition.latest_message_tokens > 0 + + +def test_metrics_merge_and_diff_include_prompt_compositions(): + baseline = Metrics(model_name="gpt-4o") + baseline.add_prompt_composition( + PromptComposition(system_prompt_tokens=10), response_id="r1" + ) + current = baseline.deep_copy() + current.add_prompt_composition( + PromptComposition(system_prompt_tokens=20), response_id="r2" + ) + + diff = current.diff(baseline) + assert len(diff.prompt_compositions) == 1 + assert diff.prompt_compositions[0].response_id == "r2" + + merged = Metrics(model_name="gpt-4o") + merged.merge(current) + assert len(merged.prompt_compositions) == 2 + assert merged.latest_prompt_composition is not None + assert merged.latest_prompt_composition.response_id == "r2" + + +def test_metrics_loads_payload_without_prompt_compositions(): + metrics = Metrics.model_validate({"model_name": "gpt-4o", "accumulated_cost": 1.0}) + + assert metrics.prompt_compositions == [] + assert metrics.latest_prompt_composition is None From 32df61a71a78a15a5e58bd78dcc8d3b11ce9e76a Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 24 Aug 2026 16:01:10 -0400 Subject: [PATCH 02/15] fix(sdk): count Responses composition on sent payload, skip empty records Address review findings on prompt composition recording: - Responses path now counts the finalized payload (instructions + input items converted to chat format) instead of the unprepared messages, so the record reflects what the provider received; tool serialization or conversion failures skip the record instead of breaking the call. - An all-zero counting result (e.g. litellm.disable_token_counter) now skips the record instead of storing a bogus all-zero estimate. - Docstrings: components do not necessarily sum to provider prompt_tokens (fixes the inverted direction claim), tool schema counts follow litellm's token_counter serialization convention rather than wire-format JSON, and counting cost is documented as linear in prompt size (~31 ms at ~100K tokens, ~61 ms at ~190K tokens measured). - Tests: mock-tools double-count guard, controlled tool-token delta, all-zero skip, Responses payload converter, tool serialization failure. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/llm/llm.py | 70 +++++-- .../openhands/sdk/llm/utils/metrics.py | 6 +- .../sdk/llm/utils/prompt_composition.py | 100 +++++++++- tests/sdk/llm/test_prompt_composition.py | 187 +++++++++++++++++- 4 files changed, 338 insertions(+), 25 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 7d202fe5ce..a05dfa7bac 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -117,14 +117,17 @@ ) from openhands.sdk.llm.utils.image_resize import maybe_resize_messages_for_provider from openhands.sdk.llm.utils.litellm_provider import LLMProvider -from openhands.sdk.llm.utils.metrics import Metrics +from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition from openhands.sdk.llm.utils.model_features import ModelFeatures, get_features from openhands.sdk.llm.utils.openhands_provider import ( LiteLLMCallKwargs, canonicalize_openhands_llm_payload, litellm_call_kwargs, ) -from openhands.sdk.llm.utils.prompt_composition import compute_prompt_composition +from openhands.sdk.llm.utils.prompt_composition import ( + compute_prompt_composition, + responses_payload_to_chat_messages, +) from openhands.sdk.llm.utils.retry_mixin import RetryMixin from openhands.sdk.llm.utils.telemetry import Telemetry from openhands.sdk.llm.utils.vertex_preflight import assert_vertex_sdk_available @@ -1356,7 +1359,6 @@ def _prepare_responses_params( """ instructions, input_items = self.format_messages_for_responses(messages) return self._finalize_responses_params( - messages, instructions, input_items, tools, @@ -1390,7 +1392,6 @@ async def _aprepare_responses_params( """ instructions, input_items = await self.aformat_messages_for_responses(messages) return self._finalize_responses_params( - messages, instructions, input_items, tools, @@ -1403,7 +1404,6 @@ async def _aprepare_responses_params( def _finalize_responses_params( self, - messages: list[Message], instructions: str | None, input_items: list[dict[str, Any]], tools: Sequence[ToolDefinition] | None, @@ -1451,20 +1451,11 @@ def _finalize_responses_params( telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0, - # Counted on the chat-format equivalent of the payload so the - # decomposition is comparable with the chat completion path. - "prompt_composition": compute_prompt_composition( - model=self.model, - messages=self._to_chat_dicts(messages), - tools=[ - t.to_openai_tool( - add_security_risk_prediction=add_security_risk_prediction, - ) - for t in tools - ] - if tools - else None, - custom_tokenizer=self._tokenizer, + "prompt_composition": self._responses_prompt_composition( + instructions, + input_items, + tools, + add_security_risk_prediction, ), } if telemetry.log_enabled: @@ -1480,6 +1471,47 @@ def _finalize_responses_params( return instructions, input_items, resp_tools, call_kwargs, telemetry_ctx + def _responses_prompt_composition( + self, + instructions: str | None, + input_items: list[dict[str, Any]], + tools: Sequence[ToolDefinition] | None, + add_security_risk_prediction: bool, + ) -> PromptComposition | None: + """Best-effort prompt composition for the Responses path. + + Counts the finalized payload (instructions + input items) so the + record reflects what the provider received. Tool schemas are counted + from their OpenAI chat-format equivalent so ``tool_tokens`` stays + comparable with the chat path. Any serialization or conversion + failure yields None rather than breaking the real call. + """ + try: + chat_messages = responses_payload_to_chat_messages( + instructions, input_items + ) + cc_tools = ( + [ + t.to_openai_tool( + add_security_risk_prediction=add_security_risk_prediction, + ) + for t in tools + ] + if tools + else None + ) + except Exception: + logger.debug( + "Responses prompt composition skipped for %s", self.model, exc_info=True + ) + return None + return compute_prompt_composition( + model=self.model, + messages=chat_messages, + tools=cc_tools, + custom_tokenizer=self._tokenizer, + ) + def _validate_chat_response( self, resp: ModelResponse, diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 732234969f..f3a178d9d8 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -79,8 +79,10 @@ class PromptComposition(BaseModel): Counts are client-side estimates computed before the request is sent; the provider-reported ``TokenUsage`` remains authoritative. Each component is counted independently, so per-message framing overhead is - included in every bucket and the components may sum to slightly more - than the provider-reported ``prompt_tokens``. + included in every bucket and the components do not necessarily sum + exactly to the provider-reported ``prompt_tokens``. Tool schema counts + follow litellm's ``token_counter`` serialization convention for tools, + which can differ from the provider's wire-format tokenization. """ model: str = Field(default="") diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py index a42a965e1f..e8a859f3fd 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -34,8 +34,13 @@ def compute_prompt_composition( custom_tokenizer: Optional LiteLLM custom tokenizer override. Returns: - A PromptComposition snapshot, or None when the model's tokenizer is - unavailable (composition recording is best-effort). + A PromptComposition snapshot, or None when counting fails or returns + no tokens at all (e.g. ``litellm.disable_token_counter``), since + composition recording is best-effort. + + Cost scales linearly with prompt size: measured ~31 ms for a ~100K-token + prompt and ~61 ms for ~190K tokens (gpt-4o tokenizer, 19 tools), versus + ~10-20 ms on typical agent-step payloads. """ def count( @@ -61,7 +66,7 @@ def count( tool_tokens = count(_TOOLS_PROBE_MESSAGES, tools) - count( _TOOLS_PROBE_MESSAGES, None ) - return PromptComposition( + composition = PromptComposition( model=model, system_prompt_tokens=count(system_messages, None) if system_messages else 0, tool_tokens=tool_tokens, @@ -73,3 +78,92 @@ def count( except Exception: logger.debug("Prompt composition counting failed for %s", model, exc_info=True) return None + + if (messages or tools) and not ( + composition.system_prompt_tokens + + composition.tool_tokens + + composition.history_tokens + + composition.latest_message_tokens + ): + logger.debug( + "Prompt composition counting returned zero tokens for %s; skipping record", + model, + ) + return None + return composition + + +def responses_payload_to_chat_messages( + instructions: str | None, input_items: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Convert a finalized Responses API payload into chat-format dicts. + + Used so prompt composition on the Responses path counts what the provider + actually received (instructions + input items) while sharing the chat + path's counting convention. Raises ValueError on unrecognized item types; + callers treat any failure as "skip the composition record". + """ + messages: list[dict[str, Any]] = [] + if instructions: + messages.append({"role": "system", "content": instructions}) + for item in input_items: + messages.extend(_responses_item_to_chat(item)) + return messages + + +def _responses_item_to_chat(item: dict[str, Any]) -> list[dict[str, Any]]: + item_type = item.get("type") + if item_type == "message": + return [ + { + "role": item["role"], + "content": [ + _responses_content_part_to_chat(part) + for part in item.get("content", []) + ], + } + ] + if item_type == "function_call": + return [ + { + "role": "assistant", + "tool_calls": [ + { + "id": item.get("call_id", ""), + "type": "function", + "function": { + "name": item.get("name", ""), + "arguments": item.get("arguments", ""), + }, + } + ], + } + ] + if item_type == "function_call_output": + output = item.get("output", "") + if isinstance(output, list): + output = [_responses_content_part_to_chat(part) for part in output] + return [ + { + "role": "tool", + "tool_call_id": item.get("call_id", ""), + "content": output, + } + ] + if item_type == "reasoning": + texts = [part.get("text", "") for part in item.get("summary", [])] + texts += [part.get("text", "") for part in item.get("content", [])] + encrypted = item.get("encrypted_content") + if encrypted: + texts.append(encrypted) + return [{"role": "assistant", "content": "\n".join(texts)}] + raise ValueError(f"Unrecognized Responses input item type: {item_type!r}") + + +def _responses_content_part_to_chat(part: dict[str, Any]) -> dict[str, Any]: + part_type = part.get("type") + if part_type in ("input_text", "output_text"): + return {"type": "text", "text": part.get("text", "")} + if part_type == "input_image": + return {"type": "image_url", "image_url": {"url": part.get("image_url", "")}} + raise ValueError(f"Unrecognized Responses content part type: {part_type!r}") diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index c5987e97f0..f8ea8885e1 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -4,6 +4,7 @@ from typing import ClassVar from unittest.mock import AsyncMock, patch +import pytest from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( Choices, @@ -17,7 +18,10 @@ from openhands.sdk.llm import LLM, Message, TextContent from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition -from openhands.sdk.llm.utils.prompt_composition import compute_prompt_composition +from openhands.sdk.llm.utils.prompt_composition import ( + compute_prompt_composition, + responses_payload_to_chat_messages, +) from openhands.sdk.tool.schema import Action from openhands.sdk.tool.tool import ToolDefinition @@ -119,6 +123,55 @@ def test_compute_prompt_composition_returns_none_when_counter_fails(): assert composition is None +def test_compute_prompt_composition_skips_all_zero_records(): + """A disabled token counter (all-zero result) must skip the record.""" + with patch( + "openhands.sdk.llm.utils.prompt_composition.token_counter", + return_value=0, + ): + composition = compute_prompt_composition( + model="gpt-4o", messages=[{"role": "user", "content": "hi"}] + ) + + assert composition is None + + +def test_compute_prompt_composition_tool_tokens_matches_controlled_delta(): + """tool_tokens must equal the marginal cost of adding tools to the call.""" + from litellm.utils import token_counter + + formatted = _make_llm().format_messages_for_llm(_sample_messages()) + tools = [ + t.to_openai_tool(add_security_risk_prediction=True) for t in _MockTool.create() + ] + + with_tools = compute_prompt_composition( + model="gpt-4o", messages=formatted, tools=tools + ) + without_tools = compute_prompt_composition(model="gpt-4o", messages=formatted) + assert with_tools is not None and without_tools is not None + assert without_tools.tool_tokens == 0 + + component_sum = ( + with_tools.system_prompt_tokens + + with_tools.tool_tokens + + with_tools.history_tokens + + with_tools.latest_message_tokens + ) + other_components = ( + without_tools.system_prompt_tokens + + without_tools.history_tokens + + without_tools.latest_message_tokens + ) + assert component_sum - other_components == with_tools.tool_tokens + + # And the estimate tracks the real marginal cost of the tools closely. + real_delta = token_counter( + model="gpt-4o", messages=formatted, tools=tools + ) - token_counter(model="gpt-4o", messages=formatted) + assert abs(with_tools.tool_tokens - real_delta) <= 8 + + def test_completion_records_prompt_composition(): llm = _make_llm() tools = list(_MockTool.create()) @@ -222,3 +275,135 @@ def test_metrics_loads_payload_without_prompt_compositions(): assert metrics.prompt_compositions == [] assert metrics.latest_prompt_composition is None + + +def test_mock_tools_does_not_double_count_tool_schemas(): + """With prompt-mocked tools, schemas live in the prompt text: tool_tokens + must be 0 (no double count) while the schemas still inflate the system + bucket.""" + mock_response = ModelResponse( + id="mock-resp", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=LiteLLMMessage( + content=( + "I'll help.\n" + "\n" + "test_value\n" + "" + ), + role="assistant", + ), + ) + ], + created=0, + model="gpt-4o", + object="chat.completion", + usage=Usage(prompt_tokens=100, completion_tokens=5, total_tokens=105), + ) + llm = LLM( + model="gpt-4o", + api_key=SecretStr("test"), + usage_id="test-llm", + native_tool_calling=False, + ) + + with patch("openhands.sdk.llm.llm.litellm_completion", return_value=mock_response): + llm.completion(messages=_sample_messages(), tools=list(_MockTool.create())) + with_tools = llm.metrics.latest_prompt_composition + llm.completion(messages=_sample_messages()) + without_tools = llm.metrics.latest_prompt_composition + + assert with_tools is not None and without_tools is not None + assert with_tools.tool_tokens == 0 + assert with_tools.system_prompt_tokens > without_tools.system_prompt_tokens + + +def _responses_api_response() -> ResponsesAPIResponse: + output = ResponseOutputMessage.model_construct( + id="m1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + return ResponsesAPIResponse( + id="r1", + created_at=0, + output=[output], + parallel_tool_calls=False, + tool_choice="auto", + top_p=None, + tools=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), + status="completed", + ) + + +def test_responses_composition_survives_tool_serialization_failure(): + """A tool that fails chat-format serialization must skip the composition + record, not break the real Responses call.""" + llm = _make_llm("gpt-5-mini") + tool = list(_MockTool.create())[0] + + with ( + patch( + "openhands.sdk.llm.llm.litellm_responses", + return_value=_responses_api_response(), + ), + patch.object( + type(tool), + "to_openai_tool", + side_effect=RuntimeError("cannot serialize"), + ), + ): + response = llm.responses(_sample_messages(), tools=[tool]) + + assert response.message.role == "assistant" + assert llm.metrics.latest_prompt_composition is None + + +def test_responses_payload_to_chat_messages_covers_sent_item_types(): + instructions, input_items = _make_llm("gpt-5-mini").format_messages_for_responses( + _sample_messages() + ) + assert instructions is not None + + chat = responses_payload_to_chat_messages(instructions, input_items) + + assert chat[0] == {"role": "system", "content": instructions} + roles = [m["role"] for m in chat[1:]] + assert roles == ["user", "assistant", "user"] + + reasoning_chat = responses_payload_to_chat_messages( + None, + [ + { + "type": "reasoning", + "id": "rid", + "summary": [{"type": "summary_text", "text": "thinking"}], + "encrypted_content": "blob", + }, + { + "type": "function_call", + "id": "fc_1", + "call_id": "c1", + "name": "do", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "c1", "output": "done"}, + ], + ) + assert reasoning_chat[0]["role"] == "assistant" + assert "blob" in reasoning_chat[0]["content"] + assert reasoning_chat[1]["tool_calls"][0]["function"]["name"] == "do" + assert reasoning_chat[2] == { + "role": "tool", + "tool_call_id": "c1", + "content": "done", + } + + with pytest.raises(ValueError, match="Unrecognized Responses input item"): + responses_payload_to_chat_messages(None, [{"type": "mystery"}]) From 7ea5c1fdf89f826b0cf772ddb6ff90adac13d681 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 24 Aug 2026 16:04:48 -0400 Subject: [PATCH 03/15] chore(pr): attach per-call prompt composition evidence (MiniMax-M3 lane) Co-authored-by: openhands --- .pr/evidence/analysis.md | 34 +++++++++++++++++++ .../minimax-m3/p09-task-01-calls.jsonl | 5 +++ .../minimax-m3/p09-task-01-summary.json | 28 +++++++++++++++ .../minimax-m3/p09-task-07-calls.jsonl | 26 ++++++++++++++ .../minimax-m3/p09-task-07-summary.json | 28 +++++++++++++++ .../minimax-m3/p09-task-10-calls.jsonl | 10 ++++++ .../minimax-m3/p09-task-10-summary.json | 28 +++++++++++++++ 7 files changed, 159 insertions(+) create mode 100644 .pr/evidence/analysis.md create mode 100644 .pr/evidence/minimax-m3/p09-task-01-calls.jsonl create mode 100644 .pr/evidence/minimax-m3/p09-task-01-summary.json create mode 100644 .pr/evidence/minimax-m3/p09-task-07-calls.jsonl create mode 100644 .pr/evidence/minimax-m3/p09-task-07-summary.json create mode 100644 .pr/evidence/minimax-m3/p09-task-10-calls.jsonl create mode 100644 .pr/evidence/minimax-m3/p09-task-10-summary.json diff --git a/.pr/evidence/analysis.md b/.pr/evidence/analysis.md new file mode 100644 index 0000000000..60492adfd4 --- /dev/null +++ b/.pr/evidence/analysis.md @@ -0,0 +1,34 @@ +# Evidence: per-call prompt composition on live agent runs + +> `.pr/` is PR-only reviewer context per repository policy (`.github/workflows/pr-artifacts.yml`); +> this directory is removed after PR approval. Nothing here is meant to merge to `main`. + +Per-call records from live runs of the default agent (`get_default_agent`, 19 tools, +browser-on) on tasks from the published harness-benchmark short suite +(). Each `calls.jsonl` row carries the +`PromptComposition` estimate and the provider-reported `TokenUsage` for one LLM call +(counts and response ids only; no message content, no credentials). + +## Lanes + +- `minimax-m3/` — `openai/MiniMax-M3` (chat completions; litellm has no tokenizer mapping for + this model, so estimates use litellm's fallback tokenizer). Tasks p09-task-01 (trivial + rename, 5 calls), p09-task-07 (medium refactor, 26 calls), p09-task-10 (hard cache + implementation, 10 calls). + +## MiniMax-M3 summary + +| task | calls | avg system | avg tools | avg history | avg latest | avg provider input | est/provider median | +|---|---:|---:|---:|---:|---:|---:|---:| +| p09-task-01 | 5 | 3,331 | 5,749 | 533 | 205 | 11,478 | 0.85 | +| p09-task-07 | 26 | 3,331 | 5,749 | 10,462 | 381 | 21,176 | 0.94 | +| p09-task-10 | 10 | 3,331 | 5,749 | 2,605 | 215 | 13,506 | 0.86 | + +- The standing preamble (system + tool schemas ≈ 9,080 estimated tokens) is constant per call: + ~79% of the average call on the trivial task (avg provider input 11,478), ~84% of first calls. +- Re-sent tool schemas totaled 149,474 estimated tokens on the 26-call task — 27% of that run's + provider-reported input (550,568). +- Estimated component sum per call runs 0.85–0.97× the provider-reported `prompt_tokens`, + rising as history grows. The residual gap is consistent with request framing and tokenizer + mapping differences; the component split is the finding, not the absolute sum. +- MiniMax's automatic prefix caching served 79–94% of prompt tokens as cache reads. diff --git a/.pr/evidence/minimax-m3/p09-task-01-calls.jsonl b/.pr/evidence/minimax-m3/p09-task-01-calls.jsonl new file mode 100644 index 0000000000..c0cb064222 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-01-calls.jsonl @@ -0,0 +1,5 @@ +{"seq": 0, "elapsed_s": 2.52, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10753, "completion_tokens": 106, "cache_read_tokens": 128, "cache_write_tokens": 0, "reasoning_tokens": 44, "context_window": 0, "per_turn_token": 10859, "response_id": "06dbb5db9493db12fb8930727ddce1c6"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 0, "latest_message_tokens": 38, "is_estimate": true, "response_id": "06dbb5db9493db12fb8930727ddce1c6"}, "latency_s": 2.0401830673217773} +{"seq": 1, "elapsed_s": 4.79, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10906, "completion_tokens": 80, "cache_read_tokens": 10752, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 10986, "response_id": "06dbb5de6e4f3bd67135995a5c4ddb3b"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 84, "latest_message_tokens": 100, "is_estimate": true, "response_id": "06dbb5de6e4f3bd67135995a5c4ddb3b"}, "latency_s": 1.3546624183654785} +{"seq": 2, "elapsed_s": 7.44, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11463, "completion_tokens": 298, "cache_read_tokens": 10905, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11761, "response_id": "06dbb5df31a5efed186e061448cc88c1"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 232, "latest_message_tokens": 487, "is_estimate": true, "response_id": "06dbb5df31a5efed186e061448cc88c1"}, "latency_s": 2.629474401473999} +{"seq": 3, "elapsed_s": 8.99, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12060, "completion_tokens": 105, "cache_read_tokens": 11462, "cache_write_tokens": 0, "reasoning_tokens": 34, "context_window": 0, "per_turn_token": 12165, "response_id": "06dbb5e27ea6bc960f36a696934697eb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 993, "latest_message_tokens": 313, "is_estimate": true, "response_id": "06dbb5e27ea6bc960f36a696934697eb"}, "latency_s": 1.5360901355743408} +{"seq": 4, "elapsed_s": 11.64, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12209, "completion_tokens": 146, "cache_read_tokens": 12059, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 12355, "response_id": "06dbb5e4e0e9b171812304c45290e354"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1357, "latest_message_tokens": 87, "is_estimate": true, "response_id": "06dbb5e4e0e9b171812304c45290e354"}, "latency_s": 1.942288875579834} diff --git a/.pr/evidence/minimax-m3/p09-task-01-summary.json b/.pr/evidence/minimax-m3/p09-task-01-summary.json new file mode 100644 index 0000000000..2ffd693954 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-01-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-01", + "prompt": "Rename the local variable `raw_items` to `source_items` in `toyapp/pagination.py`. Keep behavior unchanged and do not edit unrelated files.", + "model": "openai/MiniMax-M3", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 11.64, + "max_iterations_cap": 30, + "calls_recorded": 5, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "openai/MiniMax-M3", + "accumulated_cost": 0.0, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "openai/MiniMax-M3", + "prompt_tokens": 57391, + "completion_tokens": 735, + "cache_read_tokens": 45306, + "cache_write_tokens": 0, + "reasoning_tokens": 78, + "context_window": 0, + "per_turn_token": 12355, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/minimax-m3/p09-task-07-calls.jsonl b/.pr/evidence/minimax-m3/p09-task-07-calls.jsonl new file mode 100644 index 0000000000..29f876eef5 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-07-calls.jsonl @@ -0,0 +1,26 @@ +{"seq": 0, "elapsed_s": 2.54, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10811, "completion_tokens": 120, "cache_read_tokens": 128, "cache_write_tokens": 0, "reasoning_tokens": 23, "context_window": 0, "per_turn_token": 10931, "response_id": "06dbb621630bdaa73aa241d88930084e"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 0, "latest_message_tokens": 94, "is_estimate": true, "response_id": "06dbb621630bdaa73aa241d88930084e"}, "latency_s": 2.0828120708465576} +{"seq": 1, "elapsed_s": 4.85, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11323, "completion_tokens": 72, "cache_read_tokens": 10810, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11395, "response_id": "06dbb624284bcacf32032a62c2ce95a3"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 177, "latest_message_tokens": 425, "is_estimate": true, "response_id": "06dbb624284bcacf32032a62c2ce95a3"}, "latency_s": 1.36482572555542} +{"seq": 2, "elapsed_s": 7.08, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 13203, "completion_tokens": 72, "cache_read_tokens": 11322, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 13275, "response_id": "06dbb6262a0fbc14c7b77bd2476bca6b"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 646, "latest_message_tokens": 1755, "is_estimate": true, "response_id": "06dbb6262a0fbc14c7b77bd2476bca6b"}, "latency_s": 1.5264146327972412} +{"seq": 3, "elapsed_s": 9.27, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 13693, "completion_tokens": 86, "cache_read_tokens": 13202, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 13779, "response_id": "06dbb628602d744e87d44e34e71e2b45"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 2444, "latest_message_tokens": 426, "is_estimate": true, "response_id": "06dbb628602d744e87d44e34e71e2b45"}, "latency_s": 1.4912850856781006} +{"seq": 4, "elapsed_s": 11.31, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 13865, "completion_tokens": 82, "cache_read_tokens": 13692, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 13947, "response_id": "06dbb62ae649c708d544b9045b0acab6"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 2927, "latest_message_tokens": 101, "is_estimate": true, "response_id": "06dbb62ae649c708d544b9045b0acab6"}, "latency_s": 1.3661975860595703} +{"seq": 5, "elapsed_s": 54.25, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 14256, "completion_tokens": 6604, "cache_read_tokens": 13864, "cache_write_tokens": 0, "reasoning_tokens": 5978, "context_window": 0, "per_turn_token": 20860, "response_id": "06dbb62cd628b2fd953fc292e33aea2a"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 3082, "latest_message_tokens": 313, "is_estimate": true, "response_id": "06dbb62cd628b2fd953fc292e33aea2a"}, "latency_s": 42.22588348388672} +{"seq": 6, "elapsed_s": 56.91, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 15050, "completion_tokens": 245, "cache_read_tokens": 14255, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 15295, "response_id": "06dbb656b03ec23c38b033f0791abb70"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 4264, "latest_message_tokens": 33, "is_estimate": true, "response_id": "06dbb656b03ec23c38b033f0791abb70"}, "latency_s": 2.644223690032959} +{"seq": 7, "elapsed_s": 69.15, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 15320, "completion_tokens": 1888, "cache_read_tokens": 15049, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17208, "response_id": "06dbb659c94dbd6d07e428b2c47db4d5"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 4453, "latest_message_tokens": 40, "is_estimate": true, "response_id": "06dbb659c94dbd6d07e428b2c47db4d5"}, "latency_s": 12.213894844055176} +{"seq": 8, "elapsed_s": 71.36, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17250, "completion_tokens": 197, "cache_read_tokens": 15319, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17447, "response_id": "06dbb665f57c5f9f601120f350742337"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 6690, "latest_message_tokens": 58, "is_estimate": true, "response_id": "06dbb665f57c5f9f601120f350742337"}, "latency_s": 2.185944080352783} +{"seq": 9, "elapsed_s": 72.75, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17467, "completion_tokens": 73, "cache_read_tokens": 17249, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17540, "response_id": "06dbb668f2eba64bea7757fb744a7267"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 6933, "latest_message_tokens": 35, "is_estimate": true, "response_id": "06dbb668f2eba64bea7757fb744a7267"}, "latency_s": 1.374401569366455} +{"seq": 10, "elapsed_s": 84.08, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17612, "completion_tokens": 1640, "cache_read_tokens": 17466, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19252, "response_id": "06dbb66a29faddfe4528fc2feaa8992a"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 7013, "latest_message_tokens": 87, "is_estimate": true, "response_id": "06dbb66a29faddfe4528fc2feaa8992a"}, "latency_s": 10.641063451766968} +{"seq": 11, "elapsed_s": 85.59, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 19294, "completion_tokens": 94, "cache_read_tokens": 17611, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19388, "response_id": "06dbb6747d9a518fc781addbba4765c9"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9002, "latest_message_tokens": 57, "is_estimate": true, "response_id": "06dbb6747d9a518fc781addbba4765c9"}, "latency_s": 1.4767844676971436} +{"seq": 12, "elapsed_s": 87.79, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 19460, "completion_tokens": 89, "cache_read_tokens": 19293, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19549, "response_id": "06dbb67752070cf87aa20a0e988038a6"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9125, "latest_message_tokens": 89, "is_estimate": true, "response_id": "06dbb67752070cf87aa20a0e988038a6"}, "latency_s": 1.4858717918395996} +{"seq": 13, "elapsed_s": 90.05, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 19833, "completion_tokens": 83, "cache_read_tokens": 19459, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 19916, "response_id": "06dbb6798dc63f0bd8b424e2ee26afea"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9273, "latest_message_tokens": 299, "is_estimate": true, "response_id": "06dbb6798dc63f0bd8b424e2ee26afea"}, "latency_s": 1.5481047630310059} +{"seq": 14, "elapsed_s": 92.88, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 20188, "completion_tokens": 193, "cache_read_tokens": 19832, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 20381, "response_id": "06dbb67bcf1f0244ac14f1baddbfcd3e"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 9626, "latest_message_tokens": 276, "is_estimate": true, "response_id": "06dbb67bcf1f0244ac14f1baddbfcd3e"}, "latency_s": 2.143434524536133} +{"seq": 15, "elapsed_s": 110.16, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 20469, "completion_tokens": 2444, "cache_read_tokens": 20187, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 22913, "response_id": "06dbb67e3a44c9d91f0fc06f5b31b731"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 10071, "latest_message_tokens": 105, "is_estimate": true, "response_id": "06dbb67e3a44c9d91f0fc06f5b31b731"}, "latency_s": 16.57303738594055} +{"seq": 16, "elapsed_s": 117.4, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 25395, "completion_tokens": 746, "cache_read_tokens": 20468, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 26141, "response_id": "06dbb68f3d7b0f4f2468198ae9202206"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 12870, "latest_message_tokens": 2410, "is_estimate": true, "response_id": "06dbb68f3d7b0f4f2468198ae9202206"}, "latency_s": 6.5370707511901855} +{"seq": 17, "elapsed_s": 120.53, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 26161, "completion_tokens": 355, "cache_read_tokens": 25394, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 26516, "response_id": "06dbb69693143fae5edaca25846e90cb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 16078, "latest_message_tokens": 33, "is_estimate": true, "response_id": "06dbb69693143fae5edaca25846e90cb"}, "latency_s": 3.1108596324920654} +{"seq": 18, "elapsed_s": 134.56, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 26785, "completion_tokens": 2134, "cache_read_tokens": 26160, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 28919, "response_id": "06dbb6993da646d7823b2a8885d1edcb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 16439, "latest_message_tokens": 288, "is_estimate": true, "response_id": "06dbb6993da646d7823b2a8885d1edcb"}, "latency_s": 13.99971318244934} +{"seq": 19, "elapsed_s": 137.24, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29001, "completion_tokens": 136, "cache_read_tokens": 26784, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29137, "response_id": "06dbb6a82af4672636c66a1084e0fde5"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19076, "latest_message_tokens": 97, "is_estimate": true, "response_id": "06dbb6a82af4672636c66a1084e0fde5"}, "latency_s": 1.9628024101257324} +{"seq": 20, "elapsed_s": 139.43, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29370, "completion_tokens": 81, "cache_read_tokens": 29000, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29451, "response_id": "06dbb6aa6a4d3226286d24c4219bfbbb"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19280, "latest_message_tokens": 248, "is_estimate": true, "response_id": "06dbb6aa6a4d3226286d24c4219bfbbb"}, "latency_s": 1.5076961517333984} +{"seq": 21, "elapsed_s": 142.08, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29656, "completion_tokens": 151, "cache_read_tokens": 29369, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29807, "response_id": "06dbb6ac589e63f83385ab634cb477d3"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19581, "latest_message_tokens": 213, "is_estimate": true, "response_id": "06dbb6ac589e63f83385ab634cb477d3"}, "latency_s": 1.9515719413757324} +{"seq": 22, "elapsed_s": 144.38, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 29900, "completion_tokens": 98, "cache_read_tokens": 29655, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 29998, "response_id": "06dbb6afe525c1edc3a6dc75b999a0e4"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 19921, "latest_message_tokens": 109, "is_estimate": true, "response_id": "06dbb6afe525c1edc3a6dc75b999a0e4"}, "latency_s": 1.608078956604004} +{"seq": 23, "elapsed_s": 146.67, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 30077, "completion_tokens": 85, "cache_read_tokens": 29899, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 30162, "response_id": "06dbb6b10c6b5e003ed4301422782c43"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 20100, "latest_message_tokens": 97, "is_estimate": true, "response_id": "06dbb6b10c6b5e003ed4301422782c43"}, "latency_s": 1.5816996097564697} +{"seq": 24, "elapsed_s": 150.05, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 32388, "completion_tokens": 328, "cache_read_tokens": 30076, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 32716, "response_id": "06dbb6b35357b2e3833200731df62b36"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 20250, "latest_message_tokens": 2182, "is_estimate": true, "response_id": "06dbb6b35357b2e3833200731df62b36"}, "latency_s": 3.3420867919921875} +{"seq": 25, "elapsed_s": 156.88, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 32741, "completion_tokens": 644, "cache_read_tokens": 32387, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 33385, "response_id": "06dbb6b67ec8afa4f8aa435b7ad5ffa5"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 22679, "latest_message_tokens": 38, "is_estimate": true, "response_id": "06dbb6b67ec8afa4f8aa435b7ad5ffa5"}, "latency_s": 6.802237033843994} diff --git a/.pr/evidence/minimax-m3/p09-task-07-summary.json b/.pr/evidence/minimax-m3/p09-task-07-summary.json new file mode 100644 index 0000000000..51f1c4ced7 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-07-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-07", + "prompt": "Refactor the long `build_monthly_report` function in `toyapp/reports.py` into smaller helpers or modules. Preserve behavior and keep the tests passing. Preserve this public contract:\n\n```python\nfrom toyapp.reports import build_monthly_report\nreport = build_monthly_report(rows, \"2026-05\")\nassert report[\"totals\"][\"revenue\"].startswith(\"$\")\nassert \"narrative\" in report\n```", + "model": "openai/MiniMax-M3", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 156.89, + "max_iterations_cap": 30, + "calls_recorded": 26, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "openai/MiniMax-M3", + "accumulated_cost": 0.0, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "openai/MiniMax-M3", + "prompt_tokens": 550568, + "completion_tokens": 18740, + "cache_read_tokens": 517930, + "cache_write_tokens": 0, + "reasoning_tokens": 6001, + "context_window": 0, + "per_turn_token": 33385, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/minimax-m3/p09-task-10-calls.jsonl b/.pr/evidence/minimax-m3/p09-task-10-calls.jsonl new file mode 100644 index 0000000000..5372fbb199 --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-10-calls.jsonl @@ -0,0 +1,10 @@ +{"seq": 0, "elapsed_s": 3.71, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 10979, "completion_tokens": 113, "cache_read_tokens": 2304, "cache_write_tokens": 0, "reasoning_tokens": 13, "context_window": 0, "per_turn_token": 11092, "response_id": "06dbb6c02df6125128fa2f99d51206bd"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 0, "latest_message_tokens": 255, "is_estimate": true, "response_id": "06dbb6c02df6125128fa2f99d51206bd"}, "latency_s": 3.247368335723877} +{"seq": 1, "elapsed_s": 6.05, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11453, "completion_tokens": 78, "cache_read_tokens": 10880, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11531, "response_id": "06dbb6c51037d633135dfb26db1fe400"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 335, "latest_message_tokens": 381, "is_estimate": true, "response_id": "06dbb6c51037d633135dfb26db1fe400"}, "latency_s": 1.3597357273101807} +{"seq": 2, "elapsed_s": 9.54, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11829, "completion_tokens": 67, "cache_read_tokens": 11520, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 11896, "response_id": "06dbb6c6c2e331e83ee04dd2cd1679c9"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 763, "latest_message_tokens": 302, "is_estimate": true, "response_id": "06dbb6c6c2e331e83ee04dd2cd1679c9"}, "latency_s": 3.458706855773926} +{"seq": 3, "elapsed_s": 23.35, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 11998, "completion_tokens": 69, "cache_read_tokens": 11776, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 12067, "response_id": "06dbb6caf4498837aeb03583240e0d6f"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1105, "latest_message_tokens": 109, "is_estimate": true, "response_id": "06dbb6caf4498837aeb03583240e0d6f"}, "latency_s": 13.118934154510498} +{"seq": 4, "elapsed_s": 73.01, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12213, "completion_tokens": 5043, "cache_read_tokens": 12032, "cache_write_tokens": 0, "reasoning_tokens": 4902, "context_window": 0, "per_turn_token": 17256, "response_id": "06dbb6d8a85fc161b8d0ebd3d5c03545"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1258, "latest_message_tokens": 159, "is_estimate": true, "response_id": "06dbb6d8a85fc161b8d0ebd3d5c03545"}, "latency_s": 48.94923162460327} +{"seq": 5, "elapsed_s": 250.2, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12491, "completion_tokens": 20022, "cache_read_tokens": 12160, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 32513, "response_id": "06dbb70a42dc5e7ea00df3555590e2a1"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1535, "latest_message_tokens": 145, "is_estimate": true, "response_id": "06dbb70a42dc5e7ea00df3555590e2a1"}, "latency_s": 176.5030059814453} +{"seq": 6, "elapsed_s": 326.49, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 12613, "completion_tokens": 3445, "cache_read_tokens": 12416, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 16058, "response_id": "06dbb7ba0939ce1368334e3af96e69dc"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 1727, "latest_message_tokens": 54, "is_estimate": true, "response_id": "06dbb7ba0939ce1368334e3af96e69dc"}, "latency_s": 76.26301431655884} +{"seq": 7, "elapsed_s": 333.87, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 16079, "completion_tokens": 1051, "cache_read_tokens": 16000, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17130, "response_id": "06dbb8069722fbc7cd274869cf0db873"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 5519, "latest_message_tokens": 34, "is_estimate": true, "response_id": "06dbb8069722fbc7cd274869cf0db873"}, "latency_s": 7.361048221588135} +{"seq": 8, "elapsed_s": 338.86, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 17171, "completion_tokens": 395, "cache_read_tokens": 17024, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 17566, "response_id": "06dbb80eaf94f0fec2c19a86ce37214b"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 6676, "latest_message_tokens": 51, "is_estimate": true, "response_id": "06dbb80eaf94f0fec2c19a86ce37214b"}, "latency_s": 4.976585626602173} +{"seq": 9, "elapsed_s": 348.48, "usage": {"model": "openai/MiniMax-M3", "prompt_tokens": 18232, "completion_tokens": 742, "cache_read_tokens": 17536, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 0, "per_turn_token": 18974, "response_id": "06dbb813844ff4664a61e2ab25135957"}, "composition": {"model": "openai/MiniMax-M3", "system_prompt_tokens": 3331, "tool_tokens": 5749, "history_tokens": 7135, "latest_message_tokens": 664, "is_estimate": true, "response_id": "06dbb813844ff4664a61e2ab25135957"}, "latency_s": 8.94175910949707} diff --git a/.pr/evidence/minimax-m3/p09-task-10-summary.json b/.pr/evidence/minimax-m3/p09-task-10-summary.json new file mode 100644 index 0000000000..d6157fd23c --- /dev/null +++ b/.pr/evidence/minimax-m3/p09-task-10-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-10", + "prompt": "Implement an in-memory caching layer in `toyapp/cache.py` with `CachingAPIClient`. Calling `CachingAPIClient()` with no client should wrap `toyapp.api.APIClient`. The constructor should also accept an optional backend client, `clock` callable, `user_ttl`, and `account_summary_ttl`. Preserve `fetch_user(user_id)` and `fetch_account_summary(user_id)`, use separate read-through TTL caches for users and account summaries, expose `invalidate_user` and `invalidate_account_summary`, and make `invalidate_user` clear both the user cache and that user's dependent account summary cache. Coalesce concurrent requests for the same key without serializing different keys behind one long backend lock. Return stale cached data on backend failure when available, but do not mark that stale value fresh; the next call should retry the backend. Propagate the backend error to all waiters when a failed fill has no stale value, clear failed fills so a later retry can succeed, and expose `get_metrics()` with `cache_hit_total`, `cache_miss_total`, `cache_fill_seconds`, and `cache_invalidation_total`. Count `cache_miss_total` once per backend fill attempt; coalesced waiters must not inflate it.", + "model": "openai/MiniMax-M3", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 348.49, + "max_iterations_cap": 30, + "calls_recorded": 10, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "openai/MiniMax-M3", + "accumulated_cost": 0.0, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "openai/MiniMax-M3", + "prompt_tokens": 135058, + "completion_tokens": 31025, + "cache_read_tokens": 123648, + "cache_write_tokens": 0, + "reasoning_tokens": 4915, + "context_window": 0, + "per_turn_token": 18974, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file From 9bc895d423c51717bd6b6f98f78b0ba59717d3f1 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 24 Aug 2026 16:18:17 -0400 Subject: [PATCH 04/15] chore(pr): add gpt-4o-mini mapped-tokenizer evidence lane Co-authored-by: openhands --- .pr/evidence/analysis.md | 27 ++++++++++++++++++ .../gpt-4o-mini/p09-task-01-calls.jsonl | 9 ++++++ .../gpt-4o-mini/p09-task-01-summary.json | 28 +++++++++++++++++++ .../gpt-4o-mini/p09-task-07-calls.jsonl | 28 +++++++++++++++++++ .../gpt-4o-mini/p09-task-07-summary.json | 28 +++++++++++++++++++ 5 files changed, 120 insertions(+) create mode 100644 .pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl create mode 100644 .pr/evidence/gpt-4o-mini/p09-task-01-summary.json create mode 100644 .pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl create mode 100644 .pr/evidence/gpt-4o-mini/p09-task-07-summary.json diff --git a/.pr/evidence/analysis.md b/.pr/evidence/analysis.md index 60492adfd4..580f453743 100644 --- a/.pr/evidence/analysis.md +++ b/.pr/evidence/analysis.md @@ -15,6 +15,33 @@ browser-on) on tasks from the published harness-benchmark short suite this model, so estimates use litellm's fallback tokenizer). Tasks p09-task-01 (trivial rename, 5 calls), p09-task-07 (medium refactor, 26 calls), p09-task-10 (hard cache implementation, 10 calls). +- `gpt-4o-mini/` — `gpt-4o-mini` (chat completions; litellm maps this model to its real + `o200k_base` tokenizer — verified by comparing `token_counter` framing overhead against + `tiktoken.get_encoding("o200k_base")` vs `cl100k_base` on divergent inputs). Tasks + p09-task-01 (9 calls, verifier PASS) and p09-task-07 (28 calls, verifier FAIL — the model + left a syntax error in the repo; a completed run, labeled a model-quality failure). + +## gpt-4o-mini summary (mapped tokenizer) + +| task | calls | avg system | avg tools | avg history | avg latest | avg provider input | est/provider median | +|---|---:|---:|---:|---:|---:|---:|---:| +| p09-task-01 | 9 | 3,340 | 5,702 | 987 | 175 | 10,231 | 1.00 | +| p09-task-07 | 28 | 3,340 | 5,702 | 18,357 | 792 | 28,739 | 0.98 | + +- Per-call est/provider ratio band: 0.99–1.00 (task-01), 0.97–0.99 (task-07) — versus + 0.85–0.97 on the unmapped MiniMax-M3 lane. The mapped-tokenizer band tightens to ~1.0, so + the MiniMax underestimate was dominated by the fallback tokenizer; a residual ~1–3% + underestimate remains, consistent with uncounted request framing and litellm's tool + serialization convention. +- One notable event: between calls 9 and 10 of task-07, history dropped 30.8K → 10.0K tokens + with **no intervening LLM call** — view-property enforcement + (`View.enforce_properties`: batch/observation/atomicity properties drop events without a + summarization call), not an LLM-summarizing condensation. The composition tracked the + shrunken view exactly (ratio stayed 0.99 across the drop), which is itself evidence the + history bucket measures the payload actually sent. +- Total spend for this lane (litellm-computed `accumulated_cost`, PAYG key): $0.0081 + (task-01) + $0.0808 (task-07) ≈ **$0.089**; 87% of prompt tokens were cache reads. + ## MiniMax-M3 summary diff --git a/.pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl b/.pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl new file mode 100644 index 0000000000..de1c8904fe --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-01-calls.jsonl @@ -0,0 +1,9 @@ +{"seq": 0, "elapsed_s": 3.12, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9172, "completion_tokens": 38, "cache_read_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9210, "response_id": "chatcmpl-EGVEG8NSZIEywDIyNQnqt99X5R4tJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 0, "latest_message_tokens": 38, "is_estimate": true, "response_id": "chatcmpl-EGVEG8NSZIEywDIyNQnqt99X5R4tJ"}, "latency_s": 2.5493357181549072} +{"seq": 1, "elapsed_s": 4.67, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9675, "completion_tokens": 63, "cache_read_tokens": 9088, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9738, "response_id": "chatcmpl-EGVEICEloV8BjwzBVpfYkPDKSNhVb"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 73, "latest_message_tokens": 486, "is_estimate": true, "response_id": "chatcmpl-EGVEICEloV8BjwzBVpfYkPDKSNhVb"}, "latency_s": 1.5301110744476318} +{"seq": 2, "elapsed_s": 7.12, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9797, "completion_tokens": 61, "cache_read_tokens": 9600, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9858, "response_id": "chatcmpl-EGVEJHP3YvqiBxIXWUbhpgqhTOhE7"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 620, "latest_message_tokens": 77, "is_estimate": true, "response_id": "chatcmpl-EGVEJHP3YvqiBxIXWUbhpgqhTOhE7"}, "latency_s": 2.4455978870391846} +{"seq": 3, "elapsed_s": 8.73, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10054, "completion_tokens": 63, "cache_read_tokens": 9728, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10117, "response_id": "chatcmpl-EGVEMHTRF4V3DUDvosBEZHNMSya0m"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 755, "latest_message_tokens": 213, "is_estimate": true, "response_id": "chatcmpl-EGVEMHTRF4V3DUDvosBEZHNMSya0m"}, "latency_s": 1.593902826309204} +{"seq": 4, "elapsed_s": 10.05, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10309, "completion_tokens": 63, "cache_read_tokens": 9984, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10372, "response_id": "chatcmpl-EGVENFynBkjyDGNnJZvpU8I5yjIcr"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1028, "latest_message_tokens": 212, "is_estimate": true, "response_id": "chatcmpl-EGVENFynBkjyDGNnJZvpU8I5yjIcr"}, "latency_s": 1.288510799407959} +{"seq": 5, "elapsed_s": 10.95, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10539, "completion_tokens": 32, "cache_read_tokens": 10240, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10571, "response_id": "chatcmpl-EGVEO5r1bOirNFiH5qiMMXW33T1og"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1301, "latest_message_tokens": 187, "is_estimate": true, "response_id": "chatcmpl-EGVEO5r1bOirNFiH5qiMMXW33T1og"}, "latency_s": 0.8807761669158936} +{"seq": 6, "elapsed_s": 13.23, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10669, "completion_tokens": 29, "cache_read_tokens": 10496, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10698, "response_id": "chatcmpl-EGVEQBtKdhnHab1gSiwzkourerokO"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1517, "latest_message_tokens": 118, "is_estimate": true, "response_id": "chatcmpl-EGVEQBtKdhnHab1gSiwzkourerokO"}, "latency_s": 1.3414945602416992} +{"seq": 7, "elapsed_s": 15.53, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 10787, "completion_tokens": 97, "cache_read_tokens": 10624, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 10884, "response_id": "chatcmpl-EGVESk7f9F0Vx4trTv6Cr3B7g2hM8"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1661, "latest_message_tokens": 110, "is_estimate": true, "response_id": "chatcmpl-EGVESk7f9F0Vx4trTv6Cr3B7g2hM8"}, "latency_s": 1.62056303024292} +{"seq": 8, "elapsed_s": 18.22, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 11074, "completion_tokens": 57, "cache_read_tokens": 10752, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 11131, "response_id": "chatcmpl-EGVEVrPlyWeD92ZjuW892YFNWzZvj"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 1929, "latest_message_tokens": 135, "is_estimate": true, "response_id": "chatcmpl-EGVEVrPlyWeD92ZjuW892YFNWzZvj"}, "latency_s": 1.2971453666687012} diff --git a/.pr/evidence/gpt-4o-mini/p09-task-01-summary.json b/.pr/evidence/gpt-4o-mini/p09-task-01-summary.json new file mode 100644 index 0000000000..514d3a50db --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-01-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-01", + "prompt": "Rename the local variable `raw_items` to `source_items` in `toyapp/pagination.py`. Keep behavior unchanged and do not edit unrelated files.", + "model": "gpt-4o-mini", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.FINISHED", + "wall_clock_s": 18.22, + "max_iterations_cap": 30, + "calls_recorded": 9, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0080748, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 92076, + "completion_tokens": 503, + "cache_read_tokens": 80512, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 11131, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file diff --git a/.pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl b/.pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl new file mode 100644 index 0000000000..6616c63d8c --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-07-calls.jsonl @@ -0,0 +1,28 @@ +{"seq": 0, "elapsed_s": 2.19, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 9229, "completion_tokens": 54, "cache_read_tokens": 3968, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 9283, "response_id": "chatcmpl-EGVEaBHtEAXFwTfJuqnKMyvx8qt2z"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 0, "latest_message_tokens": 95, "is_estimate": true, "response_id": "chatcmpl-EGVEaBHtEAXFwTfJuqnKMyvx8qt2z"}, "latency_s": 1.6030583381652832} +{"seq": 1, "elapsed_s": 65.76, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 11615, "completion_tokens": 6265, "cache_read_tokens": 9216, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 17880, "response_id": "chatcmpl-EGVEbVMTDkqWJ4ho6EWwfTYTYLX7V"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 150, "latest_message_tokens": 2352, "is_estimate": true, "response_id": "chatcmpl-EGVEbVMTDkqWJ4ho6EWwfTYTYLX7V"}, "latency_s": 63.55660820007324} +{"seq": 2, "elapsed_s": 68.08, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 20060, "completion_tokens": 39, "cache_read_tokens": 17792, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 20099, "response_id": "chatcmpl-EGVFdiOXjmtDk4kIMmRdtl4UGj3jt"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10561, "latest_message_tokens": 216, "is_estimate": true, "response_id": "chatcmpl-EGVFdiOXjmtDk4kIMmRdtl4UGj3jt"}, "latency_s": 2.2881739139556885} +{"seq": 3, "elapsed_s": 91.85, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 22458, "completion_tokens": 2420, "cache_read_tokens": 19968, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 24878, "response_id": "chatcmpl-EGVFffVj591GMksiyQGFGh5v17bTa"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10810, "latest_message_tokens": 2380, "is_estimate": true, "response_id": "chatcmpl-EGVFffVj591GMksiyQGFGh5v17bTa"}, "latency_s": 23.761120080947876} +{"seq": 4, "elapsed_s": 114.08, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 24943, "completion_tokens": 2464, "cache_read_tokens": 22656, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 27407, "response_id": "chatcmpl-EGVG3sUfYwhR45fL1Ih0CxqVB21Rg"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 15589, "latest_message_tokens": 97, "is_estimate": true, "response_id": "chatcmpl-EGVG3sUfYwhR45fL1Ih0CxqVB21Rg"}, "latency_s": 22.207499265670776} +{"seq": 5, "elapsed_s": 115.49, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 32129, "completion_tokens": 58, "cache_read_tokens": 27264, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 32187, "response_id": "chatcmpl-EGVGPFc4ebzbYSDpp2rBoOlsTSvqF"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 21761, "latest_message_tokens": 509, "is_estimate": true, "response_id": "chatcmpl-EGVGPFc4ebzbYSDpp2rBoOlsTSvqF"}, "latency_s": 1.3639793395996094} +{"seq": 6, "elapsed_s": 116.7, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 36633, "completion_tokens": 34, "cache_read_tokens": 32128, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 36667, "response_id": "chatcmpl-EGVGRGyXamE9eBbbF4Qj3Gznwa1fo"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 22326, "latest_message_tokens": 4467, "is_estimate": true, "response_id": "chatcmpl-EGVGRGyXamE9eBbbF4Qj3Gznwa1fo"}, "latency_s": 1.177990436553955} +{"seq": 7, "elapsed_s": 128.0, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 37133, "completion_tokens": 956, "cache_read_tokens": 36608, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 38089, "response_id": "chatcmpl-EGVGTJYxwCVP3edBFNELpYeZsXWjJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 26824, "latest_message_tokens": 484, "is_estimate": true, "response_id": "chatcmpl-EGVGTJYxwCVP3edBFNELpYeZsXWjJ"}, "latency_s": 10.288436889648438} +{"seq": 8, "elapsed_s": 129.14, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 40656, "completion_tokens": 34, "cache_read_tokens": 37888, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 40690, "response_id": "chatcmpl-EGVGdViETiz1nvcAvkIdeyzAsfvWX"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 30436, "latest_message_tokens": 322, "is_estimate": true, "response_id": "chatcmpl-EGVGdViETiz1nvcAvkIdeyzAsfvWX"}, "latency_s": 1.0821290016174316} +{"seq": 9, "elapsed_s": 139.51, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 41156, "completion_tokens": 961, "cache_read_tokens": 40576, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 42117, "response_id": "chatcmpl-EGVGfqL1j1yIm0mHUoprQQMM8JsYq"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 30789, "latest_message_tokens": 485, "is_estimate": true, "response_id": "chatcmpl-EGVGfqL1j1yIm0mHUoprQQMM8JsYq"}, "latency_s": 9.654026985168457} +{"seq": 10, "elapsed_s": 145.03, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 19579, "completion_tokens": 35, "cache_read_tokens": 11008, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 19614, "response_id": "chatcmpl-EGVGteyULujx1MocC9a9i12g22BMd"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10028, "latest_message_tokens": 323, "is_estimate": true, "response_id": "chatcmpl-EGVGteyULujx1MocC9a9i12g22BMd"}, "latency_s": 1.008913278579712} +{"seq": 11, "elapsed_s": 168.16, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 20067, "completion_tokens": 943, "cache_read_tokens": 19584, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 21010, "response_id": "chatcmpl-EGVGvliksgwWU5TTpeW0QiW5yRQXa"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 10383, "latest_message_tokens": 475, "is_estimate": true, "response_id": "chatcmpl-EGVGvliksgwWU5TTpeW0QiW5yRQXa"}, "latency_s": 22.469878435134888} +{"seq": 12, "elapsed_s": 169.26, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 23599, "completion_tokens": 39, "cache_read_tokens": 20864, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 23638, "response_id": "chatcmpl-EGVHIKWrVQFCZN8GphndiPs28RtVQ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 14000, "latest_message_tokens": 323, "is_estimate": true, "response_id": "chatcmpl-EGVHIKWrVQFCZN8GphndiPs28RtVQ"}, "latency_s": 1.0678575038909912} +{"seq": 13, "elapsed_s": 173.16, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 24104, "completion_tokens": 193, "cache_read_tokens": 23552, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 24297, "response_id": "chatcmpl-EGVHJfDADcTSqSwejg976KyrLZfrg"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 14359, "latest_message_tokens": 486, "is_estimate": true, "response_id": "chatcmpl-EGVHJfDADcTSqSwejg976KyrLZfrg"}, "latency_s": 3.2294814586639404} +{"seq": 14, "elapsed_s": 179.02, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 28754, "completion_tokens": 418, "cache_read_tokens": 24192, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 29172, "response_id": "chatcmpl-EGVHNAjkt5ocTthcGC8GJEX3YiHBp"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 15035, "latest_message_tokens": 4474, "is_estimate": true, "response_id": "chatcmpl-EGVHNAjkt5ocTthcGC8GJEX3YiHBp"}, "latency_s": 5.823587894439697} +{"seq": 15, "elapsed_s": 201.98, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 29587, "completion_tokens": 1273, "cache_read_tokens": 29056, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 30860, "response_id": "chatcmpl-EGVHSp7sw6cWsbwWqzVkuZ61XGxEJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 19924, "latest_message_tokens": 431, "is_estimate": true, "response_id": "chatcmpl-EGVHSp7sw6cWsbwWqzVkuZ61XGxEJ"}, "latency_s": 22.9333074092865} +{"seq": 16, "elapsed_s": 210.44, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 33078, "completion_tokens": 801, "cache_read_tokens": 30720, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 33879, "response_id": "chatcmpl-EGVHpcFMO3Xuvsg0wYFUXbZdpIknq"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 22879, "latest_message_tokens": 626, "is_estimate": true, "response_id": "chatcmpl-EGVHpcFMO3Xuvsg0wYFUXbZdpIknq"}, "latency_s": 8.41625690460205} +{"seq": 17, "elapsed_s": 211.58, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 35244, "completion_tokens": 39, "cache_read_tokens": 33792, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 35283, "response_id": "chatcmpl-EGVHyhnq2Dh8HwsuKrH64dRsymNst"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 25093, "latest_message_tokens": 429, "is_estimate": true, "response_id": "chatcmpl-EGVHyhnq2Dh8HwsuKrH64dRsymNst"}, "latency_s": 1.1085383892059326} +{"seq": 18, "elapsed_s": 216.29, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 35749, "completion_tokens": 349, "cache_read_tokens": 0, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 36098, "response_id": "chatcmpl-EGVI0Nk3hyb6cbfgNTT7Dvla2P2lr"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 25558, "latest_message_tokens": 484, "is_estimate": true, "response_id": "chatcmpl-EGVI0Nk3hyb6cbfgNTT7Dvla2P2lr"}, "latency_s": 3.9800806045532227} +{"seq": 19, "elapsed_s": 230.63, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 36457, "completion_tokens": 1555, "cache_read_tokens": 35200, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 38012, "response_id": "chatcmpl-EGVI4tcM0letNC8zc3HFDqMPejIFm"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 26388, "latest_message_tokens": 378, "is_estimate": true, "response_id": "chatcmpl-EGVI4tcM0letNC8zc3HFDqMPejIFm"}, "latency_s": 14.30324649810791} +{"seq": 20, "elapsed_s": 235.18, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 28700, "completion_tokens": 39, "cache_read_tokens": 11136, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 28739, "response_id": "chatcmpl-EGVIL4IKSW9ZUQG1Vp5utKESF602P"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 18234, "latest_message_tokens": 539, "is_estimate": true, "response_id": "chatcmpl-EGVIL4IKSW9ZUQG1Vp5utKESF602P"}, "latency_s": 1.4693710803985596} +{"seq": 21, "elapsed_s": 238.2, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 29171, "completion_tokens": 177, "cache_read_tokens": 28672, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 29348, "response_id": "chatcmpl-EGVINa4VImNOx3jKptTsWgIJGXWEJ"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 18809, "latest_message_tokens": 452, "is_estimate": true, "response_id": "chatcmpl-EGVINa4VImNOx3jKptTsWgIJGXWEJ"}, "latency_s": 2.2953388690948486} +{"seq": 22, "elapsed_s": 241.14, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 29507, "completion_tokens": 272, "cache_read_tokens": 29312, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 29779, "response_id": "chatcmpl-EGVIQUULx7H0HkPwvSwPBFtBVO30I"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 19438, "latest_message_tokens": 174, "is_estimate": true, "response_id": "chatcmpl-EGVIQUULx7H0HkPwvSwPBFtBVO30I"}, "latency_s": 2.912266254425049} +{"seq": 23, "elapsed_s": 243.73, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 30032, "completion_tokens": 194, "cache_read_tokens": 29696, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 30226, "response_id": "chatcmpl-EGVITdkLHH8ywZBoOXP2kSLLWkRQ3"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 19881, "latest_message_tokens": 267, "is_estimate": true, "response_id": "chatcmpl-EGVITdkLHH8ywZBoOXP2kSLLWkRQ3"}, "latency_s": 2.567747116088867} +{"seq": 24, "elapsed_s": 248.38, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 30290, "completion_tokens": 457, "cache_read_tokens": 30208, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 30747, "response_id": "chatcmpl-EGVIVTZSaF7UByQ3SZYHlLMsk5pys"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 20338, "latest_message_tokens": 84, "is_estimate": true, "response_id": "chatcmpl-EGVIVTZSaF7UByQ3SZYHlLMsk5pys"}, "latency_s": 4.613501787185669} +{"seq": 25, "elapsed_s": 252.03, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 31089, "completion_tokens": 358, "cache_read_tokens": 30720, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 31447, "response_id": "chatcmpl-EGVIa2nGNiupew3LwWAn7M5KZeEhT"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 20876, "latest_message_tokens": 359, "is_estimate": true, "response_id": "chatcmpl-EGVIa2nGNiupew3LwWAn7M5KZeEhT"}, "latency_s": 3.6165292263031006} +{"seq": 26, "elapsed_s": 254.49, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 31538, "completion_tokens": 246, "cache_read_tokens": 31360, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 31784, "response_id": "chatcmpl-EGVIdfFtb5K0KEW7cjgnPbJWL7S2w"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 21588, "latest_message_tokens": 110, "is_estimate": true, "response_id": "chatcmpl-EGVIdfFtb5K0KEW7cjgnPbJWL7S2w"}, "latency_s": 2.427025079727173} +{"seq": 27, "elapsed_s": 256.43, "usage": {"model": "gpt-4o-mini", "prompt_tokens": 32123, "completion_tokens": 132, "cache_read_tokens": 31744, "cache_write_tokens": 0, "reasoning_tokens": 0, "context_window": 128000, "per_turn_token": 32255, "response_id": "chatcmpl-EGVIgAl8Pz7WD9Vi7Nr6KLIBTiC3T"}, "composition": {"model": "gpt-4o-mini", "system_prompt_tokens": 3340, "tool_tokens": 5702, "history_tokens": 21942, "latest_message_tokens": 359, "is_estimate": true, "response_id": "chatcmpl-EGVIgAl8Pz7WD9Vi7Nr6KLIBTiC3T"}, "latency_s": 1.9118013381958008} diff --git a/.pr/evidence/gpt-4o-mini/p09-task-07-summary.json b/.pr/evidence/gpt-4o-mini/p09-task-07-summary.json new file mode 100644 index 0000000000..5bfff7af48 --- /dev/null +++ b/.pr/evidence/gpt-4o-mini/p09-task-07-summary.json @@ -0,0 +1,28 @@ +{ + "task_id": "p09-task-07", + "prompt": "Refactor the long `build_monthly_report` function in `toyapp/reports.py` into smaller helpers or modules. Preserve behavior and keep the tests passing. Preserve this public contract:\n\n```python\nfrom toyapp.reports import build_monthly_report\nreport = build_monthly_report(rows, \"2026-05\")\nassert report[\"totals\"][\"revenue\"].startswith(\"$\")\nassert \"narrative\" in report\n```", + "model": "gpt-4o-mini", + "status": "completed", + "conversation_status": "ConversationExecutionStatus.ERROR", + "wall_clock_s": 256.44, + "max_iterations_cap": 30, + "calls_recorded": 28, + "condenser_shares_metrics": true, + "metrics_snapshot": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.080769, + "max_budget_per_task": null, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 804680, + "completion_tokens": 20805, + "cache_read_tokens": 698880, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 32255, + "response_id": "" + } + }, + "error": null +} \ No newline at end of file From 14a9f0cac7cfba1892be41191cf17ad0ebf9d4a0 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 24 Aug 2026 17:16:13 -0400 Subject: [PATCH 05/15] fix(sdk): offload composition counting off event loop, cover subscription payloads Address draft-PR review on prompt composition: - Async prepare paths now compute prompt composition via asyncio.to_thread (the pattern aformat_messages_for_responses uses for image inlining) instead of running up to 5 prompt-sized synchronous tokenization passes on the shared event loop; sync paths stay synchronous. - Subscription-mode Responses payloads normalize message items to {"role", "content"} without a "type" key; the payload converter now accepts that shape, so Codex subscription calls record compositions instead of silently skipping every call. - Nits: is_estimate documented as reserved for a future provider-reported mode; PromptComposition docstring notes divergence from the chat-template budget counter; probe comment corrected (messages=[] works in litellm 1.84.1); inline test imports moved to module top; tautological delta assertion annotated; input_image converter branch covered; async aresponses composition test added. Push-guard bypass: updating open draft PR #4623 with review fixes, approved by George. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/llm/llm.py | 113 +++++++++++++----- .../openhands/sdk/llm/utils/metrics.py | 9 +- .../sdk/llm/utils/prompt_composition.py | 25 ++-- tests/sdk/llm/test_prompt_composition.py | 83 ++++++++++++- 4 files changed, 187 insertions(+), 43 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index a05dfa7bac..526121a34f 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -1197,12 +1197,24 @@ def _prepare_completion_params( telemetry_ctx) """ formatted_messages = self.format_messages_for_llm(messages) - return self._finalize_completion_params( + formatted_messages, cc_tools, use_mock_tools, call_kwargs, telemetry_ctx = ( + self._finalize_completion_params( + formatted_messages, + tools, + add_security_risk_prediction, + kwargs, + call_context=call_context, + ) + ) + telemetry_ctx["prompt_composition"] = self._chat_prompt_composition( + formatted_messages, cc_tools, use_mock_tools + ) + return ( formatted_messages, - tools, - add_security_risk_prediction, - kwargs, - call_context=call_context, + cc_tools, + use_mock_tools, + call_kwargs, + telemetry_ctx, ) async def _aprepare_completion_params( @@ -1223,15 +1235,31 @@ async def _aprepare_completion_params( Uses :meth:`aformat_messages_for_llm` so the (potentially blocking) image-inlining pass is offloaded to a worker thread instead of running - on the event loop. + on the event loop; prompt composition counting is likewise offloaded + via :func:`asyncio.to_thread`. """ formatted_messages = await self.aformat_messages_for_llm(messages) - return self._finalize_completion_params( + formatted_messages, cc_tools, use_mock_tools, call_kwargs, telemetry_ctx = ( + self._finalize_completion_params( + formatted_messages, + tools, + add_security_risk_prediction, + kwargs, + call_context=call_context, + ) + ) + telemetry_ctx["prompt_composition"] = await asyncio.to_thread( + self._chat_prompt_composition, formatted_messages, - tools, - add_security_risk_prediction, - kwargs, - call_context=call_context, + cc_tools, + use_mock_tools, + ) + return ( + formatted_messages, + cc_tools, + use_mock_tools, + call_kwargs, + telemetry_ctx, ) def _finalize_completion_params( @@ -1307,14 +1335,6 @@ def _finalize_completion_params( telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0, - # When tool schemas are mocked into the prompt text, they are - # already inside the message buckets — don't count them twice. - "prompt_composition": compute_prompt_composition( - model=self.model, - messages=formatted_messages, - tools=None if use_mock_tools else cc_tools or None, - custom_tokenizer=self._tokenizer, - ), } if telemetry.log_enabled: telemetry_ctx.update( @@ -1335,6 +1355,24 @@ def _finalize_completion_params( telemetry_ctx, ) + def _chat_prompt_composition( + self, + formatted_messages: list[dict[str, Any]], + cc_tools: list[ChatCompletionToolParam], + use_mock_tools: bool, + ) -> PromptComposition | None: + """Prompt composition for the chat path. + + When tool schemas are mocked into the prompt text, they are already + inside the message buckets — don't count them twice. + """ + return compute_prompt_composition( + model=self.model, + messages=formatted_messages, + tools=None if use_mock_tools else cc_tools or None, + custom_tokenizer=self._tokenizer, + ) + def _prepare_responses_params( self, messages: list[Message], @@ -1358,7 +1396,13 @@ def _prepare_responses_params( telemetry_ctx) """ instructions, input_items = self.format_messages_for_responses(messages) - return self._finalize_responses_params( + ( + instructions, + input_items, + resp_tools, + call_kwargs, + telemetry_ctx, + ) = self._finalize_responses_params( instructions, input_items, tools, @@ -1368,6 +1412,10 @@ def _prepare_responses_params( kwargs, call_context=call_context, ) + telemetry_ctx["prompt_composition"] = self._responses_prompt_composition( + instructions, input_items, tools, add_security_risk_prediction + ) + return instructions, input_items, resp_tools, call_kwargs, telemetry_ctx async def _aprepare_responses_params( self, @@ -1388,10 +1436,17 @@ async def _aprepare_responses_params( """Async variant of :meth:`_prepare_responses_params`. Uses :meth:`aformat_messages_for_responses` so the image-inlining - pass runs off the event loop. + pass runs off the event loop; prompt composition counting is likewise + offloaded via :func:`asyncio.to_thread`. """ instructions, input_items = await self.aformat_messages_for_responses(messages) - return self._finalize_responses_params( + ( + instructions, + input_items, + resp_tools, + call_kwargs, + telemetry_ctx, + ) = self._finalize_responses_params( instructions, input_items, tools, @@ -1401,6 +1456,14 @@ async def _aprepare_responses_params( kwargs, call_context=call_context, ) + telemetry_ctx["prompt_composition"] = await asyncio.to_thread( + self._responses_prompt_composition, + instructions, + input_items, + tools, + add_security_risk_prediction, + ) + return instructions, input_items, resp_tools, call_kwargs, telemetry_ctx def _finalize_responses_params( self, @@ -1451,12 +1514,6 @@ def _finalize_responses_params( telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0, - "prompt_composition": self._responses_prompt_composition( - instructions, - input_items, - tools, - add_security_risk_prediction, - ), } if telemetry.log_enabled: telemetry_ctx.update( diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index f3a178d9d8..1b7cba4219 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -82,7 +82,11 @@ class PromptComposition(BaseModel): included in every bucket and the components do not necessarily sum exactly to the provider-reported ``prompt_tokens``. Tool schema counts follow litellm's ``token_counter`` serialization convention for tools, - which can differ from the provider's wire-format tokenization. + which can differ from the provider's wire-format tokenization. For + models configured with a chat-template tokenizer, counts can also + diverge from ``LLM.get_token_count``, which prefers the chat-template + path; the composition always uses litellm's generic counter so numbers + stay comparable across models. """ model: str = Field(default="") @@ -106,7 +110,8 @@ class PromptComposition(BaseModel): is_estimate: bool = Field( default=True, description="True when counts are client-side estimates rather than " - "provider-reported usage", + "provider-reported usage; reserved for a future provider-reported " + "mode (no current path sets it to False)", ) response_id: str = Field(default="") diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py index e8a859f3fd..4b88ab9d56 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -11,8 +11,10 @@ logger = get_logger(__name__) -# token_counter requires at least one message when tools are passed, so tool -# schema tokens are measured as the marginal cost over an empty probe message. +# token_counter requires a messages argument when tools are passed, so tool +# schema tokens are measured as the marginal cost over an empty probe message +# (messages=[] would also work in litellm 1.84.1; the probe keeps the call +# shape explicit either way). _TOOLS_PROBE_MESSAGES: list[dict[str, Any]] = [{"role": "user", "content": ""}] @@ -38,9 +40,10 @@ def compute_prompt_composition( no tokens at all (e.g. ``litellm.disable_token_counter``), since composition recording is best-effort. - Cost scales linearly with prompt size: measured ~31 ms for a ~100K-token - prompt and ~61 ms for ~190K tokens (gpt-4o tokenizer, 19 tools), versus - ~10-20 ms on typical agent-step payloads. + Counting is always on by deliberate choice: cost scales linearly with + prompt size and stays small in absolute terms — measured ~31 ms for a + ~100K-token prompt and ~61 ms for ~190K tokens (gpt-4o tokenizer, 19 + tools), versus ~10-20 ms on typical agent-step payloads. """ def count( @@ -113,14 +116,18 @@ def responses_payload_to_chat_messages( def _responses_item_to_chat(item: dict[str, Any]) -> list[dict[str, Any]]: item_type = item.get("type") + if item_type is None and "role" in item and "content" in item: + # Subscription mode normalizes message items to {"role", "content"} + # without a "type" key (see transform_for_subscription). + item_type = "message" if item_type == "message": + content = item.get("content", "") + if isinstance(content, str): + return [{"role": item["role"], "content": content}] return [ { "role": item["role"], - "content": [ - _responses_content_part_to_chat(part) - for part in item.get("content", []) - ], + "content": [_responses_content_part_to_chat(part) for part in content], } ] if item_type == "function_call": diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index f8ea8885e1..aeaec019e5 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -12,6 +12,7 @@ ModelResponse, Usage, ) +from litellm.utils import token_counter from openai.types.responses.response_output_message import ResponseOutputMessage from openai.types.responses.response_output_text import ResponseOutputText from pydantic import SecretStr @@ -88,8 +89,6 @@ def test_compute_prompt_composition_decomposes_into_components(): assert composition.latest_message_tokens > 0 # Buckets are counted independently with per-message framing overhead, # so their sum covers the message content of the whole prompt. - from litellm.utils import token_counter - total_messages = token_counter(model="gpt-4o", messages=formatted) component_sum = ( composition.system_prompt_tokens @@ -138,8 +137,6 @@ def test_compute_prompt_composition_skips_all_zero_records(): def test_compute_prompt_composition_tool_tokens_matches_controlled_delta(): """tool_tokens must equal the marginal cost of adding tools to the call.""" - from litellm.utils import token_counter - formatted = _make_llm().format_messages_for_llm(_sample_messages()) tools = [ t.to_openai_tool(add_security_risk_prediction=True) for t in _MockTool.create() @@ -163,6 +160,8 @@ def test_compute_prompt_composition_tool_tokens_matches_controlled_delta(): + without_tools.history_tokens + without_tools.latest_message_tokens ) + # Deliberately pins the joint-counting assumption (tools never enter the + # message buckets); it is an identity by construction, kept as a tripwire. assert component_sum - other_components == with_tools.tool_tokens # And the estimate tracks the real marginal cost of the tools closely. @@ -407,3 +406,79 @@ def test_responses_payload_to_chat_messages_covers_sent_item_types(): with pytest.raises(ValueError, match="Unrecognized Responses input item"): responses_payload_to_chat_messages(None, [{"type": "mystery"}]) + + +def test_responses_payload_to_chat_messages_image_parts(): + chat = responses_payload_to_chat_messages( + None, + [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,AAAA", + "detail": "auto", + }, + ], + } + ], + ) + + assert chat == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + }, + ], + } + ] + + +def test_responses_composition_records_subscription_shaped_payload(): + """Subscription mode strips the "type" key from message items + (transform_for_subscription); a composition must still be recorded.""" + llm = LLM( + model="openai/gpt-5.2-codex", + base_url="https://chatgpt.com/backend-api/codex", + api_key=SecretStr("test"), + usage_id="test-llm", + reasoning_effort="high", + ) + llm._is_subscription = True + + with ( + patch( + "openhands.sdk.llm.llm.litellm_responses", + return_value=_responses_api_response(), + ), + patch.object(llm, "_get_litellm_auth_values", return_value=(None, {})), + ): + llm.responses(_sample_messages(), tools=list(_MockTool.create())) + + composition = llm.metrics.latest_prompt_composition + assert composition is not None + assert composition.history_tokens > 0 + assert composition.latest_message_tokens > 0 + + +async def test_aresponses_records_prompt_composition(): + llm = _make_llm("gpt-5-mini") + + with patch( + "openhands.sdk.llm.llm.litellm_aresponses", + new_callable=AsyncMock, + return_value=_responses_api_response(), + ): + await llm.aresponses(_sample_messages(), tools=list(_MockTool.create())) + + composition = llm.metrics.latest_prompt_composition + assert composition is not None + assert composition.response_id == "r1" + assert composition.tool_tokens > 0 From 208bba2ee5300cd451ac3459615673cb8f0bb932 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 24 Aug 2026 17:25:28 -0400 Subject: [PATCH 06/15] docs(sdk): document subscription-mode bucket behavior in composition converter Review-panel disposition (accepted tradeoff): in subscription mode the auth-layer transform folds the system prompt into the first user message, so those tokens are counted in history/latest rather than system_prompt_tokens. Buckets follow the wire payload, which is the documented contract for the Responses path; reclassifying would require guessing at a lossy fold. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py index 4b88ab9d56..e8e4a78cb4 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -105,6 +105,11 @@ def responses_payload_to_chat_messages( actually received (instructions + input items) while sharing the chat path's counting convention. Raises ValueError on unrecognized item types; callers treat any failure as "skip the composition record". + + Buckets follow the wire, not the logical prompt: in subscription mode the + system prompt is folded into the first user message by the auth-layer + transform, so those tokens land in ``history_tokens`` or + ``latest_message_tokens`` rather than ``system_prompt_tokens``. """ messages: list[dict[str, Any]] = [] if instructions: From 4a9c39df3752574a058a3c2869f7fc101fa3310e Mon Sep 17 00:00:00 2001 From: george larson Date: Tue, 25 Aug 2026 03:40:19 -0400 Subject: [PATCH 07/15] feat(sdk): make prompt composition opt-in via LLM.enable_prompt_composition Address rajshah4's review on draft PR #4623: put composition counting behind an opt-in flag (default off) so it is available for prompt troubleshooting and tool-loading studies without imposing the extra tokenization pass on every default user. When off, no composition is computed on any path (chat/responses, sync/async), no records are appended, and token_counter is never called; when on, behavior is unchanged from before. Push-guard bypass: updating open draft PR #4623 with review fixes, approved by George. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/llm/llm.py | 23 +++++++- .../sdk/llm/utils/prompt_composition.py | 10 ++-- tests/sdk/llm/test_prompt_composition.py | 57 ++++++++++++++++++- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 526121a34f..e991343818 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -523,6 +523,17 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): description="Whether to use native tool calling.", json_schema_extra=field_meta(), ) + enable_prompt_composition: bool = Field( + default=False, + description=( + "Whether to record a per-call prompt token composition estimate " + "(system prompt, tool schemas, history, latest message) into " + "Metrics. Opt-in: when False (default), no tokenization pass runs " + "and no records are appended. Enable when troubleshooting a " + "prompt or running a tool-loading study." + ), + json_schema_extra=field_meta(), + ) force_string_serializer: bool | None = Field( default=None, description=( @@ -1363,9 +1374,12 @@ def _chat_prompt_composition( ) -> PromptComposition | None: """Prompt composition for the chat path. - When tool schemas are mocked into the prompt text, they are already - inside the message buckets — don't count them twice. + Returns None without counting when ``enable_prompt_composition`` is + off. When tool schemas are mocked into the prompt text, they are + already inside the message buckets — don't count them twice. """ + if not self.enable_prompt_composition: + return None return compute_prompt_composition( model=self.model, messages=formatted_messages, @@ -1537,12 +1551,15 @@ def _responses_prompt_composition( ) -> PromptComposition | None: """Best-effort prompt composition for the Responses path. - Counts the finalized payload (instructions + input items) so the + Returns None without counting when ``enable_prompt_composition`` is + off. Counts the finalized payload (instructions + input items) so the record reflects what the provider received. Tool schemas are counted from their OpenAI chat-format equivalent so ``tool_tokens`` stays comparable with the chat path. Any serialization or conversion failure yields None rather than breaking the real call. """ + if not self.enable_prompt_composition: + return None try: chat_messages = responses_payload_to_chat_messages( instructions, input_items diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py index e8e4a78cb4..2e1b700dd3 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -40,10 +40,12 @@ def compute_prompt_composition( no tokens at all (e.g. ``litellm.disable_token_counter``), since composition recording is best-effort. - Counting is always on by deliberate choice: cost scales linearly with - prompt size and stays small in absolute terms — measured ~31 ms for a - ~100K-token prompt and ~61 ms for ~190K tokens (gpt-4o tokenizer, 19 - tools), versus ~10-20 ms on typical agent-step payloads. + Recording is opt-in: LLM call sites only invoke this when + ``LLM.enable_prompt_composition`` is True, so the counting cost is only + paid when explicitly enabled. Cost scales linearly with prompt size and + stays small in absolute terms — measured ~31 ms for a ~100K-token prompt + and ~61 ms for ~190K tokens (gpt-4o tokenizer, 19 tools), versus + ~10-20 ms on typical agent-step payloads. """ def count( diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index aeaec019e5..487a90d7e4 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -56,8 +56,13 @@ def _chat_response(response_id: str = "resp-1") -> ModelResponse: ) -def _make_llm(model: str = "gpt-4o") -> LLM: - return LLM(model=model, api_key=SecretStr("test"), usage_id="test-llm") +def _make_llm(model: str = "gpt-4o", enable_prompt_composition: bool = True) -> LLM: + return LLM( + model=model, + api_key=SecretStr("test"), + usage_id="test-llm", + enable_prompt_composition=enable_prompt_composition, + ) def _sample_messages() -> list[Message]: @@ -307,6 +312,7 @@ def test_mock_tools_does_not_double_count_tool_schemas(): api_key=SecretStr("test"), usage_id="test-llm", native_tool_calling=False, + enable_prompt_composition=True, ) with patch("openhands.sdk.llm.llm.litellm_completion", return_value=mock_response): @@ -450,6 +456,7 @@ def test_responses_composition_records_subscription_shaped_payload(): api_key=SecretStr("test"), usage_id="test-llm", reasoning_effort="high", + enable_prompt_composition=True, ) llm._is_subscription = True @@ -482,3 +489,49 @@ async def test_aresponses_records_prompt_composition(): assert composition is not None assert composition.response_id == "r1" assert composition.tool_tokens > 0 + + +def test_completion_default_off_records_no_composition(): + """With the flag at its default (off), no composition record is appended.""" + llm = LLM(model="gpt-4o", api_key=SecretStr("test"), usage_id="test-llm") + assert llm.enable_prompt_composition is False + + with patch( + "openhands.sdk.llm.llm.litellm_completion", return_value=_chat_response() + ): + llm.completion(messages=_sample_messages(), tools=list(_MockTool.create())) + + assert llm.metrics.prompt_compositions == [] + assert llm.metrics.latest_prompt_composition is None + + +def test_completion_off_never_calls_token_counter(): + """The off path must not pay any tokenization pass at all.""" + llm = _make_llm(enable_prompt_composition=False) + + with ( + patch( + "openhands.sdk.llm.llm.litellm_completion", + return_value=_chat_response(), + ), + patch( + "openhands.sdk.llm.utils.prompt_composition.token_counter" + ) as counter_spy, + ): + llm.completion(messages=_sample_messages(), tools=list(_MockTool.create())) + + counter_spy.assert_not_called() + assert llm.metrics.prompt_compositions == [] + + +def test_responses_default_off_records_no_composition(): + llm = _make_llm("gpt-5-mini", enable_prompt_composition=False) + + with patch( + "openhands.sdk.llm.llm.litellm_responses", + return_value=_responses_api_response(), + ): + llm.responses(_sample_messages(), tools=list(_MockTool.create())) + + assert llm.metrics.prompt_compositions == [] + assert llm.metrics.latest_prompt_composition is None From ef2ee557c905efc172ee7b02054901a3c9d1f406 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 31 Aug 2026 14:00:36 -0400 Subject: [PATCH 08/15] refactor(sdk): revert runtime composition wiring, keep offline helpers Repackage the per-call prompt token composition feature as an offline analysis utility: llm.py, metrics.py, telemetry.py and the package __init__ exports return to upstream/main content (no runtime flag, no Metrics plumbing, no new public API surface). The PromptComposition model moves from metrics.py into prompt_composition.py, its tool_tokens field is renamed to tool_schema_tokens, and the docstrings now describe offline counting over logged payloads instead of opt-in runtime recording. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/__init__.py | 2 - openhands-sdk/openhands/sdk/llm/__init__.py | 8 +- openhands-sdk/openhands/sdk/llm/llm.py | 167 ++---------------- .../openhands/sdk/llm/utils/metrics.py | 69 -------- .../sdk/llm/utils/prompt_composition.py | 81 +++++++-- .../openhands/sdk/llm/utils/telemetry.py | 8 +- 6 files changed, 81 insertions(+), 254 deletions(-) diff --git a/openhands-sdk/openhands/sdk/__init__.py b/openhands-sdk/openhands/sdk/__init__.py index 8e1c85e067..445dca8f19 100644 --- a/openhands-sdk/openhands/sdk/__init__.py +++ b/openhands-sdk/openhands/sdk/__init__.py @@ -32,7 +32,6 @@ LLMRegistry, LLMStreamChunk, Message, - PromptComposition, RedactedThinkingBlock, RegistryEvent, TextContent, @@ -128,7 +127,6 @@ "FallbackStrategy", "TokenCallbackType", "TokenUsage", - "PromptComposition", "ConversationStats", "RegistryEvent", "Message", diff --git a/openhands-sdk/openhands/sdk/llm/__init__.py b/openhands-sdk/openhands/sdk/llm/__init__.py index 5c72a093ac..01048f885a 100644 --- a/openhands-sdk/openhands/sdk/llm/__init__.py +++ b/openhands-sdk/openhands/sdk/llm/__init__.py @@ -34,12 +34,7 @@ LLMStreamChunk, TokenCallbackType, ) -from openhands.sdk.llm.utils.metrics import ( - Metrics, - MetricsSnapshot, - PromptComposition, - TokenUsage, -) +from openhands.sdk.llm.utils.metrics import Metrics, MetricsSnapshot, TokenUsage from openhands.sdk.llm.utils.runtime_metadata import ModelRuntimeMetadata from openhands.sdk.llm.utils.unverified_models import ( UNVERIFIED_MODELS_EXCLUDING_BEDROCK, @@ -84,7 +79,6 @@ # Metrics "Metrics", "MetricsSnapshot", - "PromptComposition", "TokenUsage", # Runtime metadata "ModelRuntimeMetadata", diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index df8ef79026..85d39409a0 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -117,17 +117,13 @@ ) from openhands.sdk.llm.utils.image_resize import maybe_resize_messages_for_provider from openhands.sdk.llm.utils.litellm_provider import LLMProvider -from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition +from openhands.sdk.llm.utils.metrics import Metrics from openhands.sdk.llm.utils.model_features import ModelFeatures, get_features from openhands.sdk.llm.utils.openhands_provider import ( LiteLLMCallKwargs, canonicalize_openhands_llm_payload, litellm_call_kwargs, ) -from openhands.sdk.llm.utils.prompt_composition import ( - compute_prompt_composition, - responses_payload_to_chat_messages, -) from openhands.sdk.llm.utils.retry_mixin import RetryMixin from openhands.sdk.llm.utils.telemetry import Telemetry from openhands.sdk.llm.utils.vertex_preflight import assert_vertex_sdk_available @@ -523,17 +519,6 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): description="Whether to use native tool calling.", json_schema_extra=field_meta(), ) - enable_prompt_composition: bool = Field( - default=False, - description=( - "Whether to record a per-call prompt token composition estimate " - "(system prompt, tool schemas, history, latest message) into " - "Metrics. Opt-in: when False (default), no tokenization pass runs " - "and no records are appended. Enable when troubleshooting a " - "prompt or running a tool-loading study." - ), - json_schema_extra=field_meta(), - ) force_string_serializer: bool | None = Field( default=None, description=( @@ -1218,24 +1203,12 @@ def _prepare_completion_params( telemetry_ctx) """ formatted_messages = self.format_messages_for_llm(messages) - formatted_messages, cc_tools, use_mock_tools, call_kwargs, telemetry_ctx = ( - self._finalize_completion_params( - formatted_messages, - tools, - add_security_risk_prediction, - kwargs, - call_context=call_context, - ) - ) - telemetry_ctx["prompt_composition"] = self._chat_prompt_composition( - formatted_messages, cc_tools, use_mock_tools - ) - return ( + return self._finalize_completion_params( formatted_messages, - cc_tools, - use_mock_tools, - call_kwargs, - telemetry_ctx, + tools, + add_security_risk_prediction, + kwargs, + call_context=call_context, ) async def _aprepare_completion_params( @@ -1256,31 +1229,15 @@ async def _aprepare_completion_params( Uses :meth:`aformat_messages_for_llm` so the (potentially blocking) image-inlining pass is offloaded to a worker thread instead of running - on the event loop; prompt composition counting is likewise offloaded - via :func:`asyncio.to_thread`. + on the event loop. """ formatted_messages = await self.aformat_messages_for_llm(messages) - formatted_messages, cc_tools, use_mock_tools, call_kwargs, telemetry_ctx = ( - self._finalize_completion_params( - formatted_messages, - tools, - add_security_risk_prediction, - kwargs, - call_context=call_context, - ) - ) - telemetry_ctx["prompt_composition"] = await asyncio.to_thread( - self._chat_prompt_composition, + return self._finalize_completion_params( formatted_messages, - cc_tools, - use_mock_tools, - ) - return ( - formatted_messages, - cc_tools, - use_mock_tools, - call_kwargs, - telemetry_ctx, + tools, + add_security_risk_prediction, + kwargs, + call_context=call_context, ) def _finalize_completion_params( @@ -1355,7 +1312,7 @@ def _finalize_completion_params( # logging is disabled. telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { - "context_window": self.effective_max_input_tokens or 0, + "context_window": self.effective_max_input_tokens or 0 } if telemetry.log_enabled: telemetry_ctx.update( @@ -1376,27 +1333,6 @@ def _finalize_completion_params( telemetry_ctx, ) - def _chat_prompt_composition( - self, - formatted_messages: list[dict[str, Any]], - cc_tools: list[ChatCompletionToolParam], - use_mock_tools: bool, - ) -> PromptComposition | None: - """Prompt composition for the chat path. - - Returns None without counting when ``enable_prompt_composition`` is - off. When tool schemas are mocked into the prompt text, they are - already inside the message buckets — don't count them twice. - """ - if not self.enable_prompt_composition: - return None - return compute_prompt_composition( - model=self.model, - messages=formatted_messages, - tools=None if use_mock_tools else cc_tools or None, - custom_tokenizer=self._tokenizer, - ) - def _prepare_responses_params( self, messages: list[Message], @@ -1420,13 +1356,7 @@ def _prepare_responses_params( telemetry_ctx) """ instructions, input_items = self.format_messages_for_responses(messages) - ( - instructions, - input_items, - resp_tools, - call_kwargs, - telemetry_ctx, - ) = self._finalize_responses_params( + return self._finalize_responses_params( instructions, input_items, tools, @@ -1436,10 +1366,6 @@ def _prepare_responses_params( kwargs, call_context=call_context, ) - telemetry_ctx["prompt_composition"] = self._responses_prompt_composition( - instructions, input_items, tools, add_security_risk_prediction - ) - return instructions, input_items, resp_tools, call_kwargs, telemetry_ctx async def _aprepare_responses_params( self, @@ -1460,17 +1386,10 @@ async def _aprepare_responses_params( """Async variant of :meth:`_prepare_responses_params`. Uses :meth:`aformat_messages_for_responses` so the image-inlining - pass runs off the event loop; prompt composition counting is likewise - offloaded via :func:`asyncio.to_thread`. + pass runs off the event loop. """ instructions, input_items = await self.aformat_messages_for_responses(messages) - ( - instructions, - input_items, - resp_tools, - call_kwargs, - telemetry_ctx, - ) = self._finalize_responses_params( + return self._finalize_responses_params( instructions, input_items, tools, @@ -1480,14 +1399,6 @@ async def _aprepare_responses_params( kwargs, call_context=call_context, ) - telemetry_ctx["prompt_composition"] = await asyncio.to_thread( - self._responses_prompt_composition, - instructions, - input_items, - tools, - add_security_risk_prediction, - ) - return instructions, input_items, resp_tools, call_kwargs, telemetry_ctx def _finalize_responses_params( self, @@ -1537,7 +1448,7 @@ def _finalize_responses_params( # logging is disabled. telemetry = self.telemetry telemetry_ctx: dict[str, Any] = { - "context_window": self.effective_max_input_tokens or 0, + "context_window": self.effective_max_input_tokens or 0 } if telemetry.log_enabled: telemetry_ctx.update( @@ -1552,50 +1463,6 @@ def _finalize_responses_params( return instructions, input_items, resp_tools, call_kwargs, telemetry_ctx - def _responses_prompt_composition( - self, - instructions: str | None, - input_items: list[dict[str, Any]], - tools: Sequence[ToolDefinition] | None, - add_security_risk_prediction: bool, - ) -> PromptComposition | None: - """Best-effort prompt composition for the Responses path. - - Returns None without counting when ``enable_prompt_composition`` is - off. Counts the finalized payload (instructions + input items) so the - record reflects what the provider received. Tool schemas are counted - from their OpenAI chat-format equivalent so ``tool_tokens`` stays - comparable with the chat path. Any serialization or conversion - failure yields None rather than breaking the real call. - """ - if not self.enable_prompt_composition: - return None - try: - chat_messages = responses_payload_to_chat_messages( - instructions, input_items - ) - cc_tools = ( - [ - t.to_openai_tool( - add_security_risk_prediction=add_security_risk_prediction, - ) - for t in tools - ] - if tools - else None - ) - except Exception: - logger.debug( - "Responses prompt composition skipped for %s", self.model, exc_info=True - ) - return None - return compute_prompt_composition( - model=self.model, - messages=chat_messages, - tools=cc_tools, - custom_tokenizer=self._tokenizer, - ) - def _validate_chat_response( self, resp: ModelResponse, diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 1b7cba4219..1a3a23f421 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -73,49 +73,6 @@ def __add__(self, other: "TokenUsage") -> "TokenUsage": ) -class PromptComposition(BaseModel): - """Per-call decomposition of prompt tokens by component. - - Counts are client-side estimates computed before the request is sent; - the provider-reported ``TokenUsage`` remains authoritative. Each - component is counted independently, so per-message framing overhead is - included in every bucket and the components do not necessarily sum - exactly to the provider-reported ``prompt_tokens``. Tool schema counts - follow litellm's ``token_counter`` serialization convention for tools, - which can differ from the provider's wire-format tokenization. For - models configured with a chat-template tokenizer, counts can also - diverge from ``LLM.get_token_count``, which prefers the chat-template - path; the composition always uses litellm's generic counter so numbers - stay comparable across models. - """ - - model: str = Field(default="") - system_prompt_tokens: int = Field( - default=0, ge=0, description="Estimated tokens in system messages" - ) - tool_tokens: int = Field( - default=0, ge=0, description="Estimated tokens in tool schemas" - ) - history_tokens: int = Field( - default=0, - ge=0, - description="Estimated tokens in conversation history (all non-system " - "messages except the latest one)", - ) - latest_message_tokens: int = Field( - default=0, - ge=0, - description="Estimated tokens in the latest observation/user message", - ) - is_estimate: bool = Field( - default=True, - description="True when counts are client-side estimates rather than " - "provider-reported usage; reserved for a future provider-reported " - "mode (no current path sets it to False)", - ) - response_id: str = Field(default="") - - class MetricsSnapshot(BaseModel): """A snapshot of metrics at a point in time. @@ -171,15 +128,6 @@ class Metrics(MetricsSnapshot): token_usages: list[TokenUsage] = Field( default_factory=list, description="List of token usage records" ) - prompt_compositions: list[PromptComposition] = Field( - default_factory=list, - description="Per-call prompt token composition estimates, one per call", - ) - - @property - def latest_prompt_composition(self) -> PromptComposition | None: - """The most recent per-call prompt composition, if any.""" - return self.prompt_compositions[-1] if self.prompt_compositions else None @field_validator("accumulated_cost") @classmethod @@ -271,14 +219,6 @@ def add_token_usage( else: self.accumulated_token_usage = self.accumulated_token_usage + new_usage - def add_prompt_composition( - self, composition: PromptComposition, response_id: str = "" - ) -> None: - """Record the per-call prompt composition snapshot for one call.""" - if response_id: - composition = composition.model_copy(update={"response_id": response_id}) - self.prompt_compositions.append(composition) - def merge(self, other: "Metrics") -> None: """Merge 'other' metrics into this one.""" self.accumulated_cost += other.accumulated_cost @@ -290,7 +230,6 @@ def merge(self, other: "Metrics") -> None: self.costs += other.costs self.token_usages += other.token_usages self.response_latencies += other.response_latencies - self.prompt_compositions += other.prompt_compositions # Merge accumulated token usage using the __add__ operator if self.accumulated_token_usage is None: @@ -313,9 +252,6 @@ def get(self) -> dict: latency.model_dump() for latency in self.response_latencies ], "token_usages": [usage.model_dump() for usage in self.token_usages], - "prompt_compositions": [ - composition.model_dump() for composition in self.prompt_compositions - ], } def log(self) -> str: @@ -363,11 +299,6 @@ def diff(self, baseline: "Metrics") -> "Metrics": # Include only token usages that were added after the baseline result.token_usages = self.token_usages[len(baseline.token_usages) :] - # Include only compositions that were added after the baseline - result.prompt_compositions = self.prompt_compositions[ - len(baseline.prompt_compositions) : - ] - # Calculate accumulated token usage difference base_usage = baseline.accumulated_token_usage current_usage = self.accumulated_token_usage diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py index 2e1b700dd3..16c03eb626 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition.py @@ -1,11 +1,16 @@ -"""Client-side estimation of per-call prompt token composition.""" +"""Client-side estimation of per-call prompt token composition. + +Offline analysis utility: it tokenizes logged request payloads after a run +(see ``scripts/prompt_composition_report.py``); nothing here runs on the +LLM call path. +""" from typing import Any from litellm import ChatCompletionToolParam from litellm.utils import token_counter +from pydantic import BaseModel, Field -from openhands.sdk.llm.utils.metrics import PromptComposition from openhands.sdk.logger import get_logger @@ -18,6 +23,46 @@ _TOOLS_PROBE_MESSAGES: list[dict[str, Any]] = [{"role": "user", "content": ""}] +class PromptComposition(BaseModel): + """Per-call decomposition of prompt tokens by component. + + Counts are client-side estimates computed offline from the logged + request payload; the provider-reported ``TokenUsage`` remains + authoritative. Each component is counted independently, so per-message + framing overhead is included in every bucket and the components do not + necessarily sum exactly to the provider-reported ``prompt_tokens``. + Tool schema counts follow litellm's ``token_counter`` serialization + convention for tools, which can differ from the provider's wire-format + tokenization. The composition always uses litellm's generic counter so + numbers stay comparable across models. + """ + + model: str = Field(default="") + system_prompt_tokens: int = Field( + default=0, ge=0, description="Estimated tokens in system messages" + ) + tool_schema_tokens: int = Field( + default=0, ge=0, description="Estimated tokens in tool schemas" + ) + history_tokens: int = Field( + default=0, + ge=0, + description="Estimated tokens in conversation history (all non-system " + "messages except the latest one)", + ) + latest_message_tokens: int = Field( + default=0, + ge=0, + description="Estimated tokens in the latest observation/user message", + ) + is_estimate: bool = Field( + default=True, + description="True when counts are client-side estimates rather than " + "provider-reported usage", + ) + response_id: str = Field(default="") + + def compute_prompt_composition( *, model: str, @@ -25,7 +70,7 @@ def compute_prompt_composition( tools: list[ChatCompletionToolParam] | None = None, custom_tokenizer: Any = None, ) -> PromptComposition | None: - """Estimate prompt tokens per component for a single LLM call. + """Estimate prompt tokens per component for a single logged LLM call. Args: model: Model name used to pick the tokenizer. @@ -38,14 +83,12 @@ def compute_prompt_composition( Returns: A PromptComposition snapshot, or None when counting fails or returns no tokens at all (e.g. ``litellm.disable_token_counter``), since - composition recording is best-effort. - - Recording is opt-in: LLM call sites only invoke this when - ``LLM.enable_prompt_composition`` is True, so the counting cost is only - paid when explicitly enabled. Cost scales linearly with prompt size and - stays small in absolute terms — measured ~31 ms for a ~100K-token prompt - and ~61 ms for ~190K tokens (gpt-4o tokenizer, 19 tools), versus - ~10-20 ms on typical agent-step payloads. + composition analysis is best-effort. + + Cost scales linearly with prompt size and stays small in absolute terms — + measured ~31 ms for a ~100K-token prompt and ~61 ms for ~190K tokens + (gpt-4o tokenizer, 19 tools), versus ~10-20 ms on typical agent-step + payloads. """ def count( @@ -66,15 +109,15 @@ def count( conversation = [m for m in messages if m.get("role") != "system"] try: - tool_tokens = 0 + tool_schema_tokens = 0 if tools: - tool_tokens = count(_TOOLS_PROBE_MESSAGES, tools) - count( + tool_schema_tokens = count(_TOOLS_PROBE_MESSAGES, tools) - count( _TOOLS_PROBE_MESSAGES, None ) composition = PromptComposition( model=model, system_prompt_tokens=count(system_messages, None) if system_messages else 0, - tool_tokens=tool_tokens, + tool_schema_tokens=tool_schema_tokens, history_tokens=count(conversation[:-1], None) if len(conversation) > 1 else 0, @@ -86,7 +129,7 @@ def count( if (messages or tools) and not ( composition.system_prompt_tokens - + composition.tool_tokens + + composition.tool_schema_tokens + composition.history_tokens + composition.latest_message_tokens ): @@ -103,10 +146,10 @@ def responses_payload_to_chat_messages( ) -> list[dict[str, Any]]: """Convert a finalized Responses API payload into chat-format dicts. - Used so prompt composition on the Responses path counts what the provider - actually received (instructions + input items) while sharing the chat - path's counting convention. Raises ValueError on unrecognized item types; - callers treat any failure as "skip the composition record". + Used so offline prompt composition on the Responses path counts what the + provider actually received (instructions + input items) while sharing the + chat path's counting convention. Raises ValueError on unrecognized item + types; callers treat any failure as "skip the composition record". Buckets follow the wire, not the logical prompt: in subscription mode the system prompt is folded into the first user message by the auth-layer diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index b1c1ee63dc..737c7c3b29 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from openhands.sdk.llm.utils.litellm_provider import LLMProvider -from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition +from openhands.sdk.llm.utils.metrics import Metrics from openhands.sdk.llm.utils.openhands_provider import litellm_call_kwargs from openhands.sdk.logger import get_logger @@ -111,12 +111,6 @@ def on_response( usage, response_id, self._req_ctx.get("context_window", 0) ) - # 3a) per-call prompt composition estimate (request-side, recorded even - # when the provider returned no usage) - composition = self._req_ctx.get("prompt_composition") - if isinstance(composition, PromptComposition): - self.metrics.add_prompt_composition(composition, response_id) - # 4) optional logging if self.log_enabled: self.log_llm_call(resp, cost, raw_resp=raw_resp) From 8a72a6aa056978f5ab1b3370e00fed943c750f02 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 31 Aug 2026 14:01:36 -0400 Subject: [PATCH 09/15] test(sdk): keep offline prompt-composition unit tests only Drop the runtime-wiring tests (Metrics plumbing, enable_prompt_composition gating, per-path recording) now that composition counting is an offline utility, and adjust the carried-over tests for the PromptComposition move into prompt_composition.py and the tool_tokens -> tool_schema_tokens rename. Add a converter test for subscription-shaped message items (previously only exercised through the removed runtime path). Co-authored-by: openhands --- tests/sdk/llm/test_prompt_composition.py | 355 ++--------------------- 1 file changed, 28 insertions(+), 327 deletions(-) diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index 487a90d7e4..0132cca460 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -2,23 +2,13 @@ from collections.abc import Sequence from typing import ClassVar -from unittest.mock import AsyncMock, patch +from unittest.mock import patch import pytest -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse -from litellm.types.utils import ( - Choices, - Message as LiteLLMMessage, - ModelResponse, - Usage, -) from litellm.utils import token_counter -from openai.types.responses.response_output_message import ResponseOutputMessage -from openai.types.responses.response_output_text import ResponseOutputText from pydantic import SecretStr from openhands.sdk.llm import LLM, Message, TextContent -from openhands.sdk.llm.utils.metrics import Metrics, PromptComposition from openhands.sdk.llm.utils.prompt_composition import ( compute_prompt_composition, responses_payload_to_chat_messages, @@ -39,30 +29,8 @@ def create(cls, conv_state=None, **params) -> Sequence["_MockTool"]: return [cls(description="A test tool", action_type=_Args)] -def _chat_response(response_id: str = "resp-1") -> ModelResponse: - return ModelResponse( - id=response_id, - choices=[ - Choices( - finish_reason="stop", - index=0, - message=LiteLLMMessage(content="ok", role="assistant"), - ) - ], - created=0, - model="gpt-4o", - object="chat.completion", - usage=Usage(prompt_tokens=100, completion_tokens=5, total_tokens=105), - ) - - -def _make_llm(model: str = "gpt-4o", enable_prompt_composition: bool = True) -> LLM: - return LLM( - model=model, - api_key=SecretStr("test"), - usage_id="test-llm", - enable_prompt_composition=enable_prompt_composition, - ) +def _make_llm(model: str = "gpt-4o") -> LLM: + return LLM(model=model, api_key=SecretStr("test"), usage_id="test-llm") def _sample_messages() -> list[Message]: @@ -89,7 +57,7 @@ def test_compute_prompt_composition_decomposes_into_components(): assert composition is not None assert composition.is_estimate assert composition.system_prompt_tokens > 0 - assert composition.tool_tokens > 0 + assert composition.tool_schema_tokens > 0 assert composition.history_tokens > 0 assert composition.latest_message_tokens > 0 # Buckets are counted independently with per-message framing overhead, @@ -109,7 +77,7 @@ def test_compute_prompt_composition_single_turn_has_no_history(): composition = compute_prompt_composition(model="gpt-4o", messages=formatted) assert composition is not None - assert composition.tool_tokens == 0 + assert composition.tool_schema_tokens == 0 assert composition.history_tokens == 0 assert composition.system_prompt_tokens > 0 assert composition.latest_message_tokens > 0 @@ -140,8 +108,8 @@ def test_compute_prompt_composition_skips_all_zero_records(): assert composition is None -def test_compute_prompt_composition_tool_tokens_matches_controlled_delta(): - """tool_tokens must equal the marginal cost of adding tools to the call.""" +def test_compute_prompt_composition_tool_schema_tokens_matches_controlled_delta(): + """tool_schema_tokens must equal the marginal cost of adding tools.""" formatted = _make_llm().format_messages_for_llm(_sample_messages()) tools = [ t.to_openai_tool(add_security_risk_prediction=True) for t in _MockTool.create() @@ -152,11 +120,11 @@ def test_compute_prompt_composition_tool_tokens_matches_controlled_delta(): ) without_tools = compute_prompt_composition(model="gpt-4o", messages=formatted) assert with_tools is not None and without_tools is not None - assert without_tools.tool_tokens == 0 + assert without_tools.tool_schema_tokens == 0 component_sum = ( with_tools.system_prompt_tokens - + with_tools.tool_tokens + + with_tools.tool_schema_tokens + with_tools.history_tokens + with_tools.latest_message_tokens ) @@ -167,207 +135,13 @@ def test_compute_prompt_composition_tool_tokens_matches_controlled_delta(): ) # Deliberately pins the joint-counting assumption (tools never enter the # message buckets); it is an identity by construction, kept as a tripwire. - assert component_sum - other_components == with_tools.tool_tokens + assert component_sum - other_components == with_tools.tool_schema_tokens # And the estimate tracks the real marginal cost of the tools closely. real_delta = token_counter( model="gpt-4o", messages=formatted, tools=tools ) - token_counter(model="gpt-4o", messages=formatted) - assert abs(with_tools.tool_tokens - real_delta) <= 8 - - -def test_completion_records_prompt_composition(): - llm = _make_llm() - tools = list(_MockTool.create()) - - with patch( - "openhands.sdk.llm.llm.litellm_completion", return_value=_chat_response() - ): - llm.completion(messages=_sample_messages(), tools=tools) - - composition = llm.metrics.latest_prompt_composition - assert composition is not None - assert composition.response_id == "resp-1" - # Agent-step style call: tool schemas are part of the prompt. - assert composition.tool_tokens > 0 - assert composition.system_prompt_tokens > 0 - assert composition.history_tokens > 0 - assert composition.latest_message_tokens > 0 - - -def test_completion_without_tools_records_zero_tool_tokens(): - llm = _make_llm() - - with patch( - "openhands.sdk.llm.llm.litellm_completion", return_value=_chat_response() - ): - llm.completion(messages=_sample_messages()) - - composition = llm.metrics.latest_prompt_composition - assert composition is not None - assert composition.tool_tokens == 0 - - -async def test_acompletion_records_prompt_composition(): - llm = _make_llm() - - with patch( - "openhands.sdk.llm.llm.litellm_acompletion", - new_callable=AsyncMock, - return_value=_chat_response(), - ): - await llm.acompletion(messages=_sample_messages()) - - assert llm.metrics.latest_prompt_composition is not None - - -def test_responses_records_prompt_composition(): - llm = _make_llm("gpt-5-mini") - output = ResponseOutputMessage.model_construct( - id="m1", - type="message", - role="assistant", - status="completed", - content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], - ) - resp = ResponsesAPIResponse( - id="r1", - created_at=0, - output=[output], - parallel_tool_calls=False, - tool_choice="auto", - top_p=None, - tools=[], - usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), - status="completed", - ) - - with patch("openhands.sdk.llm.llm.litellm_responses", return_value=resp): - llm.responses(_sample_messages(), tools=list(_MockTool.create())) - - composition = llm.metrics.latest_prompt_composition - assert composition is not None - assert composition.response_id == "r1" - assert composition.system_prompt_tokens > 0 - assert composition.tool_tokens > 0 - assert composition.latest_message_tokens > 0 - - -def test_metrics_merge_and_diff_include_prompt_compositions(): - baseline = Metrics(model_name="gpt-4o") - baseline.add_prompt_composition( - PromptComposition(system_prompt_tokens=10), response_id="r1" - ) - current = baseline.deep_copy() - current.add_prompt_composition( - PromptComposition(system_prompt_tokens=20), response_id="r2" - ) - - diff = current.diff(baseline) - assert len(diff.prompt_compositions) == 1 - assert diff.prompt_compositions[0].response_id == "r2" - - merged = Metrics(model_name="gpt-4o") - merged.merge(current) - assert len(merged.prompt_compositions) == 2 - assert merged.latest_prompt_composition is not None - assert merged.latest_prompt_composition.response_id == "r2" - - -def test_metrics_loads_payload_without_prompt_compositions(): - metrics = Metrics.model_validate({"model_name": "gpt-4o", "accumulated_cost": 1.0}) - - assert metrics.prompt_compositions == [] - assert metrics.latest_prompt_composition is None - - -def test_mock_tools_does_not_double_count_tool_schemas(): - """With prompt-mocked tools, schemas live in the prompt text: tool_tokens - must be 0 (no double count) while the schemas still inflate the system - bucket.""" - mock_response = ModelResponse( - id="mock-resp", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=LiteLLMMessage( - content=( - "I'll help.\n" - "\n" - "test_value\n" - "" - ), - role="assistant", - ), - ) - ], - created=0, - model="gpt-4o", - object="chat.completion", - usage=Usage(prompt_tokens=100, completion_tokens=5, total_tokens=105), - ) - llm = LLM( - model="gpt-4o", - api_key=SecretStr("test"), - usage_id="test-llm", - native_tool_calling=False, - enable_prompt_composition=True, - ) - - with patch("openhands.sdk.llm.llm.litellm_completion", return_value=mock_response): - llm.completion(messages=_sample_messages(), tools=list(_MockTool.create())) - with_tools = llm.metrics.latest_prompt_composition - llm.completion(messages=_sample_messages()) - without_tools = llm.metrics.latest_prompt_composition - - assert with_tools is not None and without_tools is not None - assert with_tools.tool_tokens == 0 - assert with_tools.system_prompt_tokens > without_tools.system_prompt_tokens - - -def _responses_api_response() -> ResponsesAPIResponse: - output = ResponseOutputMessage.model_construct( - id="m1", - type="message", - role="assistant", - status="completed", - content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], - ) - return ResponsesAPIResponse( - id="r1", - created_at=0, - output=[output], - parallel_tool_calls=False, - tool_choice="auto", - top_p=None, - tools=[], - usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), - status="completed", - ) - - -def test_responses_composition_survives_tool_serialization_failure(): - """A tool that fails chat-format serialization must skip the composition - record, not break the real Responses call.""" - llm = _make_llm("gpt-5-mini") - tool = list(_MockTool.create())[0] - - with ( - patch( - "openhands.sdk.llm.llm.litellm_responses", - return_value=_responses_api_response(), - ), - patch.object( - type(tool), - "to_openai_tool", - side_effect=RuntimeError("cannot serialize"), - ), - ): - response = llm.responses(_sample_messages(), tools=[tool]) - - assert response.message.role == "assistant" - assert llm.metrics.latest_prompt_composition is None + assert abs(with_tools.tool_schema_tokens - real_delta) <= 8 def test_responses_payload_to_chat_messages_covers_sent_item_types(): @@ -414,6 +188,23 @@ def test_responses_payload_to_chat_messages_covers_sent_item_types(): responses_payload_to_chat_messages(None, [{"type": "mystery"}]) +def test_responses_payload_to_chat_messages_subscription_shaped_items(): + """Subscription mode strips the "type" key from message items + (transform_for_subscription); they must still convert.""" + chat = responses_payload_to_chat_messages( + None, + [ + {"role": "user", "content": "system prompt folded in"}, + {"role": "assistant", "content": "working on it"}, + ], + ) + + assert chat == [ + {"role": "user", "content": "system prompt folded in"}, + {"role": "assistant", "content": "working on it"}, + ] + + def test_responses_payload_to_chat_messages_image_parts(): chat = responses_payload_to_chat_messages( None, @@ -445,93 +236,3 @@ def test_responses_payload_to_chat_messages_image_parts(): ], } ] - - -def test_responses_composition_records_subscription_shaped_payload(): - """Subscription mode strips the "type" key from message items - (transform_for_subscription); a composition must still be recorded.""" - llm = LLM( - model="openai/gpt-5.2-codex", - base_url="https://chatgpt.com/backend-api/codex", - api_key=SecretStr("test"), - usage_id="test-llm", - reasoning_effort="high", - enable_prompt_composition=True, - ) - llm._is_subscription = True - - with ( - patch( - "openhands.sdk.llm.llm.litellm_responses", - return_value=_responses_api_response(), - ), - patch.object(llm, "_get_litellm_auth_values", return_value=(None, {})), - ): - llm.responses(_sample_messages(), tools=list(_MockTool.create())) - - composition = llm.metrics.latest_prompt_composition - assert composition is not None - assert composition.history_tokens > 0 - assert composition.latest_message_tokens > 0 - - -async def test_aresponses_records_prompt_composition(): - llm = _make_llm("gpt-5-mini") - - with patch( - "openhands.sdk.llm.llm.litellm_aresponses", - new_callable=AsyncMock, - return_value=_responses_api_response(), - ): - await llm.aresponses(_sample_messages(), tools=list(_MockTool.create())) - - composition = llm.metrics.latest_prompt_composition - assert composition is not None - assert composition.response_id == "r1" - assert composition.tool_tokens > 0 - - -def test_completion_default_off_records_no_composition(): - """With the flag at its default (off), no composition record is appended.""" - llm = LLM(model="gpt-4o", api_key=SecretStr("test"), usage_id="test-llm") - assert llm.enable_prompt_composition is False - - with patch( - "openhands.sdk.llm.llm.litellm_completion", return_value=_chat_response() - ): - llm.completion(messages=_sample_messages(), tools=list(_MockTool.create())) - - assert llm.metrics.prompt_compositions == [] - assert llm.metrics.latest_prompt_composition is None - - -def test_completion_off_never_calls_token_counter(): - """The off path must not pay any tokenization pass at all.""" - llm = _make_llm(enable_prompt_composition=False) - - with ( - patch( - "openhands.sdk.llm.llm.litellm_completion", - return_value=_chat_response(), - ), - patch( - "openhands.sdk.llm.utils.prompt_composition.token_counter" - ) as counter_spy, - ): - llm.completion(messages=_sample_messages(), tools=list(_MockTool.create())) - - counter_spy.assert_not_called() - assert llm.metrics.prompt_compositions == [] - - -def test_responses_default_off_records_no_composition(): - llm = _make_llm("gpt-5-mini", enable_prompt_composition=False) - - with patch( - "openhands.sdk.llm.llm.litellm_responses", - return_value=_responses_api_response(), - ): - llm.responses(_sample_messages(), tools=list(_MockTool.create())) - - assert llm.metrics.prompt_compositions == [] - assert llm.metrics.latest_prompt_composition is None From 439256514f5ca01353099bced0ec40b912eec654 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 31 Aug 2026 14:06:04 -0400 Subject: [PATCH 10/15] feat(sdk): add offline prompt-composition report script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/prompt_composition_report.py, a stdlib-only CLI that rebuilds per-call prompt token composition from LLM(log_completions=True) logs — no runtime changes required. Ingestion and report logic live in openhands.sdk.llm.utils.prompt_composition_report so tests can import it; the script is a thin argparse wrapper in the spirit of scripts/completion_logs_viewer.py. Per log file the report converts Responses-path payloads via responses_payload_to_chat_messages, uses chat messages directly, and passes tools=None when raw_messages is present (mock-tools path renders schemas into the prompt text — no double count). Logs serialize tools as ToolDefinition dumps without parameter schemas, so when logged tools are not OpenAI-format the row is marked tool_schema_counted=false instead of reporting a silently wrong bucket. Output is per-call JSONL rows (seq/usage/composition/latency_s), a summary JSON (per-bucket averages, est/provider median ratio over fully-counted calls), and a text chart: per-call stacked bars plus a trend table over seq. Co-authored-by: openhands --- .../llm/utils/prompt_composition_report.py | 332 ++++++++++++++++++ scripts/prompt_composition_report.py | 66 ++++ 2 files changed, 398 insertions(+) create mode 100644 openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py create mode 100644 scripts/prompt_composition_report.py diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py new file mode 100644 index 0000000000..9b3ad1f2aa --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py @@ -0,0 +1,332 @@ +"""Offline prompt-composition report over OpenHands completion logs. + +Ingests the JSON logs written by ``LLM(log_completions=True)`` (see +``scripts/completion_logs_viewer.py`` for the directory layout) and rebuilds +the per-call prompt token composition from the logged request payload, joined +with the provider-reported usage. Nothing here runs on the LLM call path. +""" + +import json +import statistics +from pathlib import Path +from typing import Any + +from litellm import ChatCompletionToolParam + +from openhands.sdk.llm.utils.prompt_composition import ( + compute_prompt_composition, + responses_payload_to_chat_messages, +) +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) + +BUCKETS = ( + "system_prompt_tokens", + "tool_schema_tokens", + "history_tokens", + "latest_message_tokens", +) + +_CHART_WIDTH = 50 +_BUCKET_CHARS = { + "system_prompt_tokens": "S", + "tool_schema_tokens": "T", + "history_tokens": "H", + "latest_message_tokens": "L", +} + + +def iter_log_files(root: Path) -> list[Path]: + """List log files under a run folder, or under a root of run folders.""" + root = Path(root) + if not root.is_dir(): + return [] + direct = sorted(root.glob("*.json")) + if direct: + return direct + return sorted(root.glob("*/*.json")) + + +def _is_openai_tool_schema(tool: Any) -> bool: + return ( + isinstance(tool, dict) + and tool.get("type") == "function" + and isinstance(tool.get("function"), dict) + ) + + +def _log_model(data: dict[str, Any], source: str) -> str: + response = data.get("response") + if isinstance(response, dict) and isinstance(response.get("model"), str): + return response["model"] + # Log filenames are "{model with '/'->'__'}-{timestamp}-{uuid4}.json". + stem = Path(source).stem + parts = stem.rsplit("-", 2) + if len(parts) == 3 and parts[0]: + return parts[0].replace("__", "/") + return "" + + +def call_record_from_log(data: Any, source: str) -> dict[str, Any] | None: + """Build one report row from a parsed log payload, or None to skip it. + + Returns None for anything that is not a countable LLM call log: unreadable + shapes, error logs, and payloads whose composition cannot be computed. + """ + if not isinstance(data, dict) or "error" in data: + return None + + if data.get("llm_path") == "responses": + input_items = data.get("input") + if not isinstance(input_items, list) or not input_items: + return None + try: + messages = responses_payload_to_chat_messages( + data.get("instructions"), input_items + ) + except ValueError: + return None + schemas_in_prompt = False + else: + messages = data.get("messages") + if not isinstance(messages, list) or not messages: + return None + # Non-native/mock-tools path: schemas are rendered into the prompt + # text, so counting them as tools too would double-count. + schemas_in_prompt = "raw_messages" in data + + tools: list[ChatCompletionToolParam] | None = None + tool_schema_counted = True + logged_tools = data.get("tools") + if not schemas_in_prompt and isinstance(logged_tools, list) and logged_tools: + if all(_is_openai_tool_schema(t) for t in logged_tools): + tools = logged_tools + else: + # Completion logs serialize tools as ToolDefinition dumps + # (name/description only, no parameter schemas), so the tool + # bucket cannot be reconstructed from them; mark the row + # rather than reporting a silently wrong count. + tool_schema_counted = False + + composition = compute_prompt_composition( + model=_log_model(data, source), messages=messages, tools=tools + ) + if composition is None: + return None + + response = data.get("response") + response_id = "" + if isinstance(response, dict) and isinstance(response.get("id"), str): + response_id = response["id"] + composition = composition.model_copy(update={"response_id": response_id}) + + usage_summary = data.get("usage_summary") + if not isinstance(usage_summary, dict): + usage_summary = {} + usage = { + "model": composition.model, + "prompt_tokens": int(usage_summary.get("prompt_tokens") or 0), + "completion_tokens": int(usage_summary.get("completion_tokens") or 0), + "cache_read_tokens": int(usage_summary.get("cache_read_tokens") or 0), + "reasoning_tokens": int(usage_summary.get("reasoning_tokens") or 0), + "context_window": int(data.get("context_window") or 0), + "response_id": response_id, + } + + return { + "timestamp": float(data.get("timestamp") or 0.0), + "source": source, + "usage": usage, + "composition": composition.model_dump(), + "latency_s": data.get("latency_sec"), + "tool_schema_counted": tool_schema_counted, + } + + +def estimated_total(composition: dict[str, Any]) -> int: + return int(sum(int(composition.get(bucket) or 0) for bucket in BUCKETS)) + + +def build_report(root: Path) -> dict[str, Any]: + """Ingest every log under ``root`` into rows plus a summary.""" + rows: list[dict[str, Any]] = [] + skipped: list[dict[str, str]] = [] + root = Path(root) + for path in iter_log_files(root): + source = str(path.relative_to(root)) if path.is_relative_to(root) else path.name + try: + data = json.loads(path.read_text()) + except (OSError, ValueError): + skipped.append({"source": source, "reason": "unreadable or invalid JSON"}) + continue + record = call_record_from_log(data, source) + if record is None: + skipped.append({"source": source, "reason": "not a countable call log"}) + continue + rows.append(record) + + rows.sort(key=lambda r: (r["timestamp"], r["source"])) + sequenced = [{"seq": seq, **row} for seq, row in enumerate(rows)] + return { + "root": str(root), + "rows": sequenced, + "skipped": skipped, + "summary": _summarize(root, sequenced, skipped), + } + + +def _est_provider_ratios(rows: list[dict[str, Any]]) -> list[float]: + ratios = [] + for row in rows: + if not row["tool_schema_counted"]: + # The estimate is missing a bucket, so the ratio would understate. + continue + provider = int(row["usage"].get("prompt_tokens") or 0) + if provider > 0: + ratios.append(estimated_total(row["composition"]) / provider) + return ratios + + +def _summarize( + root: Path, rows: list[dict[str, Any]], skipped: list[dict[str, str]] +) -> dict[str, Any]: + averages = {} + for bucket in BUCKETS: + values = [int(row["composition"].get(bucket) or 0) for row in rows] + averages[bucket] = round(statistics.fmean(values), 1) if values else 0.0 + est_totals = [estimated_total(row["composition"]) for row in rows] + provider_prompt = [int(row["usage"].get("prompt_tokens") or 0) for row in rows] + ratios = _est_provider_ratios(rows) + return { + "root": str(root), + "calls": len(rows), + "skipped_files": len(skipped), + "calls_with_uncountable_tool_schemas": sum( + 1 for row in rows if not row["tool_schema_counted"] + ), + "avg": averages, + "avg_est_total": round(statistics.fmean(est_totals), 1) if est_totals else 0.0, + "avg_provider_prompt_tokens": ( + round(statistics.fmean(provider_prompt), 1) if provider_prompt else 0.0 + ), + "est_provider_median_ratio": ( + round(statistics.median(ratios), 3) if ratios else None + ), + "est_provider_ratio_calls": len(ratios), + } + + +def _chart_bar(composition: dict[str, Any], tokens_per_char: float) -> str: + parts = [] + for bucket in BUCKETS: + chars = round(int(composition.get(bucket) or 0) / tokens_per_char) + parts.append(_BUCKET_CHARS[bucket] * chars) + return "".join(parts).ljust(_CHART_WIDTH) + + +def render_text_report(report: dict[str, Any]) -> str: + """Render the per-call stacked bars, the trend table, and the summary.""" + rows: list[dict[str, Any]] = report["rows"] + summary: dict[str, Any] = report["summary"] + lines = [ + "Prompt composition per call " + "(S=system T=tool_schema H=history L=latest, " + f"bar width ~{_CHART_WIDTH} chars)", + ] + if not rows: + lines.append(" no countable calls") + else: + tokens_per_char = max( + max(estimated_total(row["composition"]) for row in rows) / _CHART_WIDTH, + 1.0, + ) + lines.append( + f" {'seq':>4} {'composition':<{_CHART_WIDTH}} " + f"{'est_total':>9} {'provider':>9}" + ) + for row in rows: + est = estimated_total(row["composition"]) + provider = int(row["usage"].get("prompt_tokens") or 0) + bar = _chart_bar(row["composition"], tokens_per_char) + marker = "" if row["tool_schema_counted"] else " *" + lines.append(f" {row['seq']:>4} {bar} {est:>9} {provider:>9}{marker}") + if any(not row["tool_schema_counted"] for row in rows): + lines.append( + " * tool schemas not reconstructable from this log " + "(tool_schema bucket undercounted)" + ) + + lines += [ + "", + "Trend over seq:", + f" {'seq':>4} | {'system':>8} | {'tool_schema':>11} | {'history':>8} | " + f"{'latest':>8} | {'est_total':>9} | {'provider':>8} | {'ratio':>6} | " + f"{'latency_s':>9}", + ] + for row in rows: + composition = row["composition"] + est = estimated_total(composition) + provider = int(row["usage"].get("prompt_tokens") or 0) + if provider > 0 and row["tool_schema_counted"]: + ratio = f"{est / provider:.2f}" + else: + ratio = "-" + latency = row["latency_s"] + latency_s = f"{latency:.2f}" if isinstance(latency, int | float) else "-" + lines.append( + f" {row['seq']:>4} | {composition['system_prompt_tokens']:>8} | " + f"{composition['tool_schema_tokens']:>11} | " + f"{composition['history_tokens']:>8} | " + f"{composition['latest_message_tokens']:>8} | {est:>9} | " + f"{provider:>8} | {ratio:>6} | {latency_s:>9}" + ) + + avg = summary["avg"] + ratio = summary["est_provider_median_ratio"] + lines += [ + "", + "Summary:", + f" calls: {summary['calls']} " + f"(skipped files: {summary['skipped_files']}, " + f"uncountable tool schemas: " + f"{summary['calls_with_uncountable_tool_schemas']})", + f" avg system: {avg['system_prompt_tokens']}", + f" avg tool_schema: {avg['tool_schema_tokens']}", + f" avg history: {avg['history_tokens']}", + f" avg latest: {avg['latest_message_tokens']}", + f" avg est total: {summary['avg_est_total']} " + f"avg provider prompt: {summary['avg_provider_prompt_tokens']}", + f" est/provider median ratio: " + f"{ratio if ratio is not None else 'n/a'} " + f"(over {summary['est_provider_ratio_calls']} calls)", + ] + return "\n".join(lines) + + +def write_report(report: dict[str, Any], out_dir: Path) -> tuple[Path, Path]: + """Write the per-call JSONL rows and the summary JSON into ``out_dir``.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + calls_path = out_dir / "calls.jsonl" + with calls_path.open("w", encoding="utf-8") as f: + for row in report["rows"]: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + summary_path = out_dir / "summary.json" + summary_path.write_text( + json.dumps(report["summary"], indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return calls_path, summary_path + + +__all__ = [ + "BUCKETS", + "build_report", + "call_record_from_log", + "estimated_total", + "iter_log_files", + "render_text_report", + "write_report", +] diff --git a/scripts/prompt_composition_report.py b/scripts/prompt_composition_report.py new file mode 100644 index 0000000000..d423df67e8 --- /dev/null +++ b/scripts/prompt_composition_report.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Offline per-call prompt token composition report from completion logs. + +Usage: + uv run python scripts/prompt_composition_report.py --root LOGS [--out DIR] + [--no-chart] + +``--root`` follows the completion-log layout documented in +``scripts/completion_logs_viewer.py``: either a single run folder of ``*.json`` +logs, or a root directory containing such run folders. The logs are the files +written by ``LLM(log_completions=True)``. + +For each log the script rebuilds the prompt token composition (system prompt, +tool schemas, history, latest message) from the logged request payload, joins +it with the provider-reported usage, prints a text visualization, and +optionally writes the per-call rows (``calls.jsonl``) and aggregate +``summary.json`` into ``--out``. +""" + +import argparse +import sys +from pathlib import Path + +from openhands.sdk.llm.utils.prompt_composition_report import ( + build_report, + render_text_report, + write_report, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + required=True, + type=Path, + help="Run folder of *.json completion logs, or a root of run folders", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="Directory to write calls.jsonl and summary.json into", + ) + parser.add_argument( + "--no-chart", + action="store_true", + help="Skip the text visualization on stdout", + ) + args = parser.parse_args() + + report = build_report(args.root) + if not report["rows"]: + print(f"No countable LLM call logs found under {args.root}", file=sys.stderr) + return 1 + + if not args.no_chart: + print(render_text_report(report)) + if args.out: + calls_path, summary_path = write_report(report, args.out) + print(f"\nWrote {calls_path} and {summary_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 8fa60d26f6a1bcede20e6917f390cdc99adc8335 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 31 Aug 2026 14:07:57 -0400 Subject: [PATCH 11/15] test(sdk): cover prompt-composition report ingestion Add focused ingestion tests for the offline report: synthetic chat logs (rows sequenced by call timestamp, usage joined, summary averages and est/provider ratio), a synthetic Responses-path log (instructions + input items converted before counting), a mock-tools log (raw_messages present, schemas counted once via the prompt text), a native-path log with ToolDefinition-dump tools (tool bucket marked uncountable and excluded from the ratio), garbage files skipped, and the calls.jsonl/summary.json writer. Co-authored-by: openhands --- tests/sdk/llm/test_prompt_composition.py | 228 +++++++++++++++++++++++ 1 file changed, 228 insertions(+) diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index 0132cca460..64792cbf74 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -1,5 +1,6 @@ """Tests for per-call prompt token composition estimates.""" +import json from collections.abc import Sequence from typing import ClassVar from unittest.mock import patch @@ -13,6 +14,10 @@ compute_prompt_composition, responses_payload_to_chat_messages, ) +from openhands.sdk.llm.utils.prompt_composition_report import ( + build_report, + write_report, +) from openhands.sdk.tool.schema import Action from openhands.sdk.tool.tool import ToolDefinition @@ -236,3 +241,226 @@ def test_responses_payload_to_chat_messages_image_parts(): ], } ] + + +def _write_log(run_dir, name: str, payload: dict): + path = run_dir / name + path.write_text(json.dumps(payload)) + return path + + +_OPENAI_TOOL = { + "type": "function", + "function": { + "name": "test_tool", + "description": "A test tool", + "parameters": { + "type": "object", + "properties": {"param": {"type": "string"}}, + "required": ["param"], + }, + }, +} + +_TOOL_DEFINITION_DUMP = { + "description": "A test tool", + "action_type": "_Args", + "kind": "_MockTool", + "title": "test_tool", +} + + +def _chat_log( + response_id: str, + timestamp: float, + history: list[dict] | None = None, + tools: list | None = None, + prompt_tokens: int = 900, +) -> dict: + messages = [{"role": "system", "content": "You are an agent. " * 50}] + messages.extend(history or []) + messages.append({"role": "user", "content": "latest observation " * 10}) + return { + "context_window": 128000, + "messages": messages, + "tools": tools if tools is not None else [_OPENAI_TOOL], + "kwargs": {}, + "response": {"id": response_id, "model": "gpt-4o"}, + "usage_summary": { + "prompt_tokens": prompt_tokens, + "completion_tokens": 12, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + }, + "cost": 0.001, + "timestamp": timestamp, + "latency_sec": 1.5, + } + + +def test_report_ingests_chat_logs_in_timestamp_order(tmp_path): + run = tmp_path / "run1" + run.mkdir() + _write_log(run, "gpt-4o-002.000-ab12.json", _chat_log("r2", timestamp=2.0)) + _write_log(run, "gpt-4o-001.000-cd34.json", _chat_log("r1", timestamp=1.0)) + + report = build_report(run) + + assert report["skipped"] == [] + assert [row["seq"] for row in report["rows"]] == [0, 1] + first, _second = report["rows"] + # seq follows the call timestamp, not the filename order. + assert first["composition"]["response_id"] == "r1" + assert first["tool_schema_counted"] is True + assert first["composition"]["system_prompt_tokens"] > 0 + assert first["composition"]["tool_schema_tokens"] > 0 + assert first["composition"]["latest_message_tokens"] > 0 + assert first["usage"]["prompt_tokens"] == 900 + assert first["latency_s"] == 1.5 + + summary = report["summary"] + assert summary["calls"] == 2 + assert summary["calls_with_uncountable_tool_schemas"] == 0 + assert summary["avg"]["system_prompt_tokens"] > 0 + assert summary["est_provider_ratio_calls"] == 2 + assert summary["est_provider_median_ratio"] is not None + + +def test_report_ingests_responses_log(tmp_path): + run = tmp_path / "run1" + run.mkdir() + _write_log( + run, + "gpt-5-mini-001.000-ab12.json", + { + "context_window": 200000, + "llm_path": "responses", + "instructions": "You are an agent. " * 50, + "input": [ + {"type": "message", "role": "user", "content": "do the task " * 10}, + { + "type": "function_call", + "id": "fc1", + "call_id": "c1", + "name": "test_tool", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": "done " * 30, + }, + ], + "tools": [_OPENAI_TOOL], + "kwargs": {}, + "response": {"id": "resp-1", "model": "gpt-5-mini"}, + "usage_summary": { + "prompt_tokens": 800, + "completion_tokens": 8, + "reasoning_tokens": 3, + "cache_read_tokens": 0, + }, + "cost": 0.002, + "timestamp": 1.0, + "latency_sec": 2.1, + }, + ) + + report = build_report(run) + + assert report["skipped"] == [] + (row,) = report["rows"] + # instructions become the system bucket; input items fill history/latest. + assert row["composition"]["system_prompt_tokens"] > 0 + assert row["composition"]["tool_schema_tokens"] > 0 + assert row["composition"]["history_tokens"] > 0 + assert row["tool_schema_counted"] is True + assert row["usage"]["response_id"] == "resp-1" + + +def test_report_mock_tools_log_counts_schemas_once(tmp_path): + """With raw_messages present, schemas live in the prompt text: the tool + bucket stays 0 (no double count) while they inflate the system bucket.""" + run = tmp_path / "run1" + run.mkdir() + raw_messages = [ + {"role": "system", "content": "You are an agent. " * 50}, + {"role": "user", "content": "do the task"}, + ] + mocked_messages = [ + { + "role": "system", + "content": "You are an agent. " * 50 + + "TOOLS:\n" + + json.dumps(_OPENAI_TOOL) * 3, + }, + {"role": "user", "content": "do the task"}, + ] + payload = _chat_log("mock-1", timestamp=1.0, tools=[_TOOL_DEFINITION_DUMP]) + payload["messages"] = mocked_messages + payload["raw_messages"] = raw_messages + _write_log(run, "gpt-4o-001.000-ab12.json", payload) + + report = build_report(run) + + assert report["skipped"] == [] + (row,) = report["rows"] + assert row["tool_schema_counted"] is True + assert row["composition"]["tool_schema_tokens"] == 0 + raw_composition = compute_prompt_composition(model="gpt-4o", messages=raw_messages) + assert raw_composition is not None + assert ( + row["composition"]["system_prompt_tokens"] + > raw_composition.system_prompt_tokens + ) + + +def test_report_marks_uncountable_tool_schemas(tmp_path): + """Logs serialize tools as ToolDefinition dumps (no parameter schemas), so + the tool bucket is marked uncountable rather than silently wrong.""" + run = tmp_path / "run1" + run.mkdir() + payload = _chat_log("r1", timestamp=1.0, tools=[_TOOL_DEFINITION_DUMP]) + _write_log(run, "gpt-4o-001.000-ab12.json", payload) + + report = build_report(run) + + assert report["skipped"] == [] + (row,) = report["rows"] + assert row["tool_schema_counted"] is False + assert row["composition"]["tool_schema_tokens"] == 0 + assert row["composition"]["system_prompt_tokens"] > 0 + summary = report["summary"] + assert summary["calls_with_uncountable_tool_schemas"] == 1 + # A row missing a bucket is excluded from the est/provider ratio. + assert summary["est_provider_ratio_calls"] == 0 + assert summary["est_provider_median_ratio"] is None + + +def test_report_skips_garbage_files(tmp_path): + run = tmp_path / "run1" + run.mkdir() + (run / "broken.json").write_text("{not json") + _write_log(run, "not-a-log.json", {"hello": "world"}) + _write_log(run, "error-log.json", {**_chat_log("e1", 1.0), "error": {}}) + + report = build_report(run) + + assert report["rows"] == [] + assert len(report["skipped"]) == 3 + assert report["summary"]["calls"] == 0 + + +def test_write_report_emits_calls_jsonl_and_summary(tmp_path): + run = tmp_path / "run1" + run.mkdir() + _write_log(run, "gpt-4o-001.000-ab12.json", _chat_log("r1", timestamp=1.0)) + report = build_report(run) + + calls_path, summary_path = write_report(report, tmp_path / "out") + + (row,) = [json.loads(line) for line in calls_path.read_text().splitlines()] + assert row["seq"] == 0 + assert set(row) >= {"seq", "usage", "composition", "latency_s"} + summary = json.loads(summary_path.read_text()) + assert summary["calls"] == 1 From 943c2f00c6043720758f3d37a3177384297e3508 Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 31 Aug 2026 14:15:12 -0400 Subject: [PATCH 12/15] fix(sdk): log finalized tool schemas in completion logs Completion logs serialized telemetry_ctx["tools"] as ToolDefinition dumps (name/description only, no parameter schemas), so offline analysis could not reconstruct the tool-schema portion of the request. Log the finalized schemas actually sent instead: cc_tools (OpenAI chat format) on the chat path and resp_tools (Responses ToolParam) on the Responses path. Telemetry.log_llm_call still strips the duplicate kwargs["tools"], so the top-level schemas are the single logged copy. No other runtime behavior changes. Co-authored-by: openhands --- openhands-sdk/openhands/sdk/llm/llm.py | 7 +- .../test_llm_log_completions_integration.py | 112 ++++++++++++++++++ 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 85d39409a0..32e563a896 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -1318,7 +1318,9 @@ def _finalize_completion_params( telemetry_ctx.update( { "messages": formatted_messages[:], # already simple dicts - "tools": tools, + # OpenAI-format schemas as sent, so offline analysis can + # reconstruct the request without the Tool classes. + "tools": cc_tools or None, "kwargs": {k: v for k, v in call_kwargs.items()}, } ) @@ -1456,7 +1458,8 @@ def _finalize_responses_params( "llm_path": "responses", "instructions": instructions, "input": input_items[:], - "tools": tools, + # Responses-format schemas as sent (see above). + "tools": resp_tools, "kwargs": {k: v for k, v in call_kwargs.items()}, } ) diff --git a/tests/sdk/llm/test_llm_log_completions_integration.py b/tests/sdk/llm/test_llm_log_completions_integration.py index 7a68ce2b46..b558e712fc 100644 --- a/tests/sdk/llm/test_llm_log_completions_integration.py +++ b/tests/sdk/llm/test_llm_log_completions_integration.py @@ -8,11 +8,18 @@ import os import tempfile import warnings +from collections.abc import Sequence +from typing import ClassVar from unittest.mock import patch +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from openai.types.responses.response_output_message import ResponseOutputMessage +from openai.types.responses.response_output_text import ResponseOutputText from pydantic import SecretStr from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.tool.schema import Action +from openhands.sdk.tool.tool import ToolDefinition # Import common test utilities from tests.conftest import create_mock_litellm_response @@ -187,3 +194,108 @@ def test_llm_log_completions_with_tool_calls(): assert "response" in log_data assert log_data["response"]["choices"][0]["message"]["tool_calls"] is not None + + +class _Args(Action): + param: str + + +class _MockTool(ToolDefinition[_Args, None]): + name: ClassVar[str] = "test_tool" + + @classmethod + def create(cls, conv_state=None, **params) -> Sequence["_MockTool"]: + return [cls(description="A test tool", action_type=_Args)] + + +def _read_single_log(temp_dir: str) -> dict: + log_files = os.listdir(temp_dir) + assert len(log_files) == 1, f"Expected 1 log file, got {len(log_files)}" + with open(os.path.join(temp_dir, log_files[0])) as f: + return json.loads(f.read()) + + +def test_log_completions_logs_openai_format_tool_schemas(): + """The chat-path log must carry the OpenAI-format tool schemas actually + sent (full parameter properties), so offline analysis can reconstruct the + request without the Tool classes.""" + with tempfile.TemporaryDirectory() as temp_dir: + llm = LLM( + model="gpt-4o", + api_key=SecretStr("test-key"), + usage_id="test-log-tools-llm", + log_completions=True, + log_completions_folder=temp_dir, + num_retries=0, + ) + mock_response = create_mock_litellm_response( + content="ok", + response_id="chat-tools-id", + model="gpt-4o", + ) + + with patch( + "openhands.sdk.llm.llm.litellm_completion", return_value=mock_response + ): + llm.completion( + [Message(role="user", content=[TextContent(text="Call a tool")])], + tools=list(_MockTool.create()), + ) + + log_data = _read_single_log(temp_dir) + + (log_tool,) = log_data["tools"] + assert log_tool["type"] == "function" + assert log_tool["function"]["name"] == "test_tool" + assert "param" in log_tool["function"]["parameters"]["properties"] + # The duplicate copy under kwargs is still stripped by Telemetry. + assert "tools" not in log_data["kwargs"] + + +def test_log_completions_logs_responses_format_tool_schemas(): + """The Responses-path log must carry the finalized Responses ToolParam + schemas (parameter properties at the top level).""" + output = ResponseOutputMessage.model_construct( + id="m1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="ok", annotations=[])], + ) + mock_response = ResponsesAPIResponse( + id="resp-tools-id", + created_at=0, + output=[output], + parallel_tool_calls=False, + tool_choice="auto", + top_p=None, + tools=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), + status="completed", + ) + + with tempfile.TemporaryDirectory() as temp_dir: + llm = LLM( + model="gpt-5-mini", + api_key=SecretStr("test-key"), + usage_id="test-log-resp-tools-llm", + log_completions=True, + log_completions_folder=temp_dir, + num_retries=0, + ) + + with patch( + "openhands.sdk.llm.llm.litellm_responses", return_value=mock_response + ): + llm.responses( + [Message(role="user", content=[TextContent(text="Call a tool")])], + tools=list(_MockTool.create()), + ) + + log_data = _read_single_log(temp_dir) + + assert log_data["llm_path"] == "responses" + (log_tool,) = log_data["tools"] + assert log_tool["type"] == "function" + assert log_tool["name"] == "test_tool" + assert "param" in log_tool["parameters"]["properties"] From 82a45b0de3b73a6f924afb19ea76b7bab7cf6aac Mon Sep 17 00:00:00 2001 From: george larson Date: Mon, 31 Aug 2026 14:16:34 -0400 Subject: [PATCH 13/15] feat(sdk): count finalized tool schemas from new-shape logs Completion logs now carry the finalized tool schemas (OpenAI chat format on the chat path, Responses ToolParam on the Responses path), so the report normalizes both shapes to chat format for token counting and the tool_schema bucket populates on fresh logs. Logs written before the logging fix still serialize tools as ToolDefinition dumps; those remain detected and flagged tool_schema_counted=false rather than miscounted, so the script stays safe over historical log directories. Co-authored-by: openhands --- .../llm/utils/prompt_composition_report.py | 44 ++++++++++++++----- tests/sdk/llm/test_prompt_composition.py | 15 ++++++- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py index 9b3ad1f2aa..66692865a3 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py +++ b/openhands-sdk/openhands/sdk/llm/utils/prompt_composition_report.py @@ -49,12 +49,32 @@ def iter_log_files(root: Path) -> list[Path]: return sorted(root.glob("*/*.json")) -def _is_openai_tool_schema(tool: Any) -> bool: - return ( - isinstance(tool, dict) - and tool.get("type") == "function" - and isinstance(tool.get("function"), dict) - ) +def _normalize_logged_tools(logged_tools: list[Any]) -> list[Any] | None: + """Normalize logged tool schemas to OpenAI chat format for token counting. + + Completion logs carry the finalized schemas as sent: OpenAI chat format + (``{"type": "function", "function": {...}}``) on the chat path, Responses + ToolParam (schema fields at the top level) on the Responses path. Returns + None when the entries are not recognizable tool schemas — logs written + before the schemas were logged in finalized form serialize tools as + ToolDefinition dumps (name/description only, no parameter schemas). + """ + normalized: list[Any] = [] + for tool in logged_tools: + if not isinstance(tool, dict) or tool.get("type") != "function": + return None + if isinstance(tool.get("function"), dict): + normalized.append(tool) + elif "name" in tool: + normalized.append( + { + "type": "function", + "function": {k: v for k, v in tool.items() if k != "type"}, + } + ) + else: + return None + return normalized def _log_model(data: dict[str, Any], source: str) -> str: @@ -101,13 +121,13 @@ def call_record_from_log(data: Any, source: str) -> dict[str, Any] | None: tool_schema_counted = True logged_tools = data.get("tools") if not schemas_in_prompt and isinstance(logged_tools, list) and logged_tools: - if all(_is_openai_tool_schema(t) for t in logged_tools): - tools = logged_tools + normalized_tools = _normalize_logged_tools(logged_tools) + if normalized_tools is not None: + tools = normalized_tools else: - # Completion logs serialize tools as ToolDefinition dumps - # (name/description only, no parameter schemas), so the tool - # bucket cannot be reconstructed from them; mark the row - # rather than reporting a silently wrong count. + # ToolDefinition dumps carry no parameter schemas, so the tool + # bucket cannot be reconstructed from them; mark the row rather + # than reporting a silently wrong count. tool_schema_counted = False composition = compute_prompt_composition( diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index 64792cbf74..00eab50d0e 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -269,6 +269,19 @@ def _write_log(run_dir, name: str, payload: dict): "title": "test_tool", } +# Responses ToolParam as logged on the Responses path: schema fields at the +# top level, no nested "function" key. +_RESPONSES_TOOL = { + "type": "function", + "name": "test_tool", + "description": "A test tool", + "parameters": { + "type": "object", + "properties": {"param": {"type": "string"}}, + "required": ["param"], + }, +} + def _chat_log( response_id: str, @@ -351,7 +364,7 @@ def test_report_ingests_responses_log(tmp_path): "output": "done " * 30, }, ], - "tools": [_OPENAI_TOOL], + "tools": [_RESPONSES_TOOL], "kwargs": {}, "response": {"id": "resp-1", "model": "gpt-5-mini"}, "usage_summary": { From d74e6d9d765cc7ae9da26b83e7dc6573750f6b9f Mon Sep 17 00:00:00 2001 From: george larson Date: Tue, 1 Sep 2026 05:42:37 -0400 Subject: [PATCH 14/15] test(sdk): deduplicate helper Action classes across composition tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both new test modules defined a private _Args(Action) and _MockTool. When xdist placed them on the same worker, pydantic's duplicate class definition guard failed every Event/ToolDefinition model_validate_json in that worker — surfacing as unrelated golden-transcript and serialization failures only in full-suite CI runs, never single-file. Rename each module's helpers to unique names (_CompositionArgs/_CompositionMockTool, _LogCompletionsArgs/_LogCompletionsMockTool). Co-authored-by: openhands --- .../test_llm_log_completions_integration.py | 12 ++++++------ tests/sdk/llm/test_prompt_composition.py | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/sdk/llm/test_llm_log_completions_integration.py b/tests/sdk/llm/test_llm_log_completions_integration.py index b558e712fc..d1d6bee065 100644 --- a/tests/sdk/llm/test_llm_log_completions_integration.py +++ b/tests/sdk/llm/test_llm_log_completions_integration.py @@ -196,16 +196,16 @@ def test_llm_log_completions_with_tool_calls(): assert log_data["response"]["choices"][0]["message"]["tool_calls"] is not None -class _Args(Action): +class _LogCompletionsArgs(Action): param: str -class _MockTool(ToolDefinition[_Args, None]): +class _LogCompletionsMockTool(ToolDefinition[_LogCompletionsArgs, None]): name: ClassVar[str] = "test_tool" @classmethod - def create(cls, conv_state=None, **params) -> Sequence["_MockTool"]: - return [cls(description="A test tool", action_type=_Args)] + def create(cls, conv_state=None, **params) -> Sequence["_LogCompletionsMockTool"]: + return [cls(description="A test tool", action_type=_LogCompletionsArgs)] def _read_single_log(temp_dir: str) -> dict: @@ -239,7 +239,7 @@ def test_log_completions_logs_openai_format_tool_schemas(): ): llm.completion( [Message(role="user", content=[TextContent(text="Call a tool")])], - tools=list(_MockTool.create()), + tools=list(_LogCompletionsMockTool.create()), ) log_data = _read_single_log(temp_dir) @@ -289,7 +289,7 @@ def test_log_completions_logs_responses_format_tool_schemas(): ): llm.responses( [Message(role="user", content=[TextContent(text="Call a tool")])], - tools=list(_MockTool.create()), + tools=list(_LogCompletionsMockTool.create()), ) log_data = _read_single_log(temp_dir) diff --git a/tests/sdk/llm/test_prompt_composition.py b/tests/sdk/llm/test_prompt_composition.py index 00eab50d0e..6f4a63c80a 100644 --- a/tests/sdk/llm/test_prompt_composition.py +++ b/tests/sdk/llm/test_prompt_composition.py @@ -22,16 +22,16 @@ from openhands.sdk.tool.tool import ToolDefinition -class _Args(Action): +class _CompositionArgs(Action): param: str -class _MockTool(ToolDefinition[_Args, None]): +class _CompositionMockTool(ToolDefinition[_CompositionArgs, None]): name: ClassVar[str] = "test_tool" @classmethod - def create(cls, conv_state=None, **params) -> Sequence["_MockTool"]: - return [cls(description="A test tool", action_type=_Args)] + def create(cls, conv_state=None, **params) -> Sequence["_CompositionMockTool"]: + return [cls(description="A test tool", action_type=_CompositionArgs)] def _make_llm(model: str = "gpt-4o") -> LLM: @@ -52,7 +52,8 @@ def _sample_messages() -> list[Message]: def test_compute_prompt_composition_decomposes_into_components(): formatted = _make_llm().format_messages_for_llm(_sample_messages()) tools = [ - t.to_openai_tool(add_security_risk_prediction=True) for t in _MockTool.create() + t.to_openai_tool(add_security_risk_prediction=True) + for t in _CompositionMockTool.create() ] composition = compute_prompt_composition( @@ -117,7 +118,8 @@ def test_compute_prompt_composition_tool_schema_tokens_matches_controlled_delta( """tool_schema_tokens must equal the marginal cost of adding tools.""" formatted = _make_llm().format_messages_for_llm(_sample_messages()) tools = [ - t.to_openai_tool(add_security_risk_prediction=True) for t in _MockTool.create() + t.to_openai_tool(add_security_risk_prediction=True) + for t in _CompositionMockTool.create() ] with_tools = compute_prompt_composition( @@ -264,8 +266,8 @@ def _write_log(run_dir, name: str, payload: dict): _TOOL_DEFINITION_DUMP = { "description": "A test tool", - "action_type": "_Args", - "kind": "_MockTool", + "action_type": "_CompositionArgs", + "kind": "_CompositionMockTool", "title": "test_tool", } From 62a7387e1ca53a93220b26541d97c0a6597102c1 Mon Sep 17 00:00:00 2001 From: george larson Date: Thu, 3 Sep 2026 02:34:04 +0000 Subject: [PATCH 15/15] ci: clean Docker-owned TypeScript integration workspace Co-authored-by: openhands --- .github/workflows/typescript-client-integration-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/typescript-client-integration-tests.yml b/.github/workflows/typescript-client-integration-tests.yml index d84cb70e7b..25bf396192 100644 --- a/.github/workflows/typescript-client-integration-tests.yml +++ b/.github/workflows/typescript-client-integration-tests.yml @@ -151,6 +151,7 @@ jobs: run: | docker stop agent-server || true docker rm agent-server || true + sudo chown -R "$USER:$USER" ${{ env.HOST_WORKSPACE_DIR }} || true rm -rf ${{ env.HOST_WORKSPACE_DIR }} || true # Separate job for quick smoke test without LLM