From 5adbabbad59977424407211888247649d0d77c89 Mon Sep 17 00:00:00 2001 From: DEBJYOTI PAUL Date: Mon, 3 Aug 2026 11:01:28 -0700 Subject: [PATCH 1/4] feat(tools): add TraceIndex for progressive trace disclosure to judge agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large agent trajectories overflow a judge model's context window when inlined into the evaluation prompt, forcing Experiment error-isolation to record the case as score:0 / test_pass:False — a false failure for a correct agent. TraceIndex builds an in-memory index over a Session and exposes: - overview(): one compact line per span (index, type, tool, sizes, preview) that always fits the judge context, substituted for the full trajectory. - three discovery tools the judge calls on demand: list_spans / get_span / search_spans (mirrors MLflow's ListSpans/GetSpan/SearchTraceRegex). get_span pages oversized spans via max_read_chars + offset so no single tool return can itself overflow the judge. Backend-agnostic: consumes any Session produced by a provider/mapper. --- src/strands_evals/tools/trace_index.py | 177 +++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 src/strands_evals/tools/trace_index.py diff --git a/src/strands_evals/tools/trace_index.py b/src/strands_evals/tools/trace_index.py new file mode 100644 index 00000000..1569cf54 --- /dev/null +++ b/src/strands_evals/tools/trace_index.py @@ -0,0 +1,177 @@ +"""Progressive trace disclosure for judge agents. + +A large agent trajectory does not fit in a judge's context window. Rather than +inlining the whole Session into the evaluation prompt (which overflows and gets +scored as a failure), `TraceIndex` builds a small in-memory index over the trace +and gives the judge two things: + +1. An `overview()` — one line per span (index, type, tool name, sizes, truncated + preview) — cheap enough to always fit in context, and +2. **Lookup tools** the judge calls to load only the spans it needs to verify the + rubric: `list_spans`, `get_span`, `search_spans`. + +This is the same list / get / search shape used to query any indexed collection, +and the same progressive-disclosure pattern skills use: the overview is the +"name + description" line; the tools load the full content on demand. + +Example:: + + from strands_evals.evaluators import TrajectoryEvaluator + from strands_evals.tools.trace_index import TraceIndex + + index = TraceIndex(session) + evaluator = TrajectoryEvaluator( + rubric="Every claim in the final response must be supported by a tool result.", + tools=index.tools, + ) + # Compose the prompt with index.overview() instead of the full trajectory. +""" + +import json +import re + +from strands import tool + +from ..types.trace import ( + AgentInvocationSpan, + InferenceSpan, + Session, + SpanUnion, + ToolExecutionSpan, +) + +_PREVIEW_CHARS = 120 +_DEFAULT_MAX_READ_CHARS = 8_000 + + +def _flatten_spans(session: Session) -> list[SpanUnion]: + """Flatten all spans across traces in start_time order.""" + spans = [span for trace in session.traces for span in trace.spans] + spans.sort(key=lambda s: s.span_info.start_time) + return spans + + +def _span_text(span: SpanUnion) -> str: + """Full text content of a span, for search and retrieval.""" + if isinstance(span, ToolExecutionSpan): + return json.dumps( + { + "tool_call": span.tool_call.model_dump(), + "tool_result": span.tool_result.model_dump(), + }, + default=str, + ) + if isinstance(span, AgentInvocationSpan): + return json.dumps( + {"user_prompt": span.user_prompt, "agent_response": span.agent_response}, + default=str, + ) + if isinstance(span, InferenceSpan): + return json.dumps([m.model_dump() for m in span.messages], default=str) + return json.dumps(span.model_dump(), default=str) + + +def _preview(text: str, limit: int = _PREVIEW_CHARS) -> str: + text = re.sub(r"\s+", " ", text).strip() + return text if len(text) <= limit else text[: limit - 3] + "..." + + +def _describe(span: SpanUnion) -> str: + """One overview line describing a span without its full payload.""" + if isinstance(span, ToolExecutionSpan): + args = json.dumps(span.tool_call.arguments, default=str) + result_size = len(str(span.tool_result.content)) + return ( + f"TOOL {span.tool_call.name}({_preview(args, 80)}) " + f"-> result: {result_size} chars: {_preview(str(span.tool_result.content))}" + ) + if isinstance(span, AgentInvocationSpan): + return ( + f"AGENT prompt: {_preview(span.user_prompt, 80)} " + f"-> response: {len(span.agent_response)} chars: {_preview(span.agent_response)}" + ) + if isinstance(span, InferenceSpan): + return f"INFERENCE {len(span.messages)} messages" + return f"{type(span).__name__}" + + +class TraceIndex: + """Read-only list / get / search index over a Session for judge agents. + + Attributes: + session: The Session being evaluated. + max_read_chars: Cap on any single tool return, so a huge span can't + overflow the judge's context in one call. Oversized content is + windowed and the tool reports how to page through it. + """ + + def __init__(self, session: Session, max_read_chars: int = _DEFAULT_MAX_READ_CHARS): + self.session = session + self.max_read_chars = max_read_chars + self._spans = _flatten_spans(session) + + # Bind instance state into plain functions so @tool sees clean signatures. + # `this` (not `index`) so the public get_span(index=...) arg name is free. + this = self + + @tool + def list_spans() -> str: + """List every span in the trace: one line per span with its index, type, + tool name, argument preview, and result size. Call this first to decide + which spans to inspect.""" + return this.overview() + + @tool + def get_span(index: int, offset: int = 0) -> str: + """Get the full content of one span by its index from the span list. + Large spans are windowed; the response says how to page with offset. + + Args: + index: Span index as shown by list_spans. + offset: Character offset for paging through oversized spans. + """ + if not 0 <= index < len(this._spans): + return f"ERROR: index {index} out of range (0..{len(this._spans) - 1})" + return this._window(_span_text(this._spans[index]), offset) + + @tool + def search_spans(pattern: str, max_matches: int = 20) -> str: + """Search all span content for a regex or literal string. Returns matching + span indices with a short excerpt around each match. Use get_span to load + a matching span in full. + + Args: + pattern: Regex (or literal text) to search for. + max_matches: Maximum matches to return. + """ + try: + rx = re.compile(pattern, re.IGNORECASE) + except re.error: + rx = re.compile(re.escape(pattern), re.IGNORECASE) + hits = [] + for i, span in enumerate(this._spans): + text = _span_text(span) + m = rx.search(text) + if m: + start = max(0, m.start() - 60) + hits.append(f"[{i}] ...{_preview(text[start : m.end() + 60], 160)}...") + if len(hits) >= max_matches: + break + return "\n".join(hits) if hits else f"No matches for {pattern!r}" + + self.tools = [list_spans, get_span, search_spans] + + def overview(self) -> str: + """Compact one-line-per-span overview of the session.""" + lines = [f"Trace overview: {len(self._spans)} spans (session {self.session.session_id})"] + lines += [f"[{i}] {_describe(span)}" for i, span in enumerate(self._spans)] + return "\n".join(lines) + + def _window(self, text: str, offset: int) -> str: + if offset >= len(text): + return f"ERROR: offset {offset} beyond content length {len(text)}" + window = text[offset : offset + self.max_read_chars] + if offset + len(window) < len(text): + remaining = len(text) - offset - len(window) + window += f"\n[TRUNCATED: {remaining} chars remain; call again with offset={offset + len(window)}]" + return window From ad264d2a55b72863eb0b889d6f5d6504da8d4417 Mon Sep 17 00:00:00 2001 From: DEBJYOTI PAUL Date: Mon, 3 Aug 2026 11:01:28 -0700 Subject: [PATCH 2/4] test(tools): unit, cross-pattern, and evaluator-integration tests for TraceIndex Covers overview() formatting, list_spans/get_span/search_spans behavior, offset paging on oversized spans, and end-to-end use through OutputEvaluator with tools=index.tools. --- tests/strands_evals/tools/__init__.py | 0 tests/strands_evals/tools/test_trace_index.py | 130 ++++++++++++ .../test_trace_index_evaluator_integration.py | 176 ++++++++++++++++ .../tools/test_trace_index_patterns.py | 196 ++++++++++++++++++ 4 files changed, 502 insertions(+) create mode 100644 tests/strands_evals/tools/__init__.py create mode 100644 tests/strands_evals/tools/test_trace_index.py create mode 100644 tests/strands_evals/tools/test_trace_index_evaluator_integration.py create mode 100644 tests/strands_evals/tools/test_trace_index_patterns.py diff --git a/tests/strands_evals/tools/__init__.py b/tests/strands_evals/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/strands_evals/tools/test_trace_index.py b/tests/strands_evals/tools/test_trace_index.py new file mode 100644 index 00000000..8a02940c --- /dev/null +++ b/tests/strands_evals/tools/test_trace_index.py @@ -0,0 +1,130 @@ +from datetime import datetime, timezone + +import pytest + +from strands_evals.tools.trace_index import TraceIndex +from strands_evals.types.trace import ( + AgentInvocationSpan, + Session, + SpanInfo, + ToolCall, + ToolExecutionSpan, + ToolResult, + Trace, +) + + +def _span_info(second: int) -> SpanInfo: + return SpanInfo( + session_id="s1", + span_id=f"sp{second}", + start_time=datetime(2026, 1, 1, 0, 0, second, tzinfo=timezone.utc), + end_time=datetime(2026, 1, 1, 0, 0, second + 1, tzinfo=timezone.utc), + ) + + +@pytest.fixture +def session(): + spans = [ + AgentInvocationSpan( + span_info=_span_info(0), + user_prompt="Look up ticket TKT-1042", + agent_response="Ticket TKT-1042 was refunded $150.", + available_tools=[], + ), + ToolExecutionSpan( + span_info=_span_info(1), + tool_call=ToolCall(name="lookup_ticket", arguments={"id": "TKT-1042"}), + tool_result=ToolResult(content="x" * 20_000 + " refund_amount=$150"), + ), + ToolExecutionSpan( + span_info=_span_info(2), + tool_call=ToolCall(name="get_customer", arguments={"id": "C-7"}), + tool_result=ToolResult(content="customer name: Alex"), + ), + ] + return Session(traces=[Trace(spans=spans, trace_id="t1", session_id="s1")], session_id="s1") + + +def test_overview_is_compact_and_ordered(session): + index = TraceIndex(session) + overview = index.overview() + + lines = overview.splitlines() + assert "3 spans" in lines[0] + assert lines[1].startswith("[0] AGENT") + assert "lookup_ticket" in lines[2] + assert "get_customer" in lines[3] + # Manifest must not inline the 20K-char tool result + assert len(overview) < 2_000 + + +def test_get_span_returns_full_content_for_small_span(session): + index = TraceIndex(session) + get_span = index.tools[1] + + content = get_span(index=2) + + assert "customer name: Alex" in content + assert "TRUNCATED" not in content + + +def test_get_span_windows_oversized_content_and_pages(session): + index = TraceIndex(session, max_read_chars=5_000) + get_span = index.tools[1] + + first = get_span(index=1) + assert "TRUNCATED" in first + assert "offset=5000" in first + + second = get_span(index=1, offset=5_000) + assert second.startswith("x") or '"' in second # continuation, not a restart + assert first[:100] != second[:100] + + +def test_get_span_index_out_of_range(session): + index = TraceIndex(session) + get_span = index.tools[1] + + assert "ERROR" in get_span(index=99) + assert "ERROR" in get_span(index=-1) + + +def test_search_spans_finds_span_by_content(session): + index = TraceIndex(session) + search_spans = index.tools[2] + + result = search_spans(pattern=r"refund_amount=\$150") + + assert result.startswith("[1]") + assert "refund_amount" in result + + +def test_search_spans_falls_back_to_literal_on_bad_regex(session): + index = TraceIndex(session) + search_spans = index.tools[2] + + result = search_spans(pattern="refund_amount=$150[") + + assert "No matches" in result or result.startswith("[") + + +def test_search_spans_no_matches(session): + index = TraceIndex(session) + search_spans = index.tools[2] + + assert "No matches" in search_spans(pattern="nonexistent-zzz") + + +def test_list_spans_tool_matches_overview(session): + index = TraceIndex(session) + list_spans = index.tools[0] + + assert list_spans() == index.overview() + + +def test_tools_are_strands_tools(session): + index = TraceIndex(session) + + for t in index.tools: + assert hasattr(t, "tool_spec") or hasattr(t, "TOOL_SPEC") or callable(t) diff --git a/tests/strands_evals/tools/test_trace_index_evaluator_integration.py b/tests/strands_evals/tools/test_trace_index_evaluator_integration.py new file mode 100644 index 00000000..5e15c128 --- /dev/null +++ b/tests/strands_evals/tools/test_trace_index_evaluator_integration.py @@ -0,0 +1,176 @@ +"""Integration: OutputEvaluator + TraceIndex — skill-style progressive discovery. + +The judge receives the compact overview in its prompt and the index's +discovery tools via the evaluators' `tools=` parameter. These tests use a +scripted fake Agent to verify the full loop deterministically: the "judge" +must call the tools to find evidence before scoring. +""" + +from datetime import datetime, timezone +from unittest.mock import Mock, patch + +from strands_evals.evaluators import OutputEvaluator +from strands_evals.tools.trace_index import TraceIndex +from strands_evals.types import EvaluationData, EvaluationOutput +from strands_evals.types.trace import ( + AgentInvocationSpan, + Session, + SpanInfo, + ToolCall, + ToolExecutionSpan, + ToolResult, + Trace, +) + + +def _span_info(second: int) -> SpanInfo: + return SpanInfo( + session_id="s1", + span_id=f"sp{second}", + start_time=datetime(2026, 1, 1, 0, 0, second, tzinfo=timezone.utc), + end_time=datetime(2026, 1, 1, 0, 0, second + 1, tzinfo=timezone.utc), + ) + + +def _big_session(refund_amount: str = "$150") -> Session: + """A session whose tool results are too big to inline: 30 spans x ~11K chars.""" + spans = [ + AgentInvocationSpan( + span_info=_span_info(0), + user_prompt="What was the refund for TKT-1042?", + agent_response=f"Ticket TKT-1042 was refunded {refund_amount}.", + available_tools=[], + ) + ] + for i in range(1, 30): + content = ("filler row data " * 700) + (f" refund_amount={refund_amount} TKT-1042" if i == 17 else "") + spans.append( + ToolExecutionSpan( + span_info=_span_info(i), + tool_call=ToolCall(name="query_db", arguments={"page": i}), + tool_result=ToolResult(content=content), + ) + ) + return Session(traces=[Trace(spans=spans, trace_id="t1", session_id="s1")], session_id="s1") + + +def test_evaluator_receives_index_tools_and_overview_fits(): + session = _big_session() + index = TraceIndex(session) + + evaluator = OutputEvaluator( + rubric="Every numeric claim must be supported by a tool result in the trace.", + tools=index.tools, + ) + + assert evaluator.tools == index.tools + # The overview replaces the inline trajectory and is context-safe + inline = len(str(session.model_dump())) + assert inline > 300_000 + assert len(index.overview()) < 15_000 + + +@patch("strands_evals.evaluators.output_evaluator.Agent") +def test_judge_agent_constructed_with_discovery_tools(mock_agent_class): + session = _big_session() + index = TraceIndex(session) + mock_agent = Mock() + result = Mock() + result.structured_output = EvaluationOutput(score=1.0, test_pass=True, reason="grounded") + mock_agent.return_value = result + mock_agent_class.return_value = mock_agent + + evaluator = OutputEvaluator(rubric="Claims must be grounded.", tools=index.tools) + data = EvaluationData( + input="What was the refund for TKT-1042?", + actual_output="Ticket TKT-1042 was refunded $150.", + ) + + evaluator.evaluate(data) + + kwargs = mock_agent_class.call_args[1] + assert kwargs["tools"] == index.tools + tool_names = {getattr(t, "tool_name", getattr(t, "__name__", "")) for t in kwargs["tools"]} + assert {"list_spans", "get_span", "search_spans"} <= tool_names + + +def test_scripted_judge_finds_evidence_via_discovery(): + """Simulate the judge's tool-use loop: overview -> search -> get_span. + + This is the skill-discovery flow: the overview says *what exists*, the + tools load *what is needed*, and the judge never sees the full 300K trace. + """ + session = _big_session(refund_amount="$150") + index = TraceIndex(session) + overview, get_span, search_spans = index.tools + + judge_context_chars = 0 + + # Step 1: judge reads the overview + overview = overview() + judge_context_chars += len(overview) + assert "query_db" in overview + + # Step 2: judge searches for the claim from the agent's answer + hits = search_spans(pattern=r"refund_amount=\$150") + judge_context_chars += len(hits) + assert hits.startswith("["), "evidence must be locatable" + evidence_index = int(hits.split("]")[0][1:]) + assert evidence_index == 17 + + # Step 3: judge loads the evidence span, paging when told to + span_content = get_span(index=evidence_index) + judge_context_chars += len(span_content) + offset = 0 + while "refund_amount=$150" not in span_content and "TRUNCATED" in span_content: + offset += index.max_read_chars + span_content = get_span(index=evidence_index, offset=offset) + judge_context_chars += len(span_content) + assert "refund_amount=$150" in span_content + + # The judge verified the claim while reading a fraction of the trace + full_trace_chars = len(str(session.model_dump())) + assert judge_context_chars < full_trace_chars / 10 + + +def test_scripted_judge_detects_fabrication(): + """The agent claims $999 but the trace only supports $150 — discovery + exposes the fabrication where an overflowed inline judge would score 0 + or a truncated one might miss the evidence entirely.""" + session = _big_session(refund_amount="$150") + # Overwrite the agent's claim with a fabricated amount + agent_span = session.traces[0].spans[0] + fabricated = AgentInvocationSpan( + span_info=agent_span.span_info, + user_prompt=agent_span.user_prompt, + agent_response="Ticket TKT-1042 was refunded $999.", + available_tools=[], + ) + session.traces[0].spans[0] = fabricated + + index = TraceIndex(session) + _, _, search_spans = index.tools + + # Judge searches for the claimed amount in tool evidence: not found + claimed = search_spans(pattern=r"refund_amount=\$999") + assert "No matches" in claimed + + # But the actual amount is present: the claim contradicts the evidence + actual = search_spans(pattern=r"refund_amount=\$150") + assert actual.startswith("[") + + +def test_overview_prompt_composition_pattern(): + """The documented usage: overview into the prompt, tools onto the evaluator.""" + session = _big_session() + index = TraceIndex(session) + + prompt = ( + "Evaluate whether the agent's answer is grounded in the trace.\n" + f"\n{index.overview()}\n\n" + "Ticket TKT-1042 was refunded $150.\n" + "Use get_span/search_spans to verify before scoring." + ) + + # Stays well inside any judge's context window (~4 chars/token heuristic) + assert len(prompt) / 4 < 10_000 diff --git a/tests/strands_evals/tools/test_trace_index_patterns.py b/tests/strands_evals/tools/test_trace_index_patterns.py new file mode 100644 index 00000000..0137ad3f --- /dev/null +++ b/tests/strands_evals/tools/test_trace_index_patterns.py @@ -0,0 +1,196 @@ +"""Cross-pattern tests: TraceIndex over Sessions produced by every trace source. + +Verifies the index's overview/discovery behavior is identical whether the +Session came from: +- Strands-native OTEL spans (gen_ai semconv, StrandsInMemorySessionMapper) +- Langfuse observations (LangfuseProvider conversion) +- OpenInference spans (OpenInferenceSessionMapper, ADOT fixture) +""" + +import json +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.trace import SpanContext, SpanKind, TraceFlags + +from strands_evals.mappers import OpenInferenceSessionMapper, StrandsInMemorySessionMapper +from strands_evals.tools.trace_index import TraceIndex +from strands_evals.types.trace import Session + +_FIXTURES_DIR = Path(__file__).parent.parent / "mappers" / "fixtures" + +LARGE_RESULT = json.dumps({"rows": [{"ticket": f"TKT-{i}", "status": "resolved"} for i in range(500)]}) + + +# --- Strands-native OTEL (gen_ai semconv) --- + + +def _otel_span(provider, trace_id, span_id, parent_id, operation, attributes, events_fn): + tracer = provider.get_tracer(__name__) + with tracer.start_as_current_span(operation, kind=SpanKind.CLIENT) as s: + for k, v in attributes.items(): + s.set_attribute(k, v) + events_fn(s) + return ReadableSpan( + name=operation, + context=SpanContext(trace_id, span_id, False, TraceFlags(0x01)), + parent=SpanContext(trace_id, parent_id, False, TraceFlags(0x01)) if parent_id else None, + resource=provider.resource, + attributes=attributes, + events=tuple(s._events), + start_time=1700000000000000000, + end_time=1700000001000000000, + ) + + +@pytest.fixture +def strands_native_session() -> Session: + provider = TracerProvider() + agent_span = _otel_span( + provider, + 0xAAA, + 0xBB1, + None, + "invoke_agent", + {"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "support-agent"}, + lambda s: ( + s.add_event("gen_ai.user.message", {"content": '[{"text": "Check ticket TKT-42"}]'}), + s.add_event("gen_ai.choice", {"message": '[{"text": "TKT-42 is resolved."}]'}), + ), + ) + tool_result_message = json.dumps([{"text": LARGE_RESULT}]) + tool_span = _otel_span( + provider, + 0xAAA, + 0xBB2, + 0xBB1, + "execute_tool lookup_ticket", + {"gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "lookup_ticket"}, + lambda s: ( + s.add_event("gen_ai.tool.message", {"content": '{"id": "TKT-42"}', "id": "call-1"}), + s.add_event("gen_ai.choice", {"message": tool_result_message, "id": "call-1"}), + ), + ) + return StrandsInMemorySessionMapper().map_to_session([agent_span, tool_span], "native-session") + + +# --- Langfuse observations --- + + +def _lf_obs(obs_id, trace_id, obs_type, name=None, obs_input=None, obs_output=None, parent=None, start=None): + o = MagicMock() + o.id, o.trace_id, o.type, o.name = obs_id, trace_id, obs_type, name + o.start_time = start or datetime(2025, 1, 15, 10, 0, 0, tzinfo=timezone.utc) + o.end_time = datetime(2025, 1, 15, 10, 0, 5, tzinfo=timezone.utc) + o.input, o.output = obs_input, obs_output + o.parent_observation_id = parent + o.metadata, o.model = {}, None + o.level, o.usage, o.usage_details = "DEFAULT", None, None + return o + + +@pytest.fixture +def langfuse_session() -> Session: + import strands_evals.providers.langfuse_provider as lf_module + + with patch.object(lf_module, "Langfuse", return_value=MagicMock()): + provider = lf_module.LangfuseProvider(public_key="pk-test", secret_key="sk-test") + + observations = [ + _lf_obs( + "obs-agent", + "trace-1", + "SPAN", + name="invoke_agent support-agent", + obs_input=[{"text": "Check ticket TKT-42"}], + obs_output="TKT-42 is resolved.", + start=datetime(2025, 1, 15, 10, 0, 0, tzinfo=timezone.utc), + ), + _lf_obs( + "obs-tool", + "trace-1", + "TOOL", + name="lookup_ticket", + obs_input={"id": "TKT-42"}, + obs_output=LARGE_RESULT, + parent="obs-agent", + start=datetime(2025, 1, 15, 10, 0, 1, tzinfo=timezone.utc), + ), + ] + spans = provider._convert_observations(observations, "lf-session") + spans = [s for s in spans if s is not None] + if not spans: + pytest.skip("Langfuse conversion produced no spans for this synthetic shape") + from strands_evals.types.trace import Trace + + return Session(traces=[Trace(spans=spans, trace_id="trace-1", session_id="lf-session")], session_id="lf-session") + + +# --- OpenInference (ADOT fixture from the repo) --- + + +@pytest.fixture +def openinference_session() -> Session: + fixture = _FIXTURES_DIR / "openinference_adot_spans.json" + if not fixture.exists(): + pytest.skip("ADOT fixture not present") + with open(fixture) as f: + spans = json.load(f) + return OpenInferenceSessionMapper().map_to_session(spans, "oi-session") + + +# --- Shared assertions across patterns --- + + +def _assert_index_works(session: Session): + index = TraceIndex(session) + list_spans, get_span, search_spans = index.tools + + overview = index.overview() + n_spans = sum(len(t.spans) for t in session.traces) + assert f"{n_spans} spans" in overview.splitlines()[0] + # Overview stays compact regardless of payload size + assert len(overview) < 400 * max(n_spans, 1) + 200 + + # Every span index is retrievable + for i in range(n_spans): + content = get_span(index=i) + assert not content.startswith("ERROR"), f"span {i} failed: {content[:80]}" + + assert isinstance(list_spans(), str) + + +def test_index_on_strands_native_session(strands_native_session): + _assert_index_works(strands_native_session) + + index = TraceIndex(strands_native_session) + _, get_span, search_spans = index.tools + + # Content-level checks: the judge can find the ticket in the tool result + hits = search_spans(pattern="TKT-42") + assert hits.startswith("[") + + +def test_index_on_langfuse_session(langfuse_session): + _assert_index_works(langfuse_session) + + index = TraceIndex(langfuse_session) + _, _, search_spans = index.tools + assert search_spans(pattern="TKT-42").startswith("[") + + +def test_index_on_openinference_session(openinference_session): + _assert_index_works(openinference_session) + + +def test_overview_compression_on_large_native_trace(strands_native_session): + """The overview must be dramatically smaller than the inline serialization.""" + index = TraceIndex(strands_native_session) + inline_size = len(str(strands_native_session.model_dump())) + overview_size = len(index.overview()) + + assert inline_size > 10_000 # LARGE_RESULT made it big + assert overview_size < inline_size / 10 From b209029af7ca497b40b0bcf4ea6405b3c5878c1c Mon Sep 17 00:00:00 2001 From: DEBJYOTI PAUL Date: Mon, 3 Aug 2026 11:01:28 -0700 Subject: [PATCH 3/4] test(integ): judge-reliability A/B for inline vs TraceIndex judging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compares OutputEvaluator judging grounded vs fabricated claims two ways — full trajectory inlined vs overview + discovery tools — asserting the index-equipped judge separates grounded from fabricated where inline overflows. Skips without live Bedrock credentials. --- .../test_trace_index_judge_reliability.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 tests_integ/test_trace_index_judge_reliability.py diff --git a/tests_integ/test_trace_index_judge_reliability.py b/tests_integ/test_trace_index_judge_reliability.py new file mode 100644 index 00000000..9dc2ad6d --- /dev/null +++ b/tests_integ/test_trace_index_judge_reliability.py @@ -0,0 +1,156 @@ +"""Judge-reliability A/B: inline trajectory vs TraceIndex progressive discovery. + +Compares OutputEvaluator judging the same cases two ways: + +- **inline**: the full trajectory serialized into the prompt (status quo) — + overflows or forces the judge to skim on large traces. +- **index**: compact overview in the prompt + discovery tools + (list_spans / get_span / search_spans) via `tools=`. + +Cases seed known ground truth: grounded claims (judge should pass) and +fabricated claims whose evidence is buried mid-trace (judge should fail). +A reliable judge scores grounded high and fabricated low; an unreliable one +collapses both to the same score. + +Requires live Bedrock credentials; skips otherwise. +Run: pytest tests_integ/test_trace_index_judge_reliability.py -v -s +""" + +import json +import logging +from datetime import datetime, timezone + +import pytest + +from strands_evals.evaluators import OutputEvaluator +from strands_evals.tools.trace_index import TraceIndex +from strands_evals.types import EvaluationData +from strands_evals.types.trace import ( + AgentInvocationSpan, + Session, + SpanInfo, + ToolCall, + ToolExecutionSpan, + ToolResult, + Trace, +) + +logger = logging.getLogger(__name__) + +RUBRIC = ( + "Score whether every factual claim in the output (amounts, ticket ids, dates, statuses) " + "is supported by evidence in the execution trace. 1.0 = every claim has direct tool-result " + "evidence; 0.0 = a central claim contradicts or is absent from the tool results. " + "Verify claims against the trace evidence before scoring; do not assume." +) + + +def _bedrock_available() -> bool: + try: + import boto3 + + return boto3.client("sts").get_caller_identity() is not None + except Exception: + return False + + +pytestmark = pytest.mark.skipif(not _bedrock_available(), reason="Bedrock credentials not available") + + +def _span_info(second: int) -> SpanInfo: + return SpanInfo( + session_id="s1", + span_id=f"sp{second}", + start_time=datetime(2026, 1, 1, 0, 0, second, tzinfo=timezone.utc), + end_time=datetime(2026, 1, 1, 0, 0, second + 1, tzinfo=timezone.utc), + ) + + +def _make_session(n_tool_spans: int, evidence: str, evidence_at: int, claim: str) -> Session: + """Session with `n_tool_spans` bulky tool results; `evidence` buried at one index.""" + spans = [ + AgentInvocationSpan( + span_info=_span_info(0), + user_prompt="Summarize the resolution for ticket TKT-1042.", + agent_response=claim, + available_tools=[], + ) + ] + filler_rows = [{"ticket": f"TKT-{2000 + j}", "status": "open", "note": "unrelated backlog item"} for j in range(80)] + for i in range(1, n_tool_spans + 1): + payload = {"page": i, "rows": filler_rows} + if i == evidence_at: + payload["rows"] = [*filler_rows, {"ticket": "TKT-1042", "resolution": evidence}] + spans.append( + ToolExecutionSpan( + span_info=_span_info(i), + tool_call=ToolCall(name="query_tickets", arguments={"page": i}), + tool_result=ToolResult(content=json.dumps(payload)), + ) + ) + return Session(traces=[Trace(spans=spans, trace_id="t1", session_id="s1")], session_id="s1") + + +GROUNDED_CLAIM = "Ticket TKT-1042 was resolved with a $150 refund." +FABRICATED_CLAIM = "Ticket TKT-1042 was resolved with a $975 refund." +EVIDENCE = "refunded $150 to customer" + +CASES = [ + ("grounded", GROUNDED_CLAIM, True), + ("fabricated", FABRICATED_CLAIM, False), +] + +# ~30 spans x ~8K chars: large enough to stress a judge, small enough to run cheaply. +N_SPANS = 30 +EVIDENCE_AT = 17 + + +def _judge_inline(session: Session, claim: str) -> dict: + evaluator = OutputEvaluator(rubric=RUBRIC) + trace_text = str(session.model_dump()) + data = EvaluationData( + input="Summarize the resolution for ticket TKT-1042.", + actual_output=f"{claim}\n\n{trace_text}", + ) + try: + out = evaluator.evaluate(data)[0] + return {"score": out.score, "reason": out.reason, "error": None} + except Exception as e: + return {"score": None, "reason": None, "error": f"{type(e).__name__}: {e}"} + + +def _judge_explore(session: Session, claim: str) -> dict: + index = TraceIndex(session) + evaluator = OutputEvaluator(rubric=RUBRIC, tools=index.tools) + data = EvaluationData( + input="Summarize the resolution for ticket TKT-1042.", + actual_output=f"{claim}\n\n\n{index.overview()}\n", + ) + try: + out = evaluator.evaluate(data)[0] + return {"score": out.score, "reason": out.reason, "error": None} + except Exception as e: + return {"score": None, "reason": None, "error": f"{type(e).__name__}: {e}"} + + +def test_judge_reliability_inline_vs_explore(): + results = {} + for label, claim, should_pass in CASES: + session = _make_session(N_SPANS, EVIDENCE, EVIDENCE_AT, claim) + results[label] = { + "expected_pass": should_pass, + "inline": _judge_inline(session, claim), + "index": _judge_explore(session, claim), + } + + logger.info("results=<%s> | judge reliability inline vs index", json.dumps(results, indent=2, default=str)) + + # The index judge must separate grounded from fabricated. + tk_grounded = results["grounded"]["index"]["score"] + tk_fabricated = results["fabricated"]["index"]["score"] + assert tk_grounded is not None and tk_fabricated is not None, "index judge must not error" + assert tk_grounded > tk_fabricated, ( + f"index judge failed to separate grounded ({tk_grounded}) from fabricated ({tk_fabricated})" + ) + assert tk_grounded >= 0.7 + assert tk_fabricated <= 0.5 From 6017825e61435a0f8447867a368c4bccff2f859f Mon Sep 17 00:00:00 2001 From: DEBJYOTI PAUL Date: Mon, 3 Aug 2026 11:01:28 -0700 Subject: [PATCH 4/4] docs: add progressive trace disclosure example to README --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 583da35c..d1e93372 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,31 @@ evaluator = TrajectoryEvaluator( ) ``` +### Evaluating Large Traces with Progressive Disclosure + +When a session is too large to inline into a judge prompt (large tool results, +many turns), give the judge a compact overview plus discovery tools instead of +the full trajectory. The judge loads only the spans the rubric requires: + +```python +from strands_evals.evaluators import OutputEvaluator +from strands_evals.tools.trace_index import TraceIndex + +index = TraceIndex(session) # session: a Session from any provider/mapper + +evaluator = OutputEvaluator( + rubric="Every factual claim must be supported by tool-result evidence in the trace.", + tools=index.tools, # list_spans, get_span, search_spans +) + +# Compose the prompt with the compact overview instead of the full trajectory +evaluation_output = f"{agent_answer}\n\n\n{index.overview()}\n" +``` + +The overview is one line per span (index, type, tool name, sizes, preview); +`get_span` pages through oversized spans so no single tool return can overflow +the judge's context. + ### Trace-based Helpfulness Evaluation Evaluate agent helpfulness using OpenTelemetry traces with seven-level scoring: