From 9fa6675aee2d1dfd1880896c784dc662faf1a3bd Mon Sep 17 00:00:00 2001 From: liramon2 Date: Fri, 14 Aug 2026 22:13:55 -0400 Subject: [PATCH] fix: use start_time tiebreaker for all root agent cases --- src/strands_evals/types/trace.py | 16 ++-- .../extractors/test_trace_extractor.py | 53 ++++++++++++ tests/strands_evals/types/test_trace.py | 81 ++++++++++++++++++- 3 files changed, 140 insertions(+), 10 deletions(-) diff --git a/src/strands_evals/types/trace.py b/src/strands_evals/types/trace.py index 600de26a..7dab2896 100644 --- a/src/strands_evals/types/trace.py +++ b/src/strands_evals/types/trace.py @@ -135,18 +135,16 @@ def _to_aware_utc(dt: datetime) -> datetime: def _find_root_agent_span(agent_spans: Sequence[AgentInvocationSpan]) -> AgentInvocationSpan | None: """Search for a trace's root agent span. - Prefers a parentless agent that has content, then any parentless agent, - then the earliest by `start_time`. Returns `None` for an empty sequence. + Prefers a parentless agent with content, then any parentless agent, then all + agents; within the chosen tier the earliest `start_time` wins. """ if not agent_spans: return None - for span in agent_spans: - if span.span_info.parent_span_id is None and (span.user_prompt or span.agent_response): - return span - for span in agent_spans: - if span.span_info.parent_span_id is None: - return span - return min(agent_spans, key=lambda s: _to_aware_utc(s.span_info.start_time)) + + parentless = [s for s in agent_spans if s.span_info.parent_span_id is None] + with_content = [s for s in parentless if s.user_prompt or s.agent_response] + candidates = with_content or parentless or list(agent_spans) + return min(candidates, key=lambda s: _to_aware_utc(s.span_info.start_time)) class Trace(BaseModel): diff --git a/tests/strands_evals/extractors/test_trace_extractor.py b/tests/strands_evals/extractors/test_trace_extractor.py index 13072063..edf3ea5b 100644 --- a/tests/strands_evals/extractors/test_trace_extractor.py +++ b/tests/strands_evals/extractors/test_trace_extractor.py @@ -453,3 +453,56 @@ def test_span_id_none_does_not_collide(): result_b = next(r for r in result if r.tool_execution_details.tool_call.name == "tool_b") assert [t.name for t in result_a.available_tools] == ["tool_a"] assert [t.name for t in result_b.available_tools] == ["tool_b"] + + +def test_tool_level_flat_multi_agent_anchors_to_earliest_coordinator(): + """Flat trace, sub-agent first in list order: history and orphan tool anchor to the earliest coordinator.""" + from datetime import timedelta + + base = datetime(2026, 1, 1, tzinfo=timezone.utc) + # Sub-agent listed first, but starts 5s after the coordinator. + sub = AgentInvocationSpan( + span_info=SpanInfo( + session_id="test", + span_id="sub", + parent_span_id=None, + start_time=base + timedelta(seconds=5), + end_time=base + timedelta(seconds=6), + ), + user_prompt="current weather in London", + agent_response="rainy", + available_tools=[ToolConfig(name="get_weather")], + ) + coordinator = AgentInvocationSpan( + span_info=SpanInfo( + session_id="test", + span_id="coordinator", + parent_span_id=None, + start_time=base, + end_time=base + timedelta(seconds=10), + ), + user_prompt="weather in NY and London, then the difference", + agent_response="here is the difference", + available_tools=[ToolConfig(name="ask_research"), ToolConfig(name="ask_math")], + ) + # Orphan tool span: no parent and no agent_span_id, so it falls back to the root. + orphan_tool = ToolExecutionSpan( + span_info=SpanInfo( + session_id="test", + span_id="tool", + parent_span_id=None, + start_time=base + timedelta(seconds=5), + end_time=base + timedelta(seconds=6), + ), + tool_call=ToolCall(name="get_weather", arguments={"city": "London"}), + tool_result=ToolResult(content="rainy"), + ) + trace = Trace(spans=[sub, coordinator, orphan_tool], trace_id="t1", session_id="test") + session = Session(traces=[trace], session_id="test") + + result = TraceExtractor(EvaluationLevel.TOOL_LEVEL).extract(session) + + assert len(result) == 1 + assert result[0].session_history[0].content[0].text == "weather in NY and London, then the difference" + assert result[0].tool_execution_details.agent_span_id == "coordinator" + assert [t.name for t in result[0].available_tools] == ["ask_research", "ask_math"] diff --git a/tests/strands_evals/types/test_trace.py b/tests/strands_evals/types/test_trace.py index 1c05a685..a30731f5 100644 --- a/tests/strands_evals/types/test_trace.py +++ b/tests/strands_evals/types/test_trace.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from strands_evals.types.trace import ( AgentInvocationSpan, @@ -19,6 +19,7 @@ Trace, TraceLevelInput, UserMessage, + _find_root_agent_span, ) @@ -348,3 +349,81 @@ def test_tools_without_span_ids_each_owned_by_their_own_agent(): assert forecast.agent_span_id == "weather-agent" assert alerts.agent_span_id == "weather-agent" + + +_ROOT_BASE = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _root_agent( + span_id: str, *, parent: str | None = None, offset: int = 0, prompt: str = "", response: str = "" +) -> AgentInvocationSpan: + """Build an AgentInvocationSpan starting `offset` seconds after a base time.""" + start = _ROOT_BASE + timedelta(seconds=offset) + return AgentInvocationSpan( + span_info=SpanInfo(span_id=span_id, session_id="s", parent_span_id=parent, start_time=start, end_time=start), + user_prompt=prompt, + agent_response=response, + available_tools=[], + ) + + +class TestFindRootAgentSpan: + def test_empty_returns_none(self): + """An empty sequence yields None.""" + assert _find_root_agent_span([]) is None + + def test_earliest_parentless_with_content_wins_regardless_of_order(self): + """Among parentless-with-content agents the earliest start wins, not list order. + + The coordinator is listed last but starts first — the old list-order logic + would have returned the first sub-agent instead. + """ + spans = [ + _root_agent("sub_a", offset=2, prompt="current weather in London"), + _root_agent("sub_b", offset=2, prompt="current weather in New York"), + _root_agent("coordinator", offset=0, prompt="weather in NY and London, then the difference"), + ] + assert _find_root_agent_span(spans).span_info.span_id == "coordinator" + + def test_prefers_content_over_earlier_empty_parentless(self): + """A content span beats an earlier empty one; both prompt and response count as content.""" + by_prompt = [_root_agent("empty", offset=0), _root_agent("with_prompt", offset=1, prompt="do the thing")] + assert _find_root_agent_span(by_prompt).span_info.span_id == "with_prompt" + + by_response = [_root_agent("empty", offset=0), _root_agent("with_response", offset=1, response="done")] + assert _find_root_agent_span(by_response).span_info.span_id == "with_response" + + def test_falls_back_to_parentless_when_none_have_content(self): + """When no parentless agent has content, the earliest parentless wins over any parented span.""" + spans = [ + _root_agent("parented", parent="p", offset=0), + _root_agent("late", offset=5), + _root_agent("early", offset=1), + ] + assert _find_root_agent_span(spans).span_info.span_id == "early" + + def test_mixed_naive_and_aware_start_times(self): + """Naive and aware start times are compared as UTC without raising TypeError.""" + aware = _root_agent("aware", offset=0, prompt="a") + naive_start = datetime(2026, 1, 1, 0, 0, 5) + naive = AgentInvocationSpan( + span_info=SpanInfo( + span_id="naive", + session_id="s", + parent_span_id=None, + start_time=naive_start, + end_time=naive_start, + ), + user_prompt="b", + agent_response="", + available_tools=[], + ) + assert _find_root_agent_span([aware, naive]).span_info.span_id == "aware" + + def test_falls_back_to_earliest_when_all_parented(self): + """When every agent has a parent, the earliest-start agent overall is chosen.""" + spans = [ + _root_agent("late", parent="p", offset=5, prompt="x"), + _root_agent("early", parent="p", offset=1, prompt="y"), + ] + assert _find_root_agent_span(spans).span_info.span_id == "early"