Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions src/strands_evals/types/trace.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,18 +135,16 @@ def _to_aware_utc(dt: datetime) -> datetime:
def _find_root_agent_span(agent_spans: Sequence[AgentInvocationSpan]) -> AgentInvocationSpan | None:
"""Search for a trace's root agent span.

Prefers a parentless agent that has content, then any parentless agent,
then the earliest by `start_time`. Returns `None` for an empty sequence.
Prefers a parentless agent with content, then any parentless agent, then all
agents; within the chosen tier the earliest `start_time` wins.
"""
if not agent_spans:
return None
for span in agent_spans:
if span.span_info.parent_span_id is None and (span.user_prompt or span.agent_response):
return span
for span in agent_spans:
if span.span_info.parent_span_id is None:
return span
return min(agent_spans, key=lambda s: _to_aware_utc(s.span_info.start_time))

parentless = [s for s in agent_spans if s.span_info.parent_span_id is None]
with_content = [s for s in parentless if s.user_prompt or s.agent_response]
candidates = with_content or parentless or list(agent_spans)
return min(candidates, key=lambda s: _to_aware_utc(s.span_info.start_time))


class Trace(BaseModel):
Expand Down
53 changes: 53 additions & 0 deletions tests/strands_evals/extractors/test_trace_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,3 +453,56 @@ def test_span_id_none_does_not_collide():
result_b = next(r for r in result if r.tool_execution_details.tool_call.name == "tool_b")
assert [t.name for t in result_a.available_tools] == ["tool_a"]
assert [t.name for t in result_b.available_tools] == ["tool_b"]


def test_tool_level_flat_multi_agent_anchors_to_earliest_coordinator():
"""Flat trace, sub-agent first in list order: history and orphan tool anchor to the earliest coordinator."""
from datetime import timedelta

base = datetime(2026, 1, 1, tzinfo=timezone.utc)
# Sub-agent listed first, but starts 5s after the coordinator.
sub = AgentInvocationSpan(
span_info=SpanInfo(
session_id="test",
span_id="sub",
parent_span_id=None,
start_time=base + timedelta(seconds=5),
end_time=base + timedelta(seconds=6),
),
user_prompt="current weather in London",
agent_response="rainy",
available_tools=[ToolConfig(name="get_weather")],
)
coordinator = AgentInvocationSpan(
span_info=SpanInfo(
session_id="test",
span_id="coordinator",
parent_span_id=None,
start_time=base,
end_time=base + timedelta(seconds=10),
),
user_prompt="weather in NY and London, then the difference",
agent_response="here is the difference",
available_tools=[ToolConfig(name="ask_research"), ToolConfig(name="ask_math")],
)
# Orphan tool span: no parent and no agent_span_id, so it falls back to the root.
orphan_tool = ToolExecutionSpan(
span_info=SpanInfo(
session_id="test",
span_id="tool",
parent_span_id=None,
start_time=base + timedelta(seconds=5),
end_time=base + timedelta(seconds=6),
),
tool_call=ToolCall(name="get_weather", arguments={"city": "London"}),
tool_result=ToolResult(content="rainy"),
)
trace = Trace(spans=[sub, coordinator, orphan_tool], trace_id="t1", session_id="test")
session = Session(traces=[trace], session_id="test")

result = TraceExtractor(EvaluationLevel.TOOL_LEVEL).extract(session)

assert len(result) == 1
assert result[0].session_history[0].content[0].text == "weather in NY and London, then the difference"
assert result[0].tool_execution_details.agent_span_id == "coordinator"
assert [t.name for t in result[0].available_tools] == ["ask_research", "ask_math"]
81 changes: 80 additions & 1 deletion tests/strands_evals/types/test_trace.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone

from strands_evals.types.trace import (
AgentInvocationSpan,
Expand All @@ -19,6 +19,7 @@
Trace,
TraceLevelInput,
UserMessage,
_find_root_agent_span,
)


Expand Down Expand Up @@ -348,3 +349,81 @@ def test_tools_without_span_ids_each_owned_by_their_own_agent():

assert forecast.agent_span_id == "weather-agent"
assert alerts.agent_span_id == "weather-agent"


_ROOT_BASE = datetime(2026, 1, 1, tzinfo=timezone.utc)


def _root_agent(
span_id: str, *, parent: str | None = None, offset: int = 0, prompt: str = "", response: str = ""
) -> AgentInvocationSpan:
"""Build an AgentInvocationSpan starting `offset` seconds after a base time."""
start = _ROOT_BASE + timedelta(seconds=offset)
return AgentInvocationSpan(
span_info=SpanInfo(span_id=span_id, session_id="s", parent_span_id=parent, start_time=start, end_time=start),
user_prompt=prompt,
agent_response=response,
available_tools=[],
)


class TestFindRootAgentSpan:
def test_empty_returns_none(self):
"""An empty sequence yields None."""
assert _find_root_agent_span([]) is None

def test_earliest_parentless_with_content_wins_regardless_of_order(self):
"""Among parentless-with-content agents the earliest start wins, not list order.

The coordinator is listed last but starts first — the old list-order logic
would have returned the first sub-agent instead.
"""
spans = [
_root_agent("sub_a", offset=2, prompt="current weather in London"),
_root_agent("sub_b", offset=2, prompt="current weather in New York"),
_root_agent("coordinator", offset=0, prompt="weather in NY and London, then the difference"),
]
assert _find_root_agent_span(spans).span_info.span_id == "coordinator"

def test_prefers_content_over_earlier_empty_parentless(self):
"""A content span beats an earlier empty one; both prompt and response count as content."""
by_prompt = [_root_agent("empty", offset=0), _root_agent("with_prompt", offset=1, prompt="do the thing")]
assert _find_root_agent_span(by_prompt).span_info.span_id == "with_prompt"

by_response = [_root_agent("empty", offset=0), _root_agent("with_response", offset=1, response="done")]
assert _find_root_agent_span(by_response).span_info.span_id == "with_response"

def test_falls_back_to_parentless_when_none_have_content(self):
"""When no parentless agent has content, the earliest parentless wins over any parented span."""
spans = [
_root_agent("parented", parent="p", offset=0),
_root_agent("late", offset=5),
_root_agent("early", offset=1),
]
assert _find_root_agent_span(spans).span_info.span_id == "early"

def test_mixed_naive_and_aware_start_times(self):
"""Naive and aware start times are compared as UTC without raising TypeError."""
aware = _root_agent("aware", offset=0, prompt="a")
naive_start = datetime(2026, 1, 1, 0, 0, 5)
naive = AgentInvocationSpan(
span_info=SpanInfo(
span_id="naive",
session_id="s",
parent_span_id=None,
start_time=naive_start,
end_time=naive_start,
),
user_prompt="b",
agent_response="",
available_tools=[],
)
assert _find_root_agent_span([aware, naive]).span_info.span_id == "aware"

def test_falls_back_to_earliest_when_all_parented(self):
"""When every agent has a parent, the earliest-start agent overall is chosen."""
spans = [
_root_agent("late", parent="p", offset=5, prompt="x"),
_root_agent("early", parent="p", offset=1, prompt="y"),
]
assert _find_root_agent_span(spans).span_info.span_id == "early"