feat: add evaluator metadata method and types

src/strands_evals/evaluators/correctness_evaluator.py

@@ -6,6 +6,7 @@
 from strands.models.model import Model
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import EvaluationLevel, TraceLevelInput
 from .evaluator import Evaluator
 from .prompt_templates.correctness import get_reference_template, get_template
@@ -87,6 +88,20 @@ def __init__(
         self.version = version
         self.model = model
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether the agent's response is factually correct",
+            "method": {
+                "category": "llm_judge_output",
+                "summary": (
+                    "An LLM judge evaluates correctness of the response using either a "
+                    "3-level rubric or a binary reference comparison."
+                ),
+            },
+            "threshold": "score >= 1.0 (basic) or CORRECT verdict (reference)",
+            "tier": "quality",
+        }
+
     def _has_reference(self, evaluation_case: EvaluationData[InputT, OutputT]) -> bool:
         """Check if the evaluation case contains an expected_assertion for reference-based evaluation."""
         return bool(evaluation_case.expected_assertion)

src/strands_evals/evaluators/deterministic/environment_state.py

@@ -1,6 +1,7 @@
 from typing_extensions import Any
 
 from ...types.evaluation import EnvironmentState, EvaluationData, EvaluationOutput, InputT, OutputT
+from ...types.evaluator_metadata import EvaluatorMetadata
 from ..evaluator import Evaluator
 
 
@@ -27,6 +28,17 @@ def __init__(self, name: str, value: Any | None = None):
         super().__init__(name=name)
         self.value = value
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": f"Whether environment state '{self.name}' matches the expected value",
+            "method": {
+                "category": "deterministic_extraction",
+                "summary": "Exact equality comparison of a named environment state against an expected value.",
+            },
+            "threshold": "exact match",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         if not evaluation_case.actual_environment_state:
             return [

src/strands_evals/evaluators/deterministic/output.py

@@ -1,6 +1,7 @@
 from typing_extensions import Any
 
 from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ...types.evaluator_metadata import EvaluatorMetadata
 from ..evaluator import Evaluator
 
 
@@ -11,6 +12,17 @@ def __init__(self, value: Any | None = None, name: str | None = None):
         super().__init__(name=name)
         self.value = value
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether actual_output exactly equals an expected value",
+            "method": {
+                "category": "deterministic_string",
+                "summary": "Exact equality comparison between actual_output and expected value.",
+            },
+            "threshold": "exact match",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         expected = self.value if self.value is not None else evaluation_case.expected_output
         match = evaluation_case.actual_output == expected
@@ -34,6 +46,18 @@ def __init__(self, value: str, case_sensitive: bool = True, name: str | None = N
         self.value = value
         self.case_sensitive = case_sensitive
 
+    def metadata(self) -> EvaluatorMetadata:
+        sensitivity = "Case-sensitive" if self.case_sensitive else "Case-insensitive"
+        return {
+            "checks": "Whether actual_output contains a required substring",
+            "method": {
+                "category": "deterministic_string",
+                "summary": f"{sensitivity} substring search on actual_output.",
+            },
+            "threshold": "substring present",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         actual = str(evaluation_case.actual_output)
         target = self.value
@@ -61,6 +85,18 @@ def __init__(self, value: str, case_sensitive: bool = True, name: str | None = N
         self.value = value
         self.case_sensitive = case_sensitive
 
+    def metadata(self) -> EvaluatorMetadata:
+        sensitivity = "Case-sensitive" if self.case_sensitive else "Case-insensitive"
+        return {
+            "checks": "Whether actual_output starts with a required prefix",
+            "method": {
+                "category": "deterministic_string",
+                "summary": f"{sensitivity} prefix check on actual_output.",
+            },
+            "threshold": "prefix present",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         actual = str(evaluation_case.actual_output)
         target = self.value

src/strands_evals/evaluators/deterministic/trajectory.py

@@ -1,4 +1,5 @@
 from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ...types.evaluator_metadata import EvaluatorMetadata
 from ...types.trace import Session, ToolExecutionSpan
 from ..evaluator import Evaluator
 
@@ -10,6 +11,17 @@ def __init__(self, tool_name: str, name: str | None = None):
         super().__init__(name=name)
         self.tool_name = tool_name
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": f"Whether the tool '{self.tool_name}' was called during execution",
+            "method": {
+                "category": "deterministic_extraction",
+                "summary": "Searches the trajectory for a tool execution span matching the target tool name.",
+            },
+            "threshold": "tool called at least once",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         trajectory = evaluation_case.actual_trajectory
         if trajectory is None:

src/strands_evals/evaluators/evaluator.py

@@ -7,6 +7,7 @@
 
 from ..extractors import TraceExtractor
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import (
     AssistantMessage,
     Context,
@@ -57,6 +58,18 @@ class name when unset.
         elif self.evaluation_level:
             self._trace_extractor = TraceExtractor(self.evaluation_level)
 
+    def metadata(self) -> EvaluatorMetadata | None:
+        """Declare what this evaluator checks and how it works.
+
+        Subclasses override this method to return a typed dict describing
+        themselves. The base implementation returns None, which signals
+        that the evaluator has not declared metadata.
+
+        Returns:
+            An EvaluatorMetadata dict, or None if not declared.
+        """
+        return None
+
     def _get_model_id(self, model: Model | str | None) -> str:
         """Extract model_id from a Model instance or string for serialization.
 

src/strands_evals/evaluators/faithfulness_evaluator.py

@@ -6,6 +6,7 @@
 from strands.models.model import Model
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import EvaluationLevel
 from .evaluator import Evaluator
 from .prompt_templates.faithfulness import get_template
@@ -53,6 +54,20 @@ def __init__(
         self.version = version
         self.model = model
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether the agent's response is grounded in the conversation history",
+            "method": {
+                "category": "llm_judge_output",
+                "summary": (
+                    "An LLM judge compares the agent's last response against prior "
+                    "tool outputs and conversation for unsupported claims."
+                ),
+            },
+            "threshold": "score >= 0.50",
+            "tier": "guardrail",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_trace_level_prompt(parsed_input)

src/strands_evals/evaluators/goal_success_rate_evaluator.py

@@ -6,6 +6,7 @@
 from strands.models.model import Model
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import EvaluationLevel, SessionLevelInput
 from .evaluator import Evaluator
 from .prompt_templates.goal_success_rate import get_assertion_template, get_template
@@ -84,6 +85,20 @@ def __init__(
         self.version = version
         self.model = model
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether all user goals were successfully achieved in the conversation",
+            "method": {
+                "category": "llm_judge_trajectory",
+                "summary": (
+                    "An LLM judge analyzes the full session to determine if user goals "
+                    "were met, either by inferring goals or checking explicit assertions."
+                ),
+            },
+            "threshold": "score == 1.0 (Yes or SUCCESS)",
+            "tier": "quality",
+        }
+
     def _has_assertion(self, evaluation_case: EvaluationData[InputT, OutputT]) -> bool:
         """Check if the evaluation case contains expected_assertion for assertion mode."""
         return bool(evaluation_case.expected_assertion)

src/strands_evals/evaluators/harmfulness_evaluator.py

@@ -6,6 +6,7 @@
 from strands.models.model import Model
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import EvaluationLevel
 from .evaluator import Evaluator
 from .prompt_templates.harmfulness import get_template
@@ -47,6 +48,19 @@ def __init__(
         self.version = version
         self.model = model
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether the agent's response contains harmful content",
+            "method": {
+                "category": "llm_judge_output",
+                "summary": (
+                    "An LLM judge examines the agent's response for harmful, dangerous, or inappropriate content."
+                ),
+            },
+            "threshold": "score == 1.0 (not harmful)",
+            "tier": "guardrail",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_trace_level_prompt(parsed_input)

src/strands_evals/evaluators/tool_parameter_accuracy_evaluator.py

@@ -6,6 +6,7 @@
 from strands.models.model import Model
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import EvaluationLevel
 from .evaluator import Evaluator
 from .prompt_templates.tool_parameter_accuracy import get_template
@@ -47,6 +48,20 @@ def __init__(
         self.version = version
         self.model = model
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether tool call parameters faithfully use information from the conversation context",
+            "method": {
+                "category": "llm_judge_trajectory",
+                "summary": (
+                    "An LLM judge evaluates each tool call's parameters to verify they "
+                    "accurately reflect information from the preceding conversation and tool results."
+                ),
+            },
+            "threshold": "all tool calls scored Yes",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         tool_inputs = self._parse_trajectory(evaluation_case)
         results = []

src/strands_evals/evaluators/tool_selection_accuracy_evaluator.py

@@ -6,6 +6,7 @@
 from strands.models.model import Model
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
+from ..types.evaluator_metadata import EvaluatorMetadata
 from ..types.trace import EvaluationLevel
 from .evaluator import Evaluator
 from .prompt_templates.tool_selection_accuracy import get_template
@@ -47,6 +48,20 @@ def __init__(
         self.version = version
         self.model = model
 
+    def metadata(self) -> EvaluatorMetadata:
+        return {
+            "checks": "Whether each tool call is justified given the conversation context",
+            "method": {
+                "category": "llm_judge_trajectory",
+                "summary": (
+                    "An LLM judge evaluates each tool call in the trajectory to determine "
+                    "if it was appropriate given the available tools and conversation context."
+                ),
+            },
+            "threshold": "all tool calls scored Yes",
+            "tier": "quality",
+        }
+
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         tool_inputs = self._parse_trajectory(evaluation_case)
         results = []

src/strands_evals/experiment.py

@@ -47,6 +47,7 @@
 from .types.detector import DiagnosisConfig
 from .types.evaluation import EvaluationData, InputT, OutputT
 from .types.evaluation_report import EvaluationReport
+from .types.evaluator_metadata import validate_metadata
 from .types.trace import Session
 from .utils import is_throttling_error
 
@@ -192,6 +193,19 @@ def _validate_evaluator_names(self) -> None:
                 f"multiple instances of the same Evaluator subclass."
             )
 
+    def _validate_evaluator_metadata(self) -> None:
+        """Validate metadata for all evaluators that declare it.
+
+        Iterates over evaluators and calls validate_metadata on any that
+        return non-None from metadata(). Evaluators without metadata are
+        skipped silently.
+
+        Raises:
+            ValueError: If any evaluator's metadata has invalid structure.
+        """
+        for evaluator in self._evaluators:
+            validate_metadata(evaluator.metadata(), evaluator.get_name())
+
     def _validate_case_names(self) -> None:
         """Validate that all cases have unique, non-None names.
 
@@ -627,6 +641,7 @@ async def run_evaluations_async(
             an `evaluator` key naming which evaluator produced it.
         """
         self._validate_evaluator_names()
+        self._validate_evaluator_metadata()
 
         if evaluation_data_store is not None:
             self._validate_case_names()

src/strands_evals/types/__init__.py

@@ -8,13 +8,24 @@
     RCAStructuredOutput,
 )
 from .evaluation import EnvironmentState, EvaluationData, EvaluationOutput, InputT, Interaction, OutputT, TaskOutput
+from .evaluator_metadata import (
+    EvaluatorMetadata,
+    MethodCategory,
+    MethodInfo,
+    Tier,
+    validate_metadata,
+)
 from .multimodal import AnyMediaData, ImageData, MultimodalInput, resolve_image_bytes
 from .simulation import ActorProfile, ActorResponse
 
 __all__ = [
     "EnvironmentState",
+    "EvaluatorMetadata",
     "Interaction",
+    "MethodCategory",
+    "MethodInfo",
     "TaskOutput",
+    "Tier",
     "EvaluationData",
     "EvaluationOutput",
     "ActorProfile",
@@ -32,4 +43,5 @@
     "RCAItem",
     "RCAOutput",
     "RCAStructuredOutput",
+    "validate_metadata",
 ]