From 974e056a4468c90f778136d564561ba6ef8df10c Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Wed, 24 Jun 2026 13:42:19 -0700 Subject: [PATCH 1/8] feat(chaos): add P1 model-output corruption effects Adds model-output corruption to the existing ChaosPlugin via a guarded MessageAddedEvent hook, alongside the P0 tool-chaos hooks. Effects: FormatCorruption (malformed JSON, truncation, schema violation, empty, garbage), Hallucination (confabulation), Refusal (full refusal), plus optional success-framing. The MessageAddedEvent callback is guarded to corrupt only final assistant responses (role==assistant, no toolUse blocks present) so destructive effects can't delete toolUse blocks mid-turn and break the agent loop. Includes 13 tests covering each effect and all guard conditions. --- src/strands_evals/chaos/__init__.py | 16 + src/strands_evals/chaos/model_effects.py | 300 ++++++++++++++++ src/strands_evals/chaos/model_types.py | 72 ++++ src/strands_evals/chaos/model_utils.py | 265 ++++++++++++++ src/strands_evals/chaos/plugin.py | 192 +++++++++- tests/strands_evals/chaos/test_model_chaos.py | 335 ++++++++++++++++++ 6 files changed, 1167 insertions(+), 13 deletions(-) create mode 100644 src/strands_evals/chaos/model_effects.py create mode 100644 src/strands_evals/chaos/model_types.py create mode 100644 src/strands_evals/chaos/model_utils.py create mode 100644 tests/strands_evals/chaos/test_model_chaos.py diff --git a/src/strands_evals/chaos/__init__.py b/src/strands_evals/chaos/__init__.py index 8670012d..97000a24 100644 --- a/src/strands_evals/chaos/__init__.py +++ b/src/strands_evals/chaos/__init__.py @@ -18,6 +18,16 @@ ValidationError, ) from .experiment import ChaosExperiment +from .model_effects import ( + FormatCorruptionEffect, + HallucinationEffect, + RefusalEffect, +) +from .model_types import ( + ModelOutputCorruptionConfig, + ModelOutputCorruptionType, + ModelOutputHallucinationType, +) from .plugin import ChaosPlugin __all__ = [ @@ -38,4 +48,10 @@ "TruncateFields", "RemoveFields", "CorruptValues", + "ModelOutputCorruptionConfig", + "ModelOutputCorruptionType", + "ModelOutputHallucinationType", + "FormatCorruptionEffect", + "HallucinationEffect", + "RefusalEffect", ] diff --git a/src/strands_evals/chaos/model_effects.py b/src/strands_evals/chaos/model_effects.py new file mode 100644 index 00000000..66d5a906 --- /dev/null +++ b/src/strands_evals/chaos/model_effects.py @@ -0,0 +1,300 @@ +"""Model output corruption effect classes. + +Effects are polymorphic — they accept both ``str`` and ``list[dict]`` content. +The caller (ModelChaosPlugin) is responsible for locating the content within +the framework-specific response; these effects only corrupt it. + +Effect hierarchy: + ChaosEffect (from .effects) → ModelEffect → concrete effects +""" + +import logging +import random +import re +from typing import Any + +from .effects import ChaosEffect +from .model_types import ModelOutputCorruptionConfig, ModelOutputCorruptionType, ModelOutputHallucinationType +from .model_utils import ( + _CONFABULATION_TEMPLATES, + _CONTRADICTION_PHRASES, + _DATE_PATTERNS, + _LOCATION_SUBSTITUTIONS, + _NUMBER_PATTERNS, + _PRICE_PATTERNS, + _REFUSAL_TEMPLATES, + _TOOL_NARRATIVES, + _map_text_in_blocks, +) + +logger = logging.getLogger(__name__) + +_GARBAGE_STRINGS = [ + "\x00\xff\xfe GARBAGE DATA \x00\x01", + "asdkjh2398yr2h3f CORRUPTED asjkdhf98", + "NOT_A_VALID_RESPONSE", + "NaN undefined null [object Object]", +] + +# Set of hallucination types for dispatch +HALLUCINATION_TYPES = set(ModelOutputHallucinationType) + + +class FormatCorruptionEffect(ChaosEffect): + """Corrupts model output content format. + + Supports: EMPTY_RESPONSE, TRUNCATED_RESPONSE, MALFORMED_JSON, + SCHEMA_VIOLATION, GARBAGE_OUTPUT. + """ + + hook = "post" + effect_type: str = "format_corruption" + + def __init__(self, config: ModelOutputCorruptionConfig): + super().__init__() + self._config = config + + def apply(self, content: Any = None) -> Any: + """Corrupt *content* and return the same shape.""" + if content is None: + raise ValueError("FormatCorruptionEffect.apply() requires content") + + ct = self._config.corruption_type + + if isinstance(content, str): + return self._apply_to_str(content, ct) + elif isinstance(content, list): + return self._apply_to_blocks(content, ct) + else: + raise ValueError( + f"FormatCorruptionEffect.apply() received unsupported content type " + f"{type(content).__name__}; expected str or list[dict]." + ) + + def _truncate(self, text: str) -> str: + if not text: + return text + end = max(1, round(len(text) * (1 - self._config.truncate_ratio))) + return text[:end] + + def _apply_to_str(self, text: str, ct: Any) -> str: + match ct: + case ModelOutputCorruptionType.EMPTY_RESPONSE: + return "" + case ModelOutputCorruptionType.TRUNCATED_RESPONSE: + return self._truncate(text) + case ModelOutputCorruptionType.GARBAGE_OUTPUT: + return random.choice(_GARBAGE_STRINGS) + case ModelOutputCorruptionType.MALFORMED_JSON: + return self._malform_text(text) + case _: + return text + + def _apply_to_blocks(self, blocks: list, ct: Any) -> list: + match ct: + case ModelOutputCorruptionType.EMPTY_RESPONSE: + return [] + case ModelOutputCorruptionType.TRUNCATED_RESPONSE: + return _map_text_in_blocks(blocks, self._truncate) + case ModelOutputCorruptionType.GARBAGE_OUTPUT: + return _map_text_in_blocks(blocks, lambda _: random.choice(_GARBAGE_STRINGS)) + case ModelOutputCorruptionType.MALFORMED_JSON: + return self._malform_json(blocks) + case ModelOutputCorruptionType.SCHEMA_VIOLATION: + return self._violate_schema(blocks) + case _: + return list(blocks) + + @staticmethod + def _malform_text(text: str) -> str: + stripped = text.strip() + if stripped.startswith("{") or stripped.startswith("["): + return stripped[: len(stripped) // 2] + return text + + @staticmethod + def _malform_json(blocks: list) -> list: + import json as json_mod + + def corrupt_tool_use(tool_use: dict) -> dict: + raw = json_mod.dumps(tool_use.get("input", {})) + tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" + return tool_use + + result = [] + for block in blocks: + block = dict(block) + if "toolUse" in block: + block["toolUse"] = corrupt_tool_use(dict(block["toolUse"])) + elif "text" in block and isinstance(block["text"], str): + block["text"] = FormatCorruptionEffect._malform_text(block["text"]) + result.append(block) + return result + + @staticmethod + def _violate_schema(blocks: list) -> list: + type_swaps: list[Any] = [None, 99999, True, [], "WRONG_TYPE"] + + def corrupt_tool_use(tool_use: dict) -> dict: + inp = tool_use.get("input", {}) + if isinstance(inp, dict) and inp: + corrupted = {} + for k, v in inp.items(): + candidates = [s for s in type_swaps if not isinstance(v, type(s))] + corrupted[k] = random.choice(candidates) if candidates else "WRONG_TYPE" + tool_use["input"] = corrupted + return tool_use + + result = [] + for block in blocks: + block = dict(block) + if "toolUse" in block: + block["toolUse"] = corrupt_tool_use(dict(block["toolUse"])) + result.append(block) + return result + + +class HallucinationEffect(ChaosEffect): + """Corrupts model output text with hallucination mutations. + + Supports: FACTUAL_ERROR, CONFABULATION, CONTEXT_UNFAITHFULNESS, + TOOL_CLAIM_FABRICATION. + """ + + hook = "post" + effect_type: str = "hallucination" + + def __init__(self, config: ModelOutputCorruptionConfig) -> None: + super().__init__() + self._config = config + + def apply(self, content: Any = None) -> Any: + """Mutate *content* and return the same shape.""" + if content is None: + raise ValueError("HallucinationEffect.apply() requires content") + + ct = self._config.corruption_type + + if isinstance(content, str): + return self._apply_to_str(content, ct) + elif isinstance(content, list): + return self._apply_to_blocks(content, ct) + else: + raise ValueError( + f"HallucinationEffect.apply() received unsupported content type " + f"{type(content).__name__}; expected str or list[dict]." + ) + + def _apply_to_str(self, text: str, ct: Any) -> str: + match ct: + case ModelOutputHallucinationType.FACTUAL_ERROR: + return self._factual_error(text) + case ModelOutputHallucinationType.CONFABULATION: + return self._confabulation(text) + case ModelOutputHallucinationType.CONTEXT_UNFAITHFULNESS: + return self._context_unfaithfulness(text) + case ModelOutputHallucinationType.TOOL_CLAIM_FABRICATION: + return self._tool_claim_fabrication(text) + case _: + return text + + def _apply_to_blocks(self, blocks: list, ct: Any) -> list: + match ct: + case ModelOutputHallucinationType.FACTUAL_ERROR: + return _map_text_in_blocks(blocks, self._factual_error) + case ModelOutputHallucinationType.CONFABULATION: + return _map_text_in_blocks(blocks, self._confabulation) + case ModelOutputHallucinationType.CONTEXT_UNFAITHFULNESS: + return _map_text_in_blocks(blocks, self._context_unfaithfulness) + case ModelOutputHallucinationType.TOOL_CLAIM_FABRICATION: + return _map_text_in_blocks(blocks, self._tool_claim_fabrication) + case _: + return list(blocks) + + def _factual_error(self, text: str) -> str: + target_types = self._config.target_entity_types + result = text + + entity_patterns: list[tuple] = [] + if target_types is None or "date" in target_types: + entity_patterns.extend(_DATE_PATTERNS) + if target_types is None or "number" in target_types: + entity_patterns.extend(_NUMBER_PATTERNS) + if target_types is None or "price" in target_types: + entity_patterns.extend(_PRICE_PATTERNS) + + for pattern, replacer in entity_patterns: + try: + result = re.sub(pattern, replacer, result) + except Exception: + pass + + if target_types is None or "location" in target_types: + for old_loc, new_loc in _LOCATION_SUBSTITUTIONS.items(): + result = result.replace(old_loc, new_loc) + + return result + + def _confabulation(self, text: str) -> str: + if not text: + return text + template = random.choice(_CONFABULATION_TEMPLATES) + sentences = re.split(r"(?<=[.!?])\s+", text) + if len(sentences) <= 1: + return template + text + insert_pos = random.randint(1, len(sentences) - 1) + sentences.insert(insert_pos, template) + return " ".join(sentences) + + def _context_unfaithfulness(self, text: str) -> str: + if not text: + return text + phrase = random.choice(_CONTRADICTION_PHRASES) + sentences = re.split(r"(?<=[.!?])\s+", text) + if len(sentences) <= 1: + return text + phrase + insert_pos = random.randint(1, len(sentences) - 1) + sentences.insert(insert_pos, phrase.strip()) + return " ".join(sentences) + + def _tool_claim_fabrication(self, text: str) -> str: + if not text: + return text + narrative = random.choice(_TOOL_NARRATIVES) + sentences = re.split(r"(?<=[.!?])\s+", text) + if len(sentences) <= 1: + return text + " " + narrative + insert_pos = random.randint(1, len(sentences) - 1) + sentences.insert(insert_pos, narrative.strip()) + return " ".join(sentences) + + +class RefusalEffect(ChaosEffect): + """Replaces model output content with a refusal message. + + Supports: FULL_REFUSAL only. + """ + + hook = "post" + effect_type: str = "refusal" + + def __init__(self, config: ModelOutputCorruptionConfig) -> None: + super().__init__() + self._config = config + + def apply(self, content: Any = None) -> Any: + """Replace *content* with a refusal message.""" + if content is None: + raise ValueError("RefusalEffect.apply() requires content") + + template = random.choice(_REFUSAL_TEMPLATES) + + if isinstance(content, str): + return template + elif isinstance(content, list): + return [{"text": template}] + else: + raise ValueError( + f"RefusalEffect.apply() received unsupported content type " + f"{type(content).__name__}; expected str or list[dict]." + ) diff --git a/src/strands_evals/chaos/model_types.py b/src/strands_evals/chaos/model_types.py new file mode 100644 index 00000000..a35b6a7c --- /dev/null +++ b/src/strands_evals/chaos/model_types.py @@ -0,0 +1,72 @@ +"""Model output corruption types and configuration. + +Defines the enums and Pydantic config model used to parameterize model +output corruption effects. +""" + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field + + +class ModelOutputCorruptionType(str, Enum): + """Output corruption types for model response format mutation.""" + + MALFORMED_JSON = "malformed_json" + TRUNCATED_RESPONSE = "truncated_response" + SCHEMA_VIOLATION = "schema_violation" + EMPTY_RESPONSE = "empty_response" + GARBAGE_OUTPUT = "garbage_output" + FULL_REFUSAL = "full_refusal" + TOXIC_CONTENT = "toxic_content" + + +class ModelOutputHallucinationType(str, Enum): + """Hallucination types for model response semantic mutation. + + Separated from ModelOutputCorruptionType to allow future LLM-based + hallucination generation strategies. + """ + + FACTUAL_ERROR = "factual_error" + CONFABULATION = "confabulation" + CONTEXT_UNFAITHFULNESS = "context_unfaithfulness" + TOOL_CLAIM_FABRICATION = "tool_claim_fabrication" + + +class ModelOutputCorruptionConfig(BaseModel): + """Configuration for model output corruption. + + Mutates model responses after generation to test agent resilience + against malformed, truncated, or otherwise corrupted LLM output. + + Attributes: + apply_rate: Probability (0.0 to 1.0) that corruption is applied per call. + 0.0 disables output corruption. + corruption_type: Type of output corruption to apply. + truncate_ratio: Fraction of content to corrupt for TRUNCATED_RESPONSE. + target_entity_types: Entity types for FACTUAL_ERROR filtering. + add_success_framing: Prepend a confident success prefix after corruption. + """ + + apply_rate: float = Field(default=0.0, ge=0.0, le=1.0, description="Probability of corruption per model call") + corruption_type: ModelOutputCorruptionType | ModelOutputHallucinationType = Field( + default=ModelOutputCorruptionType.MALFORMED_JSON, + description="Type of output corruption or hallucination to apply", + ) + truncate_ratio: float = Field( + default=0.5, + ge=0.0, + le=1.0, + description="Fraction of content to corrupt for TRUNCATED_RESPONSE", + ) + target_entity_types: Optional[list[str]] = Field( + default=None, + description="Entity types for FACTUAL_ERROR: 'date', 'number', 'location', 'price'. None = all.", + ) + add_success_framing: bool = Field( + default=False, + description="Prepend a confident success prefix before the corrupted content. " + "Composable with any corruption_type or hallucination_type.", + ) diff --git a/src/strands_evals/chaos/model_utils.py b/src/strands_evals/chaos/model_utils.py new file mode 100644 index 00000000..4c6cfdf8 --- /dev/null +++ b/src/strands_evals/chaos/model_utils.py @@ -0,0 +1,265 @@ +"""Shared model corruption helper functions and constants. + +Used by model output corruption effects for factual-error perturbation, +confabulation, context-unfaithfulness, tool-claim fabrication, refusal +templates, and success framing prefixes. +""" + +import random +import re +from datetime import datetime, timedelta +from typing import Any + +# --------------------------------------------------------------------------- +# Shared helper — module-level +# --------------------------------------------------------------------------- + + +def _map_text_in_blocks(blocks: list, fn: Any) -> list: + """Apply *fn* to every ``"text"`` value; leave other blocks untouched.""" + result = [] + for block in blocks: + block = dict(block) + if "text" in block and isinstance(block["text"], str): + block["text"] = fn(block["text"]) + result.append(block) + return result + + +# --------------------------------------------------------------------------- +# FACTUAL_ERROR: Perturbation helpers +# --------------------------------------------------------------------------- + +_MONTH_NAMES = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +] + + +def _shift_date_named(m: re.Match) -> str: + """Shift a named date like 'January 15, 2023' by ±1-3 months and ±1-2 years.""" + text = m.group(0) + for i, month in enumerate(_MONTH_NAMES): + if month in text: + new_month_idx = (i + random.randint(1, 3)) % 12 + new_month = _MONTH_NAMES[new_month_idx] + text = text.replace(month, new_month) + break + year_match = re.search(r"\d{4}", text) + if year_match: + old_year = int(year_match.group()) + new_year = old_year + random.choice([-2, -1, 1, 2]) + text = text.replace(str(old_year), str(new_year)) + return text + + +def _shift_date_slash(m: re.Match) -> str: + """Shift a slash date like '12/25/2023' by ±1-5 days and ±1-3 months.""" + text = m.group(0) + parts = text.split("/") + if len(parts) >= 2: + month = int(parts[0]) + day = int(parts[1]) + new_month = max(1, min(12, month + random.randint(-3, 3))) + new_day = max(1, min(28, day + random.randint(-5, 5))) + parts[0] = str(new_month) + parts[1] = str(new_day) + return "/".join(parts) + + +def _shift_date_iso(m: re.Match) -> str: + """Shift an ISO date like '2023-01-15' by ±1-10 days.""" + text = m.group(0) + try: + dt = datetime.strptime(text, "%Y-%m-%d") + delta = timedelta(days=random.randint(1, 10) * random.choice([-1, 1])) + new_dt = dt + delta + return new_dt.strftime("%Y-%m-%d") + except ValueError: + return text + + +def _perturb_number(m: re.Match) -> str: + """Perturb a comma-formatted number like '1,234,567' by factor [0.5, 2.0].""" + text = m.group(1) if m.lastindex else m.group(0) + try: + num = int(text.replace(",", "")) + factor = random.uniform(0.5, 2.0) + new_num = int(num * factor) + if new_num == num: + new_num = num + random.choice([-1, 1]) + return f"{new_num:,}" + except ValueError: + return text + + +def _perturb_float(m: re.Match) -> str: + """Perturb a float like '42.5' by ±20%.""" + text = m.group(1) if m.lastindex else m.group(0) + try: + num = float(text) + offset = num * random.uniform(-0.2, 0.2) + if abs(offset) < 0.01: + offset = random.choice([-0.1, 0.1]) + new_num = num + offset + decimal_places = len(text.split(".")[1]) if "." in text else 1 + return f"{new_num:.{decimal_places}f}" + except ValueError: + return text + + +def _perturb_integer(m: re.Match) -> str: + """Perturb an integer like '100' by ±30%.""" + text = m.group(1) if m.lastindex else m.group(0) + try: + num = int(text) + offset = int(num * random.uniform(-0.3, 0.3)) + if offset == 0: + offset = random.choice([-1, 1]) + return str(num + offset) + except ValueError: + return text + + +def _perturb_price(m: re.Match) -> str: + """Perturb a price like '$1,234.56' by factor [0.7, 1.5], preserving currency symbol.""" + currency = m.group(1) + amount_str = m.group(2) + try: + amount = float(amount_str.replace(",", "")) + factor = random.uniform(0.7, 1.5) + new_amount = amount * factor + if abs(new_amount - amount) < 0.01: + new_amount = amount + random.choice([-1.0, 1.0]) + if "." in amount_str: + formatted = f"{new_amount:,.2f}" + else: + formatted = f"{int(new_amount):,}" + return f"{currency}{formatted}" + except ValueError: + return m.group(0) + + +# --------------------------------------------------------------------------- +# FACTUAL_ERROR: Regex patterns +# --------------------------------------------------------------------------- + +_DATE_PATTERNS = [ + ( + r"\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}\b", + lambda m: _shift_date_named(m), + ), + (r"\b\d{1,2}/\d{1,2}/\d{2,4}\b", lambda m: _shift_date_slash(m)), + (r"\b\d{4}-\d{2}-\d{2}\b", lambda m: _shift_date_iso(m)), +] + +_NUMBER_PATTERNS = [ + (r"\b(\d{1,3}(?:,\d{3})+)\b", lambda m: _perturb_number(m)), + (r"\b(\d+\.\d+)\b", lambda m: _perturb_float(m)), + (r"\b(\d{2,})\b", lambda m: _perturb_integer(m)), +] + +_LOCATION_SUBSTITUTIONS = { + "New York": "Chicago", + "Chicago": "Houston", + "London": "Manchester", + "Paris": "Lyon", + "Tokyo": "Osaka", + "Berlin": "Munich", + "Sydney": "Melbourne", + "San Francisco": "Seattle", + "Seattle": "Portland", + "Los Angeles": "San Diego", + "Beijing": "Shanghai", + "Mumbai": "Delhi", + "Toronto": "Vancouver", + "United States": "Canada", + "United Kingdom": "Australia", +} + +_PRICE_PATTERNS = [ + (r"([$€£¥])\s*(\d{1,3}(?:,\d{3})*(?:\.\d{2})?)", lambda m: _perturb_price(m)), +] + + +# --------------------------------------------------------------------------- +# CONFABULATION: Template phrases +# --------------------------------------------------------------------------- + +_CONFABULATION_TEMPLATES = [ + "According to a 2019 study published in Nature, ", + "Research conducted by MIT in 2021 demonstrated that ", + "A comprehensive report by the World Health Organization found that ", + "Based on data from the U.S. Census Bureau (2020), ", + "As documented in the IEEE Transactions on Software Engineering, ", + "A peer-reviewed analysis by Stanford researchers confirmed that ", + "The European Commission's 2022 technical report states that ", + "Historical records from the Smithsonian Institution indicate that ", +] + + +# --------------------------------------------------------------------------- +# CONTEXT_UNFAITHFULNESS: Contradiction phrases +# --------------------------------------------------------------------------- + +_CONTRADICTION_PHRASES = [ + " However, this is actually incorrect. ", + " On the contrary, the opposite is true. ", + " In fact, recent evidence contradicts this entirely. ", + " Note: the above statement has been disputed by multiple sources. ", + " Actually, this information was later retracted. ", + " Importantly, this conflicts with the data presented earlier. ", + " Correction: the previous claim is not supported by evidence. ", +] + + +# --------------------------------------------------------------------------- +# TOOL_CLAIM_FABRICATION: Fake tool-call narratives +# --------------------------------------------------------------------------- + +_TOOL_NARRATIVES = [ + "I called the database_lookup tool and it returned the following results: ", + "After running the web_search tool, I found that ", + "The file_reader tool confirmed that the document contains ", + "I used the calculator tool to verify, and the result is ", + "The api_query tool returned a 200 OK response indicating ", +] + + +# --------------------------------------------------------------------------- +# FULL_REFUSAL: Refusal message templates +# --------------------------------------------------------------------------- + +_REFUSAL_TEMPLATES = [ + "I'm sorry, but I can't help with that request.", + "I'm not able to help with this particular request. Could you try rephrasing?", + "I can't assist with that. Please try a different approach.", + "I'm unable to process this request at this time.", + "This is beyond what I'm able to do. Please consider an alternative approach.", + "I can't assist with the request as described. Could you provide more context?", +] + + +# --------------------------------------------------------------------------- +# SUCCESS FRAMING: Success prefix templates +# --------------------------------------------------------------------------- + +_SUCCESS_PREFIXES = [ + "Successfully completed the requested operation.", + "Done! Here are the results you asked for.", + "Great news — everything worked as expected.", + "Operation finished successfully. Here's what I found:", + "All done! The task has been completed without issues.", + "I've successfully processed your request. Here's the output:", + "Task completed. Below are the verified results:", +] diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index aa4b326f..9c9dfe08 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -1,20 +1,45 @@ """Chaos Plugin for Strands Agents. Implements chaos injection as a standard Strands Plugin using the SDK's -native hook system (BeforeToolCallEvent / AfterToolCallEvent). +native hook system. Handles BOTH tool-level and model-output chaos: -The plugin reads the active ChaosCase from a module-level ContextVar at hook -time. The ChaosExperiment manages the ContextVar lifecycle. +- BeforeToolCallEvent: cancels tool calls for pre-hook effects (Timeout, etc.) +- AfterToolCallEvent: corrupts tool responses for post-hook effects (TruncateFields, etc.) +- MessageAddedEvent: corrupts model output for the final assistant response + +Model output corruption uses dict mutation on event.message["content"]. This is +necessary because AfterModelCallEvent.stop_response is read-only (only `retry` +is writeable). Dict mutation bypasses _can_write (which only intercepts __setattr__). + +The model-chaos callback is GUARDED to only corrupt the FINAL agent response: +- role == "assistant" +- message content contains NO toolUse blocks + +This guard prevents destructive effects (EMPTY_RESPONSE, FULL_REFUSAL) from +deleting toolUse blocks on mid-turn tool_use messages, which would break the +agent loop. MessageAddedEvent fires BEFORE tool dispatch (event_loop.py L427-428 +fires the event; L187 branches on stop_reason; L476 extracts toolUse from the +same message object). """ import json import logging +import random +from typing import Any -from strands.hooks import AfterToolCallEvent, BeforeToolCallEvent +from strands.hooks import AfterToolCallEvent, BeforeToolCallEvent, MessageAddedEvent from strands.plugins import Plugin, hook from ._context import _current_chaos_case from .effects import ChaosEffect, TruncateFields +from .model_effects import ( + HALLUCINATION_TYPES, + FormatCorruptionEffect, + HallucinationEffect, + RefusalEffect, +) +from .model_types import ModelOutputCorruptionConfig, ModelOutputCorruptionType +from .model_utils import _SUCCESS_PREFIXES logger = logging.getLogger(__name__) @@ -22,33 +47,72 @@ class ChaosPlugin(Plugin): """Strands Plugin that injects deterministic chaos based on the active ChaosCase. - The plugin intercepts tool calls via Strands' native hook system: - - BeforeToolCallEvent: cancels tool calls for pre-hook effects (Timeout, NetworkError, etc.) - - AfterToolCallEvent: corrupts tool responses for post-hook effects (TruncateFields, etc.) + Handles both tool-level chaos (P0) and model-output chaos (P1): + + Tool chaos (P0): + - BeforeToolCallEvent: cancels tool calls for pre-hook effects + - AfterToolCallEvent: corrupts tool responses for post-hook effects + + Model output chaos (P1): + - MessageAddedEvent: corrupts the final assistant response content The active ChaosCase is managed via a ContextVar (set by ChaosExperiment). - When no ChaosCase is active or the case has no effects, all tools behave normally. + When no ChaosCase is active or the case has no effects, all tools and model + output behave normally. + + Model output corruption is configured via `model_output_config` on the + ChaosPlugin instance or via the ChaosCase effects dict (key: "model_effects"). Example:: from strands import Agent from strands_evals.chaos import ChaosPlugin + from strands_evals.chaos.model_types import ( + ModelOutputCorruptionConfig, + ModelOutputHallucinationType, + ) + + chaos = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputHallucinationType.CONFABULATION, + add_success_framing=True, + ) + ) - chaos = ChaosPlugin() agent = Agent( model=my_model, tools=[search_tool, database_tool], plugins=[chaos], ) - - # The ChaosExperiment handles ChaosCase activation via ContextVar. - # The user's task body contains zero chaos concepts. """ name = "chaos-testing" - def __init__(self) -> None: + def __init__(self, model_output_config: ModelOutputCorruptionConfig | None = None) -> None: + """Initialize the ChaosPlugin. + + Args: + model_output_config: Optional configuration for model output corruption. + When provided, enables model-output chaos on final assistant responses. + When None, only tool-level chaos (from ChaosCase effects) is active. + """ super().__init__() + self._model_output_config = model_output_config + + @property + def model_output_config(self) -> ModelOutputCorruptionConfig | None: + """The active model output corruption configuration.""" + return self._model_output_config + + @model_output_config.setter + def model_output_config(self, value: ModelOutputCorruptionConfig | None) -> None: + """Update the model output corruption configuration.""" + self._model_output_config = value + + # ------------------------------------------------------------------ + # Tool chaos hooks (P0) — from PR #224 + # ------------------------------------------------------------------ @hook # type: ignore[call-overload] def before_tool_call(self, event: BeforeToolCallEvent) -> None: @@ -106,6 +170,108 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: logger.info("effect=<%s>, tool=<%s> | applied chaos post-hook", type(effect).__name__, tool_name) + # ------------------------------------------------------------------ + # Model output chaos hook (P1) + # ------------------------------------------------------------------ + + @hook # type: ignore[call-overload] + def message_added(self, event: MessageAddedEvent) -> None: + """Intercept messages to corrupt the final assistant response. + + GUARD: corruption is applied ONLY when ALL conditions hold: + 1. message role == "assistant" + 2. message content contains NO toolUse blocks + + This prevents destructive effects from breaking mid-turn tool dispatch. + MessageAddedEvent fires BEFORE the agent extracts toolUse blocks for + execution, so corrupting a tool_use message would break the agent loop. + + NOTE: stop_reason is NOT available on MessageAddedEvent (the event only + carries `message: Message`). We use the toolUse-presence check as a proxy: + messages with toolUse blocks are mid-turn tool_use messages; messages + without are final end_turn responses. This is reliable because end_turn + messages never contain toolUse blocks. + """ + if self._model_output_config is None: + return + + if self._model_output_config.apply_rate <= 0: + return + + message = event.message + + # Guard 1: only assistant messages + if message.get("role") != "assistant": + return + + # Guard 2: skip messages with toolUse blocks (mid-turn tool dispatch) + content = message.get("content") + if content is None: + return + + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and "toolUse" in block: + return + + # Guard 3: apply_rate probabilistic check + if random.random() >= self._model_output_config.apply_rate: + return + + # Dispatch to effect + corrupted = self._apply_model_corruption(content) + + # Apply success framing if content was mutated + if self._model_output_config.add_success_framing and corrupted != content: + corrupted = self._apply_success_framing(corrupted) + + # Mutate via dict assignment (NOT attribute assignment on the event) + message["content"] = corrupted + + logger.info( + "corruption_type=<%s> | applied model output chaos to assistant message", + self._model_output_config.corruption_type.value, + ) + + # ------------------------------------------------------------------ + # Model corruption helpers + # ------------------------------------------------------------------ + + def _apply_model_corruption(self, content: Any) -> Any: + """Dispatch to the appropriate model effect and apply corruption.""" + config = self._model_output_config + assert config is not None + ct = config.corruption_type + + effect: ChaosEffect + if ct in HALLUCINATION_TYPES: + effect = HallucinationEffect(config) + elif ct == ModelOutputCorruptionType.FULL_REFUSAL: + effect = RefusalEffect(config) + else: + effect = FormatCorruptionEffect(config) + + return effect.apply(content) + + @staticmethod + def _apply_success_framing(content: Any) -> Any: + """Prepend a success prefix to corrupted content.""" + prefix = random.choice(_SUCCESS_PREFIXES) + + if isinstance(content, str): + return prefix + " " + content + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and "text" in block and isinstance(block["text"], str): + block["text"] = prefix + " " + block["text"] + return content + return [{"text": prefix}] + content + return content + + # ------------------------------------------------------------------ + # Tool corruption helpers (P0) + # ------------------------------------------------------------------ + def _apply_to_blocks(self, effect: ChaosEffect, blocks: list) -> list: """Apply effect to text blocks in a content list.""" corrupted_blocks = [] diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py new file mode 100644 index 00000000..4467741a --- /dev/null +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -0,0 +1,335 @@ +"""Unit tests for model output chaos via ChaosPlugin MessageAddedEvent callback. + +Tests cover: +- Per-effect corruption on final assistant messages (end_turn, no toolUse) +- Guard: toolUse-carrying messages are NOT corrupted +- Guard: user/tool messages are NOT corrupted +- Guard: passthrough when no config set +- structured_output_model path: messages with toolUse are skipped (deferred scope) +""" + +import copy +from unittest.mock import MagicMock + +from strands_evals.chaos.model_types import ( + ModelOutputCorruptionConfig, + ModelOutputCorruptionType, + ModelOutputHallucinationType, +) +from strands_evals.chaos.model_utils import ( + _REFUSAL_TEMPLATES, + _SUCCESS_PREFIXES, +) +from strands_evals.chaos.plugin import ChaosPlugin + + +def _make_event(message: dict) -> MagicMock: + """Create a mock MessageAddedEvent with the given message. + + The event's `message` attribute is a real dict (not a Mock) so that + dict mutation works as in production. + """ + event = MagicMock() + event.message = message + return event + + +def _final_assistant_message(text: str = "The answer is 42.") -> dict: + """An end_turn assistant message with text content only (no toolUse).""" + return { + "role": "assistant", + "content": [{"text": text}], + } + + +def _tooluse_assistant_message() -> dict: + """A tool_use assistant message containing a toolUse block.""" + return { + "role": "assistant", + "content": [ + {"text": "Let me search for that."}, + {"toolUse": {"toolUseId": "tu_1", "name": "search", "input": {"query": "test"}}}, + ], + } + + +def _user_message() -> dict: + """A user message.""" + return { + "role": "user", + "content": [{"text": "Hello, what is 2+2?"}], + } + + +def _tool_result_message() -> dict: + """A tool result message.""" + return { + "role": "user", + "content": [{"toolResult": {"toolUseId": "tu_1", "status": "success", "content": [{"text": "4"}]}}], + } + + +# --------------------------------------------------------------------------- +# Per-effect corruption tests on final end_turn assistant messages +# --------------------------------------------------------------------------- + + +class TestModelChaosFormatCorruptionMalformedJson: + """MALFORMED_JSON effect corrupts the final assistant message.""" + + def test_malformed_json_corrupts_json_text(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.MALFORMED_JSON, + ) + ) + message = _final_assistant_message('{"key": "value", "nested": {"a": 1}}') + event = _make_event(message) + + plugin.message_added(event) + + # Content should be corrupted (JSON truncated) + result_text = message["content"][0]["text"] + assert result_text != '{"key": "value", "nested": {"a": 1}}' + + +class TestModelChaosFormatCorruptionEmptyResponse: + """EMPTY_RESPONSE effect empties the final assistant message content.""" + + def test_empty_response_on_final_message(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + message = _final_assistant_message("Hello world") + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == [] + + +class TestModelChaosHallucination: + """Hallucination effect corrupts the final assistant message text.""" + + def test_confabulation_injects_template(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputHallucinationType.CONFABULATION, + ) + ) + original_text = "The weather is sunny. It is warm outside. Birds are singing." + message = _final_assistant_message(original_text) + event = _make_event(message) + + plugin.message_added(event) + + result_text = message["content"][0]["text"] + assert result_text != original_text + # Should contain original text fragments + assert "sunny" in result_text or "warm" in result_text + + +class TestModelChaosRefusal: + """FULL_REFUSAL replaces the final assistant message content with a refusal.""" + + def test_refusal_replaces_content(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.FULL_REFUSAL, + ) + ) + message = _final_assistant_message("Here is the code you requested...") + event = _make_event(message) + + plugin.message_added(event) + + # Content should be a single refusal text block + assert len(message["content"]) == 1 + assert message["content"][0]["text"] in _REFUSAL_TEMPLATES + + +class TestModelChaosSuccessFraming: + """Success framing prepends a confident prefix after corruption.""" + + def test_success_framing_with_refusal(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.FULL_REFUSAL, + add_success_framing=True, + ) + ) + message = _final_assistant_message("Here is the code you requested...") + event = _make_event(message) + + plugin.message_added(event) + + result_text = message["content"][0]["text"] + assert any(result_text.startswith(prefix) for prefix in _SUCCESS_PREFIXES) + + +# --------------------------------------------------------------------------- +# Guard tests +# --------------------------------------------------------------------------- + + +class TestModelChaosGuardToolUseMessage: + """Messages with toolUse blocks are NOT corrupted (guard skips them).""" + + def test_empty_response_on_tooluse_message_not_corrupted(self): + """EMPTY_RESPONSE on a tool_use message passes through; toolUse intact.""" + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + message = _tooluse_assistant_message() + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + # Content should be UNCHANGED — guard skipped corruption + assert message["content"] == original_content + # toolUse blocks should be intact + tool_blocks = [b for b in message["content"] if "toolUse" in b] + assert len(tool_blocks) == 1 + assert tool_blocks[0]["toolUse"]["name"] == "search" + + def test_full_refusal_on_tooluse_message_not_corrupted(self): + """FULL_REFUSAL on a tool_use message passes through; toolUse intact.""" + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.FULL_REFUSAL, + ) + ) + message = _tooluse_assistant_message() + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == original_content + + +class TestModelChaosGuardStructuredOutputPath: + """structured_output_model path fires MessageAddedEvent with toolUse — guard skips it.""" + + def test_structured_output_tool_message_not_corrupted(self): + """A message containing the structured output tool call is NOT corrupted.""" + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + # Simulate the structured output tool invocation message + message = { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "so_1", + "name": "structured_output__MyModel", + "input": {"field1": "value1"}, + } + }, + ], + } + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + # Guard should skip — toolUse block present + assert message["content"] == original_content + + +class TestModelChaosGuardFinalMessage: + """Final end_turn assistant message IS corrupted.""" + + def test_final_end_turn_message_corrupted(self): + """An end_turn message with text only (no toolUse) gets corrupted.""" + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + message = _final_assistant_message("Hello world") + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == [] + + +class TestModelChaosGuardRoleFiltering: + """User and tool result messages are NOT corrupted.""" + + def test_user_message_not_corrupted(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + message = _user_message() + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == original_content + + def test_tool_result_message_not_corrupted(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=1.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + message = _tool_result_message() + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == original_content + + +class TestModelChaosPassthrough: + """No corruption when no model_output_config is set.""" + + def test_no_config_passes_through(self): + plugin = ChaosPlugin() # No model_output_config + message = _final_assistant_message("Hello world") + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == original_content + + def test_zero_apply_rate_passes_through(self): + plugin = ChaosPlugin( + model_output_config=ModelOutputCorruptionConfig( + apply_rate=0.0, + corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, + ) + ) + message = _final_assistant_message("Hello world") + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) + + plugin.message_added(event) + + assert message["content"] == original_content From afe7dc0a317e1a41e16dbb280839be0b93bc1920 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Thu, 25 Jun 2026 17:39:02 -0700 Subject: [PATCH 2/8] refactor(chaos): align P1 model effects with #224 conventions Folded model effects into effects.py with ModelEffect base parallel to ToolEffect Replaced ModelOutputCorruptionType enum with class-per-effect + ModelEffectUnion Removed apply_rate; chaos always applies when configured Dropped speculative untested effects (TOXIC_CONTENT, truncation, schema violation, etc.) Moved perturbation data onto owning effect classes Deleted model_effects.py / model_types.py / model_utils.py --- src/strands_evals/chaos/__init__.py | 30 +- src/strands_evals/chaos/effects.py | 218 +++++++++++++ src/strands_evals/chaos/model_effects.py | 300 ------------------ src/strands_evals/chaos/model_types.py | 72 ----- src/strands_evals/chaos/model_utils.py | 265 ---------------- src/strands_evals/chaos/plugin.py | 144 ++++----- tests/strands_evals/chaos/test_model_chaos.py | 121 ++----- 7 files changed, 307 insertions(+), 843 deletions(-) delete mode 100644 src/strands_evals/chaos/model_effects.py delete mode 100644 src/strands_evals/chaos/model_types.py delete mode 100644 src/strands_evals/chaos/model_utils.py diff --git a/src/strands_evals/chaos/__init__.py b/src/strands_evals/chaos/__init__.py index 97000a24..156e3f55 100644 --- a/src/strands_evals/chaos/__init__.py +++ b/src/strands_evals/chaos/__init__.py @@ -7,10 +7,17 @@ from .case import ChaosCase from .effects import ( ChaosEffect, + Confabulation, CorruptValues, + EmptyResponse, ExecutionError, + FullRefusal, + MalformedJson, + ModelEffect, + ModelEffectUnion, NetworkError, RemoveFields, + SuccessFraming, Timeout, ToolEffect, ToolEffectUnion, @@ -18,16 +25,6 @@ ValidationError, ) from .experiment import ChaosExperiment -from .model_effects import ( - FormatCorruptionEffect, - HallucinationEffect, - RefusalEffect, -) -from .model_types import ( - ModelOutputCorruptionConfig, - ModelOutputCorruptionType, - ModelOutputHallucinationType, -) from .plugin import ChaosPlugin __all__ = [ @@ -48,10 +45,11 @@ "TruncateFields", "RemoveFields", "CorruptValues", - "ModelOutputCorruptionConfig", - "ModelOutputCorruptionType", - "ModelOutputHallucinationType", - "FormatCorruptionEffect", - "HallucinationEffect", - "RefusalEffect", + "ModelEffect", + "ModelEffectUnion", + "MalformedJson", + "EmptyResponse", + "Confabulation", + "FullRefusal", + "SuccessFraming", ] diff --git a/src/strands_evals/chaos/effects.py b/src/strands_evals/chaos/effects.py index 7a22dd1b..9fc5c44b 100644 --- a/src/strands_evals/chaos/effects.py +++ b/src/strands_evals/chaos/effects.py @@ -11,8 +11,10 @@ discriminated-union serialization, ensuring full round-trip fidelity. """ +import json import math import random +import re from abc import abstractmethod from typing import Annotated, Any, ClassVar, Literal, Union @@ -329,3 +331,219 @@ def apply(self, response: Any = None) -> Any: Used in ChaosCase.effects to ensure full round-trip serialization fidelity with Pydantic's model_dump() / model_validate(). """ + + +class ModelEffect(ChaosEffect): + """Effect that operates on model output content. + + Intermediate class parallel to ToolEffect. Enables type-based dispatch + so the plugin can distinguish model-output effects from tool-level effects. + """ + + hook: ClassVar[Literal["pre", "post"]] = "post" + + +# --------------------------------------------------------------------------- +# Helper — module-level (used by multiple model effects) +# --------------------------------------------------------------------------- + + +def _map_text_in_blocks(blocks: list, fn: Any) -> list: + """Apply *fn* to every ``"text"`` value; leave other blocks untouched.""" + result = [] + for block in blocks: + block = dict(block) + if "text" in block and isinstance(block["text"], str): + block["text"] = fn(block["text"]) + result.append(block) + return result + + +# --------------------------------------------------------------------------- +# a) MalformedJson +# --------------------------------------------------------------------------- + + +class MalformedJson(ModelEffect): + """Corrupts JSON structures in model output.""" + + effect_type: Literal["malformed_json"] = "malformed_json" + + def apply(self, content: Any = None) -> Any: + if content is None: + raise ValueError("MalformedJson.apply() requires content") + if isinstance(content, str): + return self._malform_text(content) + elif isinstance(content, list): + return self._malform_blocks(content) + raise ValueError(f"MalformedJson.apply() received unsupported type {type(content).__name__}") + + @staticmethod + def _malform_text(text: str) -> str: + stripped = text.strip() + if stripped.startswith("{") or stripped.startswith("["): + return stripped[: len(stripped) // 2] + return text + + @staticmethod + def _malform_blocks(blocks: list) -> list: + result = [] + for block in blocks: + block = dict(block) + if "toolUse" in block: + tool_use = dict(block["toolUse"]) + raw = json.dumps(tool_use.get("input", {})) + tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" + block["toolUse"] = tool_use + elif "text" in block and isinstance(block["text"], str): + block["text"] = MalformedJson._malform_text(block["text"]) + result.append(block) + return result + + +# --------------------------------------------------------------------------- +# b) EmptyResponse +# --------------------------------------------------------------------------- + + +class EmptyResponse(ModelEffect): + """Returns empty content.""" + + effect_type: Literal["empty_response"] = "empty_response" + + def apply(self, content: Any = None) -> Any: + if content is None: + raise ValueError("EmptyResponse.apply() requires content") + if isinstance(content, str): + return "" + elif isinstance(content, list): + return [] + raise ValueError(f"EmptyResponse.apply() received unsupported type {type(content).__name__}") + + +# --------------------------------------------------------------------------- +# c) Confabulation +# --------------------------------------------------------------------------- + + +class Confabulation(ModelEffect): + """Injects fabricated citations into model output text.""" + + effect_type: Literal["confabulation"] = "confabulation" + + _CONFABULATION_TEMPLATES: ClassVar[list[str]] = [ + "According to a 2019 study published in Nature, ", + "Research conducted by MIT in 2021 demonstrated that ", + "A comprehensive report by the World Health Organization found that ", + "Based on data from the U.S. Census Bureau (2020), ", + "As documented in the IEEE Transactions on Software Engineering, ", + "A peer-reviewed analysis by Stanford researchers confirmed that ", + "The European Commission's 2022 technical report states that ", + "Historical records from the Smithsonian Institution indicate that ", + ] + + def apply(self, content: Any = None) -> Any: + if content is None: + raise ValueError("Confabulation.apply() requires content") + if isinstance(content, str): + return self._confabulate(content) + elif isinstance(content, list): + return _map_text_in_blocks(content, self._confabulate) + raise ValueError(f"Confabulation.apply() received unsupported type {type(content).__name__}") + + def _confabulate(self, text: str) -> str: + if not text: + return text + template = random.choice(self._CONFABULATION_TEMPLATES) + sentences = re.split(r"(?<=[.!?])\s+", text) + if len(sentences) <= 1: + return template + text + insert_pos = random.randint(1, len(sentences) - 1) + sentences.insert(insert_pos, template) + return " ".join(sentences) + + +# --------------------------------------------------------------------------- +# d) FullRefusal +# --------------------------------------------------------------------------- + + +class FullRefusal(ModelEffect): + """Replaces model output with a refusal message.""" + + effect_type: Literal["full_refusal"] = "full_refusal" + + _REFUSAL_TEMPLATES: ClassVar[list[str]] = [ + "I'm sorry, but I can't help with that request.", + "I'm not able to help with this particular request. Could you try rephrasing?", + "I can't assist with that. Please try a different approach.", + "I'm unable to process this request at this time.", + "This is beyond what I'm able to do. Please consider an alternative approach.", + "I can't assist with the request as described. Could you provide more context?", + ] + + def apply(self, content: Any = None) -> Any: + if content is None: + raise ValueError("FullRefusal.apply() requires content") + template = random.choice(self._REFUSAL_TEMPLATES) + if isinstance(content, str): + return template + elif isinstance(content, list): + return [{"text": template}] + raise ValueError(f"FullRefusal.apply() received unsupported type {type(content).__name__}") + + +# --------------------------------------------------------------------------- +# e) SuccessFraming (composable post-step) +# --------------------------------------------------------------------------- + + +class SuccessFraming(ModelEffect): + """Prepends a confident success prefix to content. + + This is composable — applied AFTER another effect to disguise corruption. + """ + + effect_type: Literal["success_framing"] = "success_framing" + + _SUCCESS_PREFIXES: ClassVar[list[str]] = [ + "Successfully completed the requested operation.", + "Done! Here are the results you asked for.", + "Great news \u2014 everything worked as expected.", + "Operation finished successfully. Here's what I found:", + "All done! The task has been completed without issues.", + "I've successfully processed your request. Here's the output:", + "Task completed. Below are the verified results:", + ] + + def apply(self, content: Any = None) -> Any: + if content is None: + raise ValueError("SuccessFraming.apply() requires content") + prefix = random.choice(self._SUCCESS_PREFIXES) + if isinstance(content, str): + return prefix + " " + content + elif isinstance(content, list): + # Prepend into first text block if one exists + for block in content: + if isinstance(block, dict) and "text" in block and isinstance(block["text"], str): + block["text"] = prefix + " " + block["text"] + return content + # No text block — prepend a new one + return [{"text": prefix}] + content + raise ValueError(f"SuccessFraming.apply() received unsupported type {type(content).__name__}") + + +# --------------------------------------------------------------------------- +# ModelEffectUnion — discriminated union for Pydantic deserialization +# --------------------------------------------------------------------------- + +ModelEffectUnion = Annotated[ + Union[ + Annotated[MalformedJson, Tag("malformed_json")], + Annotated[EmptyResponse, Tag("empty_response")], + Annotated[Confabulation, Tag("confabulation")], + Annotated[FullRefusal, Tag("full_refusal")], + Annotated[SuccessFraming, Tag("success_framing")], + ], + Discriminator("effect_type"), +] diff --git a/src/strands_evals/chaos/model_effects.py b/src/strands_evals/chaos/model_effects.py deleted file mode 100644 index 66d5a906..00000000 --- a/src/strands_evals/chaos/model_effects.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Model output corruption effect classes. - -Effects are polymorphic — they accept both ``str`` and ``list[dict]`` content. -The caller (ModelChaosPlugin) is responsible for locating the content within -the framework-specific response; these effects only corrupt it. - -Effect hierarchy: - ChaosEffect (from .effects) → ModelEffect → concrete effects -""" - -import logging -import random -import re -from typing import Any - -from .effects import ChaosEffect -from .model_types import ModelOutputCorruptionConfig, ModelOutputCorruptionType, ModelOutputHallucinationType -from .model_utils import ( - _CONFABULATION_TEMPLATES, - _CONTRADICTION_PHRASES, - _DATE_PATTERNS, - _LOCATION_SUBSTITUTIONS, - _NUMBER_PATTERNS, - _PRICE_PATTERNS, - _REFUSAL_TEMPLATES, - _TOOL_NARRATIVES, - _map_text_in_blocks, -) - -logger = logging.getLogger(__name__) - -_GARBAGE_STRINGS = [ - "\x00\xff\xfe GARBAGE DATA \x00\x01", - "asdkjh2398yr2h3f CORRUPTED asjkdhf98", - "NOT_A_VALID_RESPONSE", - "NaN undefined null [object Object]", -] - -# Set of hallucination types for dispatch -HALLUCINATION_TYPES = set(ModelOutputHallucinationType) - - -class FormatCorruptionEffect(ChaosEffect): - """Corrupts model output content format. - - Supports: EMPTY_RESPONSE, TRUNCATED_RESPONSE, MALFORMED_JSON, - SCHEMA_VIOLATION, GARBAGE_OUTPUT. - """ - - hook = "post" - effect_type: str = "format_corruption" - - def __init__(self, config: ModelOutputCorruptionConfig): - super().__init__() - self._config = config - - def apply(self, content: Any = None) -> Any: - """Corrupt *content* and return the same shape.""" - if content is None: - raise ValueError("FormatCorruptionEffect.apply() requires content") - - ct = self._config.corruption_type - - if isinstance(content, str): - return self._apply_to_str(content, ct) - elif isinstance(content, list): - return self._apply_to_blocks(content, ct) - else: - raise ValueError( - f"FormatCorruptionEffect.apply() received unsupported content type " - f"{type(content).__name__}; expected str or list[dict]." - ) - - def _truncate(self, text: str) -> str: - if not text: - return text - end = max(1, round(len(text) * (1 - self._config.truncate_ratio))) - return text[:end] - - def _apply_to_str(self, text: str, ct: Any) -> str: - match ct: - case ModelOutputCorruptionType.EMPTY_RESPONSE: - return "" - case ModelOutputCorruptionType.TRUNCATED_RESPONSE: - return self._truncate(text) - case ModelOutputCorruptionType.GARBAGE_OUTPUT: - return random.choice(_GARBAGE_STRINGS) - case ModelOutputCorruptionType.MALFORMED_JSON: - return self._malform_text(text) - case _: - return text - - def _apply_to_blocks(self, blocks: list, ct: Any) -> list: - match ct: - case ModelOutputCorruptionType.EMPTY_RESPONSE: - return [] - case ModelOutputCorruptionType.TRUNCATED_RESPONSE: - return _map_text_in_blocks(blocks, self._truncate) - case ModelOutputCorruptionType.GARBAGE_OUTPUT: - return _map_text_in_blocks(blocks, lambda _: random.choice(_GARBAGE_STRINGS)) - case ModelOutputCorruptionType.MALFORMED_JSON: - return self._malform_json(blocks) - case ModelOutputCorruptionType.SCHEMA_VIOLATION: - return self._violate_schema(blocks) - case _: - return list(blocks) - - @staticmethod - def _malform_text(text: str) -> str: - stripped = text.strip() - if stripped.startswith("{") or stripped.startswith("["): - return stripped[: len(stripped) // 2] - return text - - @staticmethod - def _malform_json(blocks: list) -> list: - import json as json_mod - - def corrupt_tool_use(tool_use: dict) -> dict: - raw = json_mod.dumps(tool_use.get("input", {})) - tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" - return tool_use - - result = [] - for block in blocks: - block = dict(block) - if "toolUse" in block: - block["toolUse"] = corrupt_tool_use(dict(block["toolUse"])) - elif "text" in block and isinstance(block["text"], str): - block["text"] = FormatCorruptionEffect._malform_text(block["text"]) - result.append(block) - return result - - @staticmethod - def _violate_schema(blocks: list) -> list: - type_swaps: list[Any] = [None, 99999, True, [], "WRONG_TYPE"] - - def corrupt_tool_use(tool_use: dict) -> dict: - inp = tool_use.get("input", {}) - if isinstance(inp, dict) and inp: - corrupted = {} - for k, v in inp.items(): - candidates = [s for s in type_swaps if not isinstance(v, type(s))] - corrupted[k] = random.choice(candidates) if candidates else "WRONG_TYPE" - tool_use["input"] = corrupted - return tool_use - - result = [] - for block in blocks: - block = dict(block) - if "toolUse" in block: - block["toolUse"] = corrupt_tool_use(dict(block["toolUse"])) - result.append(block) - return result - - -class HallucinationEffect(ChaosEffect): - """Corrupts model output text with hallucination mutations. - - Supports: FACTUAL_ERROR, CONFABULATION, CONTEXT_UNFAITHFULNESS, - TOOL_CLAIM_FABRICATION. - """ - - hook = "post" - effect_type: str = "hallucination" - - def __init__(self, config: ModelOutputCorruptionConfig) -> None: - super().__init__() - self._config = config - - def apply(self, content: Any = None) -> Any: - """Mutate *content* and return the same shape.""" - if content is None: - raise ValueError("HallucinationEffect.apply() requires content") - - ct = self._config.corruption_type - - if isinstance(content, str): - return self._apply_to_str(content, ct) - elif isinstance(content, list): - return self._apply_to_blocks(content, ct) - else: - raise ValueError( - f"HallucinationEffect.apply() received unsupported content type " - f"{type(content).__name__}; expected str or list[dict]." - ) - - def _apply_to_str(self, text: str, ct: Any) -> str: - match ct: - case ModelOutputHallucinationType.FACTUAL_ERROR: - return self._factual_error(text) - case ModelOutputHallucinationType.CONFABULATION: - return self._confabulation(text) - case ModelOutputHallucinationType.CONTEXT_UNFAITHFULNESS: - return self._context_unfaithfulness(text) - case ModelOutputHallucinationType.TOOL_CLAIM_FABRICATION: - return self._tool_claim_fabrication(text) - case _: - return text - - def _apply_to_blocks(self, blocks: list, ct: Any) -> list: - match ct: - case ModelOutputHallucinationType.FACTUAL_ERROR: - return _map_text_in_blocks(blocks, self._factual_error) - case ModelOutputHallucinationType.CONFABULATION: - return _map_text_in_blocks(blocks, self._confabulation) - case ModelOutputHallucinationType.CONTEXT_UNFAITHFULNESS: - return _map_text_in_blocks(blocks, self._context_unfaithfulness) - case ModelOutputHallucinationType.TOOL_CLAIM_FABRICATION: - return _map_text_in_blocks(blocks, self._tool_claim_fabrication) - case _: - return list(blocks) - - def _factual_error(self, text: str) -> str: - target_types = self._config.target_entity_types - result = text - - entity_patterns: list[tuple] = [] - if target_types is None or "date" in target_types: - entity_patterns.extend(_DATE_PATTERNS) - if target_types is None or "number" in target_types: - entity_patterns.extend(_NUMBER_PATTERNS) - if target_types is None or "price" in target_types: - entity_patterns.extend(_PRICE_PATTERNS) - - for pattern, replacer in entity_patterns: - try: - result = re.sub(pattern, replacer, result) - except Exception: - pass - - if target_types is None or "location" in target_types: - for old_loc, new_loc in _LOCATION_SUBSTITUTIONS.items(): - result = result.replace(old_loc, new_loc) - - return result - - def _confabulation(self, text: str) -> str: - if not text: - return text - template = random.choice(_CONFABULATION_TEMPLATES) - sentences = re.split(r"(?<=[.!?])\s+", text) - if len(sentences) <= 1: - return template + text - insert_pos = random.randint(1, len(sentences) - 1) - sentences.insert(insert_pos, template) - return " ".join(sentences) - - def _context_unfaithfulness(self, text: str) -> str: - if not text: - return text - phrase = random.choice(_CONTRADICTION_PHRASES) - sentences = re.split(r"(?<=[.!?])\s+", text) - if len(sentences) <= 1: - return text + phrase - insert_pos = random.randint(1, len(sentences) - 1) - sentences.insert(insert_pos, phrase.strip()) - return " ".join(sentences) - - def _tool_claim_fabrication(self, text: str) -> str: - if not text: - return text - narrative = random.choice(_TOOL_NARRATIVES) - sentences = re.split(r"(?<=[.!?])\s+", text) - if len(sentences) <= 1: - return text + " " + narrative - insert_pos = random.randint(1, len(sentences) - 1) - sentences.insert(insert_pos, narrative.strip()) - return " ".join(sentences) - - -class RefusalEffect(ChaosEffect): - """Replaces model output content with a refusal message. - - Supports: FULL_REFUSAL only. - """ - - hook = "post" - effect_type: str = "refusal" - - def __init__(self, config: ModelOutputCorruptionConfig) -> None: - super().__init__() - self._config = config - - def apply(self, content: Any = None) -> Any: - """Replace *content* with a refusal message.""" - if content is None: - raise ValueError("RefusalEffect.apply() requires content") - - template = random.choice(_REFUSAL_TEMPLATES) - - if isinstance(content, str): - return template - elif isinstance(content, list): - return [{"text": template}] - else: - raise ValueError( - f"RefusalEffect.apply() received unsupported content type " - f"{type(content).__name__}; expected str or list[dict]." - ) diff --git a/src/strands_evals/chaos/model_types.py b/src/strands_evals/chaos/model_types.py deleted file mode 100644 index a35b6a7c..00000000 --- a/src/strands_evals/chaos/model_types.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Model output corruption types and configuration. - -Defines the enums and Pydantic config model used to parameterize model -output corruption effects. -""" - -from enum import Enum -from typing import Optional - -from pydantic import BaseModel, Field - - -class ModelOutputCorruptionType(str, Enum): - """Output corruption types for model response format mutation.""" - - MALFORMED_JSON = "malformed_json" - TRUNCATED_RESPONSE = "truncated_response" - SCHEMA_VIOLATION = "schema_violation" - EMPTY_RESPONSE = "empty_response" - GARBAGE_OUTPUT = "garbage_output" - FULL_REFUSAL = "full_refusal" - TOXIC_CONTENT = "toxic_content" - - -class ModelOutputHallucinationType(str, Enum): - """Hallucination types for model response semantic mutation. - - Separated from ModelOutputCorruptionType to allow future LLM-based - hallucination generation strategies. - """ - - FACTUAL_ERROR = "factual_error" - CONFABULATION = "confabulation" - CONTEXT_UNFAITHFULNESS = "context_unfaithfulness" - TOOL_CLAIM_FABRICATION = "tool_claim_fabrication" - - -class ModelOutputCorruptionConfig(BaseModel): - """Configuration for model output corruption. - - Mutates model responses after generation to test agent resilience - against malformed, truncated, or otherwise corrupted LLM output. - - Attributes: - apply_rate: Probability (0.0 to 1.0) that corruption is applied per call. - 0.0 disables output corruption. - corruption_type: Type of output corruption to apply. - truncate_ratio: Fraction of content to corrupt for TRUNCATED_RESPONSE. - target_entity_types: Entity types for FACTUAL_ERROR filtering. - add_success_framing: Prepend a confident success prefix after corruption. - """ - - apply_rate: float = Field(default=0.0, ge=0.0, le=1.0, description="Probability of corruption per model call") - corruption_type: ModelOutputCorruptionType | ModelOutputHallucinationType = Field( - default=ModelOutputCorruptionType.MALFORMED_JSON, - description="Type of output corruption or hallucination to apply", - ) - truncate_ratio: float = Field( - default=0.5, - ge=0.0, - le=1.0, - description="Fraction of content to corrupt for TRUNCATED_RESPONSE", - ) - target_entity_types: Optional[list[str]] = Field( - default=None, - description="Entity types for FACTUAL_ERROR: 'date', 'number', 'location', 'price'. None = all.", - ) - add_success_framing: bool = Field( - default=False, - description="Prepend a confident success prefix before the corrupted content. " - "Composable with any corruption_type or hallucination_type.", - ) diff --git a/src/strands_evals/chaos/model_utils.py b/src/strands_evals/chaos/model_utils.py deleted file mode 100644 index 4c6cfdf8..00000000 --- a/src/strands_evals/chaos/model_utils.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Shared model corruption helper functions and constants. - -Used by model output corruption effects for factual-error perturbation, -confabulation, context-unfaithfulness, tool-claim fabrication, refusal -templates, and success framing prefixes. -""" - -import random -import re -from datetime import datetime, timedelta -from typing import Any - -# --------------------------------------------------------------------------- -# Shared helper — module-level -# --------------------------------------------------------------------------- - - -def _map_text_in_blocks(blocks: list, fn: Any) -> list: - """Apply *fn* to every ``"text"`` value; leave other blocks untouched.""" - result = [] - for block in blocks: - block = dict(block) - if "text" in block and isinstance(block["text"], str): - block["text"] = fn(block["text"]) - result.append(block) - return result - - -# --------------------------------------------------------------------------- -# FACTUAL_ERROR: Perturbation helpers -# --------------------------------------------------------------------------- - -_MONTH_NAMES = [ - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", -] - - -def _shift_date_named(m: re.Match) -> str: - """Shift a named date like 'January 15, 2023' by ±1-3 months and ±1-2 years.""" - text = m.group(0) - for i, month in enumerate(_MONTH_NAMES): - if month in text: - new_month_idx = (i + random.randint(1, 3)) % 12 - new_month = _MONTH_NAMES[new_month_idx] - text = text.replace(month, new_month) - break - year_match = re.search(r"\d{4}", text) - if year_match: - old_year = int(year_match.group()) - new_year = old_year + random.choice([-2, -1, 1, 2]) - text = text.replace(str(old_year), str(new_year)) - return text - - -def _shift_date_slash(m: re.Match) -> str: - """Shift a slash date like '12/25/2023' by ±1-5 days and ±1-3 months.""" - text = m.group(0) - parts = text.split("/") - if len(parts) >= 2: - month = int(parts[0]) - day = int(parts[1]) - new_month = max(1, min(12, month + random.randint(-3, 3))) - new_day = max(1, min(28, day + random.randint(-5, 5))) - parts[0] = str(new_month) - parts[1] = str(new_day) - return "/".join(parts) - - -def _shift_date_iso(m: re.Match) -> str: - """Shift an ISO date like '2023-01-15' by ±1-10 days.""" - text = m.group(0) - try: - dt = datetime.strptime(text, "%Y-%m-%d") - delta = timedelta(days=random.randint(1, 10) * random.choice([-1, 1])) - new_dt = dt + delta - return new_dt.strftime("%Y-%m-%d") - except ValueError: - return text - - -def _perturb_number(m: re.Match) -> str: - """Perturb a comma-formatted number like '1,234,567' by factor [0.5, 2.0].""" - text = m.group(1) if m.lastindex else m.group(0) - try: - num = int(text.replace(",", "")) - factor = random.uniform(0.5, 2.0) - new_num = int(num * factor) - if new_num == num: - new_num = num + random.choice([-1, 1]) - return f"{new_num:,}" - except ValueError: - return text - - -def _perturb_float(m: re.Match) -> str: - """Perturb a float like '42.5' by ±20%.""" - text = m.group(1) if m.lastindex else m.group(0) - try: - num = float(text) - offset = num * random.uniform(-0.2, 0.2) - if abs(offset) < 0.01: - offset = random.choice([-0.1, 0.1]) - new_num = num + offset - decimal_places = len(text.split(".")[1]) if "." in text else 1 - return f"{new_num:.{decimal_places}f}" - except ValueError: - return text - - -def _perturb_integer(m: re.Match) -> str: - """Perturb an integer like '100' by ±30%.""" - text = m.group(1) if m.lastindex else m.group(0) - try: - num = int(text) - offset = int(num * random.uniform(-0.3, 0.3)) - if offset == 0: - offset = random.choice([-1, 1]) - return str(num + offset) - except ValueError: - return text - - -def _perturb_price(m: re.Match) -> str: - """Perturb a price like '$1,234.56' by factor [0.7, 1.5], preserving currency symbol.""" - currency = m.group(1) - amount_str = m.group(2) - try: - amount = float(amount_str.replace(",", "")) - factor = random.uniform(0.7, 1.5) - new_amount = amount * factor - if abs(new_amount - amount) < 0.01: - new_amount = amount + random.choice([-1.0, 1.0]) - if "." in amount_str: - formatted = f"{new_amount:,.2f}" - else: - formatted = f"{int(new_amount):,}" - return f"{currency}{formatted}" - except ValueError: - return m.group(0) - - -# --------------------------------------------------------------------------- -# FACTUAL_ERROR: Regex patterns -# --------------------------------------------------------------------------- - -_DATE_PATTERNS = [ - ( - r"\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}\b", - lambda m: _shift_date_named(m), - ), - (r"\b\d{1,2}/\d{1,2}/\d{2,4}\b", lambda m: _shift_date_slash(m)), - (r"\b\d{4}-\d{2}-\d{2}\b", lambda m: _shift_date_iso(m)), -] - -_NUMBER_PATTERNS = [ - (r"\b(\d{1,3}(?:,\d{3})+)\b", lambda m: _perturb_number(m)), - (r"\b(\d+\.\d+)\b", lambda m: _perturb_float(m)), - (r"\b(\d{2,})\b", lambda m: _perturb_integer(m)), -] - -_LOCATION_SUBSTITUTIONS = { - "New York": "Chicago", - "Chicago": "Houston", - "London": "Manchester", - "Paris": "Lyon", - "Tokyo": "Osaka", - "Berlin": "Munich", - "Sydney": "Melbourne", - "San Francisco": "Seattle", - "Seattle": "Portland", - "Los Angeles": "San Diego", - "Beijing": "Shanghai", - "Mumbai": "Delhi", - "Toronto": "Vancouver", - "United States": "Canada", - "United Kingdom": "Australia", -} - -_PRICE_PATTERNS = [ - (r"([$€£¥])\s*(\d{1,3}(?:,\d{3})*(?:\.\d{2})?)", lambda m: _perturb_price(m)), -] - - -# --------------------------------------------------------------------------- -# CONFABULATION: Template phrases -# --------------------------------------------------------------------------- - -_CONFABULATION_TEMPLATES = [ - "According to a 2019 study published in Nature, ", - "Research conducted by MIT in 2021 demonstrated that ", - "A comprehensive report by the World Health Organization found that ", - "Based on data from the U.S. Census Bureau (2020), ", - "As documented in the IEEE Transactions on Software Engineering, ", - "A peer-reviewed analysis by Stanford researchers confirmed that ", - "The European Commission's 2022 technical report states that ", - "Historical records from the Smithsonian Institution indicate that ", -] - - -# --------------------------------------------------------------------------- -# CONTEXT_UNFAITHFULNESS: Contradiction phrases -# --------------------------------------------------------------------------- - -_CONTRADICTION_PHRASES = [ - " However, this is actually incorrect. ", - " On the contrary, the opposite is true. ", - " In fact, recent evidence contradicts this entirely. ", - " Note: the above statement has been disputed by multiple sources. ", - " Actually, this information was later retracted. ", - " Importantly, this conflicts with the data presented earlier. ", - " Correction: the previous claim is not supported by evidence. ", -] - - -# --------------------------------------------------------------------------- -# TOOL_CLAIM_FABRICATION: Fake tool-call narratives -# --------------------------------------------------------------------------- - -_TOOL_NARRATIVES = [ - "I called the database_lookup tool and it returned the following results: ", - "After running the web_search tool, I found that ", - "The file_reader tool confirmed that the document contains ", - "I used the calculator tool to verify, and the result is ", - "The api_query tool returned a 200 OK response indicating ", -] - - -# --------------------------------------------------------------------------- -# FULL_REFUSAL: Refusal message templates -# --------------------------------------------------------------------------- - -_REFUSAL_TEMPLATES = [ - "I'm sorry, but I can't help with that request.", - "I'm not able to help with this particular request. Could you try rephrasing?", - "I can't assist with that. Please try a different approach.", - "I'm unable to process this request at this time.", - "This is beyond what I'm able to do. Please consider an alternative approach.", - "I can't assist with the request as described. Could you provide more context?", -] - - -# --------------------------------------------------------------------------- -# SUCCESS FRAMING: Success prefix templates -# --------------------------------------------------------------------------- - -_SUCCESS_PREFIXES = [ - "Successfully completed the requested operation.", - "Done! Here are the results you asked for.", - "Great news — everything worked as expected.", - "Operation finished successfully. Here's what I found:", - "All done! The task has been completed without issues.", - "I've successfully processed your request. Here's the output:", - "Task completed. Below are the verified results:", -] diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index 9c9dfe08..71f1c88b 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -24,28 +24,24 @@ import json import logging -import random -from typing import Any from strands.hooks import AfterToolCallEvent, BeforeToolCallEvent, MessageAddedEvent from strands.plugins import Plugin, hook from ._context import _current_chaos_case -from .effects import ChaosEffect, TruncateFields -from .model_effects import ( - HALLUCINATION_TYPES, - FormatCorruptionEffect, - HallucinationEffect, - RefusalEffect, +from .effects import ( + ChaosEffect, + ModelEffect, + ModelEffectUnion, + SuccessFraming, + TruncateFields, ) -from .model_types import ModelOutputCorruptionConfig, ModelOutputCorruptionType -from .model_utils import _SUCCESS_PREFIXES logger = logging.getLogger(__name__) class ChaosPlugin(Plugin): - """Strands Plugin that injects deterministic chaos based on the active ChaosCase. + """Strands Plugin that injects deterministic chaos based on configuration. Handles both tool-level chaos (P0) and model-output chaos (P1): @@ -57,27 +53,20 @@ class ChaosPlugin(Plugin): - MessageAddedEvent: corrupts the final assistant response content The active ChaosCase is managed via a ContextVar (set by ChaosExperiment). - When no ChaosCase is active or the case has no effects, all tools and model - output behave normally. + When no ChaosCase is active or the case has no effects, all tools behave normally. - Model output corruption is configured via `model_output_config` on the - ChaosPlugin instance or via the ChaosCase effects dict (key: "model_effects"). + Model output corruption is configured via `model_effects` on the ChaosPlugin + instance. Effects are applied sequentially. SuccessFraming is always applied + LAST (composable post-step). Example:: from strands import Agent from strands_evals.chaos import ChaosPlugin - from strands_evals.chaos.model_types import ( - ModelOutputCorruptionConfig, - ModelOutputHallucinationType, - ) + from strands_evals.chaos.effects import EmptyResponse, SuccessFraming chaos = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputHallucinationType.CONFABULATION, - add_success_framing=True, - ) + model_effects=[EmptyResponse(), SuccessFraming()], ) agent = Agent( @@ -85,34 +74,42 @@ class ChaosPlugin(Plugin): tools=[search_tool, database_tool], plugins=[chaos], ) + + NOTE on structured_output: When the agent uses structured_output_model, the + final response is a toolUse block (containing the structured output tool call). + The toolUse guard will skip these messages, so model chaos does NOT affect + structured_output responses. This is intentional — corrupting the structured + output tool call would break parsing. Future work may add a dedicated + structured_output chaos effect that corrupts the tool input fields specifically. """ name = "chaos-testing" - def __init__(self, model_output_config: ModelOutputCorruptionConfig | None = None) -> None: + def __init__(self, model_effects: list[ModelEffectUnion] | None = None) -> None: """Initialize the ChaosPlugin. Args: - model_output_config: Optional configuration for model output corruption. + model_effects: Optional list of model-output effects to apply sequentially. When provided, enables model-output chaos on final assistant responses. + SuccessFraming (if included) is always applied last regardless of list order. When None, only tool-level chaos (from ChaosCase effects) is active. """ super().__init__() - self._model_output_config = model_output_config + self._model_effects = model_effects @property - def model_output_config(self) -> ModelOutputCorruptionConfig | None: - """The active model output corruption configuration.""" - return self._model_output_config + def model_effects(self) -> list[ModelEffectUnion] | None: + """The active model output effects list.""" + return self._model_effects - @model_output_config.setter - def model_output_config(self, value: ModelOutputCorruptionConfig | None) -> None: - """Update the model output corruption configuration.""" - self._model_output_config = value + @model_effects.setter + def model_effects(self, value: list[ModelEffectUnion] | None) -> None: + """Update the model output effects list.""" + self._model_effects = value - # ------------------------------------------------------------------ + # ----------------------------------------------------------------------- # Tool chaos hooks (P0) — from PR #224 - # ------------------------------------------------------------------ + # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] def before_tool_call(self, event: BeforeToolCallEvent) -> None: @@ -170,17 +167,17 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: logger.info("effect=<%s>, tool=<%s> | applied chaos post-hook", type(effect).__name__, tool_name) - # ------------------------------------------------------------------ + # ----------------------------------------------------------------------- # Model output chaos hook (P1) - # ------------------------------------------------------------------ + # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] def message_added(self, event: MessageAddedEvent) -> None: """Intercept messages to corrupt the final assistant response. GUARD: corruption is applied ONLY when ALL conditions hold: - 1. message role == "assistant" - 2. message content contains NO toolUse blocks + 1. message role == "assistant" + 2. message content contains NO toolUse blocks This prevents destructive effects from breaking mid-turn tool dispatch. MessageAddedEvent fires BEFORE the agent extracts toolUse blocks for @@ -192,10 +189,7 @@ def message_added(self, event: MessageAddedEvent) -> None: without are final end_turn responses. This is reliable because end_turn messages never contain toolUse blocks. """ - if self._model_output_config is None: - return - - if self._model_output_config.apply_rate <= 0: + if self._model_effects is None: return message = event.message @@ -214,63 +208,31 @@ def message_added(self, event: MessageAddedEvent) -> None: if isinstance(block, dict) and "toolUse" in block: return - # Guard 3: apply_rate probabilistic check - if random.random() >= self._model_output_config.apply_rate: - return + # Separate SuccessFraming from primary effects (applied last) + primary_effects: list[ModelEffect] = [e for e in self._model_effects if not isinstance(e, SuccessFraming)] + framing_effects: list[ModelEffect] = [e for e in self._model_effects if isinstance(e, SuccessFraming)] - # Dispatch to effect - corrupted = self._apply_model_corruption(content) + # Apply primary effects sequentially + corrupted = content + for effect in primary_effects: + corrupted = effect.apply(corrupted) - # Apply success framing if content was mutated - if self._model_output_config.add_success_framing and corrupted != content: - corrupted = self._apply_success_framing(corrupted) + # Apply success framing last (composable post-step) + for effect in framing_effects: + corrupted = effect.apply(corrupted) # Mutate via dict assignment (NOT attribute assignment on the event) message["content"] = corrupted + effect_names = ", ".join(type(e).__name__ for e in self._model_effects) logger.info( - "corruption_type=<%s> | applied model output chaos to assistant message", - self._model_output_config.corruption_type.value, + "effects=<%s> | applied model output chaos to assistant message", + effect_names, ) - # ------------------------------------------------------------------ - # Model corruption helpers - # ------------------------------------------------------------------ - - def _apply_model_corruption(self, content: Any) -> Any: - """Dispatch to the appropriate model effect and apply corruption.""" - config = self._model_output_config - assert config is not None - ct = config.corruption_type - - effect: ChaosEffect - if ct in HALLUCINATION_TYPES: - effect = HallucinationEffect(config) - elif ct == ModelOutputCorruptionType.FULL_REFUSAL: - effect = RefusalEffect(config) - else: - effect = FormatCorruptionEffect(config) - - return effect.apply(content) - - @staticmethod - def _apply_success_framing(content: Any) -> Any: - """Prepend a success prefix to corrupted content.""" - prefix = random.choice(_SUCCESS_PREFIXES) - - if isinstance(content, str): - return prefix + " " + content - elif isinstance(content, list): - for block in content: - if isinstance(block, dict) and "text" in block and isinstance(block["text"], str): - block["text"] = prefix + " " + block["text"] - return content - return [{"text": prefix}] + content - return content - - # ------------------------------------------------------------------ + # ----------------------------------------------------------------------- # Tool corruption helpers (P0) - # ------------------------------------------------------------------ + # ----------------------------------------------------------------------- def _apply_to_blocks(self, effect: ChaosEffect, blocks: list) -> list: """Apply effect to text blocks in a content list.""" diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py index 4467741a..113c6e46 100644 --- a/tests/strands_evals/chaos/test_model_chaos.py +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -11,24 +11,18 @@ import copy from unittest.mock import MagicMock -from strands_evals.chaos.model_types import ( - ModelOutputCorruptionConfig, - ModelOutputCorruptionType, - ModelOutputHallucinationType, -) -from strands_evals.chaos.model_utils import ( - _REFUSAL_TEMPLATES, - _SUCCESS_PREFIXES, +from strands_evals.chaos.effects import ( + Confabulation, + EmptyResponse, + FullRefusal, + MalformedJson, + SuccessFraming, ) from strands_evals.chaos.plugin import ChaosPlugin def _make_event(message: dict) -> MagicMock: - """Create a mock MessageAddedEvent with the given message. - - The event's `message` attribute is a real dict (not a Mock) so that - dict mutation works as in production. - """ + """Create a mock MessageAddedEvent with the given message.""" event = MagicMock() event.message = message return event @@ -78,12 +72,7 @@ class TestModelChaosFormatCorruptionMalformedJson: """MALFORMED_JSON effect corrupts the final assistant message.""" def test_malformed_json_corrupts_json_text(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.MALFORMED_JSON, - ) - ) + plugin = ChaosPlugin(model_effects=[MalformedJson()]) message = _final_assistant_message('{"key": "value", "nested": {"a": 1}}') event = _make_event(message) @@ -98,12 +87,7 @@ class TestModelChaosFormatCorruptionEmptyResponse: """EMPTY_RESPONSE effect empties the final assistant message content.""" def test_empty_response_on_final_message(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin(model_effects=[EmptyResponse()]) message = _final_assistant_message("Hello world") event = _make_event(message) @@ -116,12 +100,7 @@ class TestModelChaosHallucination: """Hallucination effect corrupts the final assistant message text.""" def test_confabulation_injects_template(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputHallucinationType.CONFABULATION, - ) - ) + plugin = ChaosPlugin(model_effects=[Confabulation()]) original_text = "The weather is sunny. It is warm outside. Birds are singing." message = _final_assistant_message(original_text) event = _make_event(message) @@ -138,12 +117,7 @@ class TestModelChaosRefusal: """FULL_REFUSAL replaces the final assistant message content with a refusal.""" def test_refusal_replaces_content(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.FULL_REFUSAL, - ) - ) + plugin = ChaosPlugin(model_effects=[FullRefusal()]) message = _final_assistant_message("Here is the code you requested...") event = _make_event(message) @@ -151,27 +125,21 @@ def test_refusal_replaces_content(self): # Content should be a single refusal text block assert len(message["content"]) == 1 - assert message["content"][0]["text"] in _REFUSAL_TEMPLATES + assert message["content"][0]["text"] in FullRefusal._REFUSAL_TEMPLATES class TestModelChaosSuccessFraming: """Success framing prepends a confident prefix after corruption.""" def test_success_framing_with_refusal(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.FULL_REFUSAL, - add_success_framing=True, - ) - ) + plugin = ChaosPlugin(model_effects=[FullRefusal(), SuccessFraming()]) message = _final_assistant_message("Here is the code you requested...") event = _make_event(message) plugin.message_added(event) result_text = message["content"][0]["text"] - assert any(result_text.startswith(prefix) for prefix in _SUCCESS_PREFIXES) + assert any(result_text.startswith(prefix) for prefix in SuccessFraming._SUCCESS_PREFIXES) # --------------------------------------------------------------------------- @@ -184,12 +152,7 @@ class TestModelChaosGuardToolUseMessage: def test_empty_response_on_tooluse_message_not_corrupted(self): """EMPTY_RESPONSE on a tool_use message passes through; toolUse intact.""" - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin(model_effects=[EmptyResponse()]) message = _tooluse_assistant_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) @@ -205,12 +168,7 @@ def test_empty_response_on_tooluse_message_not_corrupted(self): def test_full_refusal_on_tooluse_message_not_corrupted(self): """FULL_REFUSAL on a tool_use message passes through; toolUse intact.""" - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.FULL_REFUSAL, - ) - ) + plugin = ChaosPlugin(model_effects=[FullRefusal()]) message = _tooluse_assistant_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) @@ -225,12 +183,7 @@ class TestModelChaosGuardStructuredOutputPath: def test_structured_output_tool_message_not_corrupted(self): """A message containing the structured output tool call is NOT corrupted.""" - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin(model_effects=[EmptyResponse()]) # Simulate the structured output tool invocation message message = { "role": "assistant", @@ -258,12 +211,7 @@ class TestModelChaosGuardFinalMessage: def test_final_end_turn_message_corrupted(self): """An end_turn message with text only (no toolUse) gets corrupted.""" - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin(model_effects=[EmptyResponse()]) message = _final_assistant_message("Hello world") event = _make_event(message) @@ -276,12 +224,7 @@ class TestModelChaosGuardRoleFiltering: """User and tool result messages are NOT corrupted.""" def test_user_message_not_corrupted(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin(model_effects=[EmptyResponse()]) message = _user_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) @@ -291,12 +234,7 @@ def test_user_message_not_corrupted(self): assert message["content"] == original_content def test_tool_result_message_not_corrupted(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=1.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin(model_effects=[EmptyResponse()]) message = _tool_result_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) @@ -307,25 +245,10 @@ def test_tool_result_message_not_corrupted(self): class TestModelChaosPassthrough: - """No corruption when no model_output_config is set.""" + """No corruption when no model_effects is set.""" def test_no_config_passes_through(self): - plugin = ChaosPlugin() # No model_output_config - message = _final_assistant_message("Hello world") - original_content = copy.deepcopy(message["content"]) - event = _make_event(message) - - plugin.message_added(event) - - assert message["content"] == original_content - - def test_zero_apply_rate_passes_through(self): - plugin = ChaosPlugin( - model_output_config=ModelOutputCorruptionConfig( - apply_rate=0.0, - corruption_type=ModelOutputCorruptionType.EMPTY_RESPONSE, - ) - ) + plugin = ChaosPlugin() # No model_effects message = _final_assistant_message("Hello world") original_content = copy.deepcopy(message["content"]) event = _make_event(message) From a41ca668bbc67cfa4501573c530e19214334049c Mon Sep 17 00:00:00 2001 From: venkatkrish543re Date: Fri, 26 Jun 2026 10:00:45 -0700 Subject: [PATCH 3/8] Update plugin.py --- src/strands_evals/chaos/plugin.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index 71f1c88b..f48bb843 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -17,9 +17,7 @@ This guard prevents destructive effects (EMPTY_RESPONSE, FULL_REFUSAL) from deleting toolUse blocks on mid-turn tool_use messages, which would break the -agent loop. MessageAddedEvent fires BEFORE tool dispatch (event_loop.py L427-428 -fires the event; L187 branches on stop_reason; L476 extracts toolUse from the -same message object). +agent loop. MessageAddedEvent fires BEFORE tool dispatch. """ import json @@ -43,13 +41,13 @@ class ChaosPlugin(Plugin): """Strands Plugin that injects deterministic chaos based on configuration. - Handles both tool-level chaos (P0) and model-output chaos (P1): + Handles both tool-level chaos and model-output chaos: - Tool chaos (P0): + Tool chaos: - BeforeToolCallEvent: cancels tool calls for pre-hook effects - AfterToolCallEvent: corrupts tool responses for post-hook effects - Model output chaos (P1): + Model output chaos: - MessageAddedEvent: corrupts the final assistant response content The active ChaosCase is managed via a ContextVar (set by ChaosExperiment). @@ -108,7 +106,7 @@ def model_effects(self, value: list[ModelEffectUnion] | None) -> None: self._model_effects = value # ----------------------------------------------------------------------- - # Tool chaos hooks (P0) — from PR #224 + # Tool chaos hooks # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] @@ -168,7 +166,7 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: logger.info("effect=<%s>, tool=<%s> | applied chaos post-hook", type(effect).__name__, tool_name) # ----------------------------------------------------------------------- - # Model output chaos hook (P1) + # Model output chaos hook # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] @@ -231,7 +229,7 @@ def message_added(self, event: MessageAddedEvent) -> None: ) # ----------------------------------------------------------------------- - # Tool corruption helpers (P0) + # Tool corruption helpers # ----------------------------------------------------------------------- def _apply_to_blocks(self, effect: ChaosEffect, blocks: list) -> list: From 9d9328b367e56b51e5460115844ddec89be186c5 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Fri, 26 Jun 2026 23:59:29 +0000 Subject: [PATCH 4/8] refactor: implement two-hook model chaos (pre/post split) - FullRefusal becomes a true pre-model effect using BeforeModelCallEvent.cancel (strands-agents >= 1.45.0) - All other model effects (EmptyResponse, Confabulation, MalformedJson, SuccessFraming) remain post-model via MessageAddedEvent - Move model_effects from ChaosPlugin to ChaosCase as a sibling field (flat list, no tool-name dimension) - Add before_model_invocation hook; rename message_added to after_model_invocation - Add pre-effects guard in post hook to prevent double-corruption in mixed pre+post cases - Delete _map_text_in_blocks, replace with _apply_text_to_blocks - Rewrite tests for case-based construction via ContextVar - Add pre-hook integration test and mixed-case test 14 tests pass. --- AGENTS.md | 2 +- src/strands_evals/chaos/case.py | 14 +- src/strands_evals/chaos/effects.py | 17 +- src/strands_evals/chaos/plugin.py | 123 +++++++----- tests/strands_evals/chaos/test_model_chaos.py | 190 ++++++++++++++---- 5 files changed, 256 insertions(+), 90 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 78880852..e8e1bb26 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,7 +96,7 @@ strands-evals/ │ │ │ # ValidationError / TruncateFields / │ │ │ # RemoveFields / CorruptValues │ │ ├── experiment.py # ChaosExperiment (sets active case via ContextVar) -│ │ ├── plugin.py # ChaosPlugin (BeforeToolCallEvent / AfterToolCallEvent) +│ │ ├── plugin.py # ChaosPlugin (tool + model hooks via ContextVar) │ │ └── _context.py # ContextVar holding the active ChaosCase │ │ │ ├── experimental/ # Stable public API, evolving surface diff --git a/src/strands_evals/chaos/case.py b/src/strands_evals/chaos/case.py index 29d6de9d..6ffc442c 100644 --- a/src/strands_evals/chaos/case.py +++ b/src/strands_evals/chaos/case.py @@ -12,7 +12,7 @@ from ..case import Case from ..types.evaluation import InputT, OutputT -from .effects import ToolEffectUnion +from .effects import ModelEffectUnion, ToolEffectUnion class ChaosCase(Case, Generic[InputT, OutputT]): @@ -61,6 +61,12 @@ class ChaosCase(Case, Generic[InputT, OutputT]): "tool_name -> list of effects. Empty dict means baseline (no chaos).", ) + model_effects: list[ModelEffectUnion] = Field( + default_factory=list, + description="List of model output effects to apply. FullRefusal is pre-hook " + "(cancels model call); others are post-hook (corrupt response).", + ) + @model_validator(mode="after") def _validate_tool_effects(self) -> "ChaosCase": """Validate tool effects configuration.""" @@ -152,4 +158,8 @@ def __repr__(self) -> str: effects_str = ", ".join( f"{target}: [{', '.join(type(e).__name__ for e in effs)}]" for target, effs in self.tool_effects.items() ) - return f"ChaosCase(name='{self.name}', effects={{{effects_str}}})" + parts = [f"name='{self.name}'", f"effects={{{effects_str}}}"] + if self.model_effects: + model_str = ", ".join(type(e).__name__ for e in self.model_effects) + parts.append(f"model_effects=[{model_str}]") + return f"ChaosCase({', '.join(parts)})" diff --git a/src/strands_evals/chaos/effects.py b/src/strands_evals/chaos/effects.py index 9fc5c44b..ca0997ad 100644 --- a/src/strands_evals/chaos/effects.py +++ b/src/strands_evals/chaos/effects.py @@ -16,7 +16,7 @@ import random import re from abc import abstractmethod -from typing import Annotated, Any, ClassVar, Literal, Union +from typing import Annotated, Any, Callable, ClassVar, Literal, Union from pydantic import BaseModel, Discriminator, Field, Tag @@ -344,11 +344,11 @@ class ModelEffect(ChaosEffect): # --------------------------------------------------------------------------- -# Helper — module-level (used by multiple model effects) +# Helper — module-level (used by text-transforming model effects) # --------------------------------------------------------------------------- -def _map_text_in_blocks(blocks: list, fn: Any) -> list: +def _apply_text_to_blocks(blocks: list, fn: Callable[[str], str]) -> list: """Apply *fn* to every ``"text"`` value; leave other blocks untouched.""" result = [] for block in blocks: @@ -367,6 +367,7 @@ def _map_text_in_blocks(blocks: list, fn: Any) -> list: class MalformedJson(ModelEffect): """Corrupts JSON structures in model output.""" + hook: ClassVar[Literal["pre", "post"]] = "post" effect_type: Literal["malformed_json"] = "malformed_json" def apply(self, content: Any = None) -> Any: @@ -409,6 +410,7 @@ def _malform_blocks(blocks: list) -> list: class EmptyResponse(ModelEffect): """Returns empty content.""" + hook: ClassVar[Literal["pre", "post"]] = "post" effect_type: Literal["empty_response"] = "empty_response" def apply(self, content: Any = None) -> Any: @@ -429,6 +431,7 @@ def apply(self, content: Any = None) -> Any: class Confabulation(ModelEffect): """Injects fabricated citations into model output text.""" + hook: ClassVar[Literal["pre", "post"]] = "post" effect_type: Literal["confabulation"] = "confabulation" _CONFABULATION_TEMPLATES: ClassVar[list[str]] = [ @@ -448,7 +451,7 @@ def apply(self, content: Any = None) -> Any: if isinstance(content, str): return self._confabulate(content) elif isinstance(content, list): - return _map_text_in_blocks(content, self._confabulate) + return _apply_text_to_blocks(content, self._confabulate) raise ValueError(f"Confabulation.apply() received unsupported type {type(content).__name__}") def _confabulate(self, text: str) -> str: @@ -471,6 +474,7 @@ def _confabulate(self, text: str) -> str: class FullRefusal(ModelEffect): """Replaces model output with a refusal message.""" + hook: ClassVar[Literal["pre", "post"]] = "pre" effect_type: Literal["full_refusal"] = "full_refusal" _REFUSAL_TEMPLATES: ClassVar[list[str]] = [ @@ -482,6 +486,10 @@ class FullRefusal(ModelEffect): "I can't assist with the request as described. Could you provide more context?", ] + def cancel_message(self) -> str: + """Return a random refusal template string for use with event.cancel.""" + return random.choice(self._REFUSAL_TEMPLATES) + def apply(self, content: Any = None) -> Any: if content is None: raise ValueError("FullRefusal.apply() requires content") @@ -504,6 +512,7 @@ class SuccessFraming(ModelEffect): This is composable — applied AFTER another effect to disguise corruption. """ + hook: ClassVar[Literal["pre", "post"]] = "post" effect_type: Literal["success_framing"] = "success_framing" _SUCCESS_PREFIXES: ClassVar[list[str]] = [ diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index f48bb843..c4d798d7 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -5,32 +5,41 @@ - BeforeToolCallEvent: cancels tool calls for pre-hook effects (Timeout, etc.) - AfterToolCallEvent: corrupts tool responses for post-hook effects (TruncateFields, etc.) -- MessageAddedEvent: corrupts model output for the final assistant response +- BeforeModelCallEvent: cancels model call for pre-hook effects (FullRefusal) +- MessageAddedEvent: corrupts model output for post-hook effects (EmptyResponse, etc.) Model output corruption uses dict mutation on event.message["content"]. This is necessary because AfterModelCallEvent.stop_response is read-only (only `retry` is writeable). Dict mutation bypasses _can_write (which only intercepts __setattr__). -The model-chaos callback is GUARDED to only corrupt the FINAL agent response: +The post-model-chaos callback is GUARDED to only corrupt the FINAL agent response: - role == "assistant" - message content contains NO toolUse blocks This guard prevents destructive effects (EMPTY_RESPONSE, FULL_REFUSAL) from deleting toolUse blocks on mid-turn tool_use messages, which would break the agent loop. MessageAddedEvent fires BEFORE tool dispatch. + +The pre-model-chaos callback uses BeforeModelCallEvent.cancel to inject a refusal +message before the model is called. The cancel path builds an assistant message, +sets stop_reason="end_turn", and ends the agent cycle — no guard needed since +there is no existing message at pre time. """ import json import logging -from strands.hooks import AfterToolCallEvent, BeforeToolCallEvent, MessageAddedEvent +from strands.hooks import ( + AfterToolCallEvent, + BeforeModelCallEvent, + BeforeToolCallEvent, + MessageAddedEvent, +) from strands.plugins import Plugin, hook from ._context import _current_chaos_case from .effects import ( ChaosEffect, - ModelEffect, - ModelEffectUnion, SuccessFraming, TruncateFields, ) @@ -48,30 +57,30 @@ class ChaosPlugin(Plugin): - AfterToolCallEvent: corrupts tool responses for post-hook effects Model output chaos: - - MessageAddedEvent: corrupts the final assistant response content + - BeforeModelCallEvent: cancels model call for pre-hook effects (FullRefusal) + - MessageAddedEvent: corrupts the final assistant response content (post effects) The active ChaosCase is managed via a ContextVar (set by ChaosExperiment). - When no ChaosCase is active or the case has no effects, all tools behave normally. + When no ChaosCase is active or the case has no model_effects, all hooks + pass through without modification. - Model output corruption is configured via `model_effects` on the ChaosPlugin - instance. Effects are applied sequentially. SuccessFraming is always applied - LAST (composable post-step). + Model output effects are configured via `model_effects` on the ChaosCase. + Effects are applied sequentially. SuccessFraming is always applied LAST + (composable post-step). Example:: from strands import Agent - from strands_evals.chaos import ChaosPlugin - from strands_evals.chaos.effects import EmptyResponse, SuccessFraming - - chaos = ChaosPlugin( - model_effects=[EmptyResponse(), SuccessFraming()], - ) + from strands_evals.chaos import ChaosCase, ChaosPlugin + from strands_evals.chaos.effects import FullRefusal, EmptyResponse - agent = Agent( - model=my_model, - tools=[search_tool, database_tool], - plugins=[chaos], + chaos_case = ChaosCase( + name="refusal_test", + input="Tell me about quantum physics", + model_effects=[FullRefusal()], ) + chaos = ChaosPlugin() + agent = Agent(model=my_model, tools=[...], plugins=[chaos]) NOTE on structured_output: When the agent uses structured_output_model, the final response is a toolUse block (containing the structured output tool call). @@ -83,28 +92,6 @@ class ChaosPlugin(Plugin): name = "chaos-testing" - def __init__(self, model_effects: list[ModelEffectUnion] | None = None) -> None: - """Initialize the ChaosPlugin. - - Args: - model_effects: Optional list of model-output effects to apply sequentially. - When provided, enables model-output chaos on final assistant responses. - SuccessFraming (if included) is always applied last regardless of list order. - When None, only tool-level chaos (from ChaosCase effects) is active. - """ - super().__init__() - self._model_effects = model_effects - - @property - def model_effects(self) -> list[ModelEffectUnion] | None: - """The active model output effects list.""" - return self._model_effects - - @model_effects.setter - def model_effects(self, value: list[ModelEffectUnion] | None) -> None: - """Update the model output effects list.""" - self._model_effects = value - # ----------------------------------------------------------------------- # Tool chaos hooks # ----------------------------------------------------------------------- @@ -166,11 +153,35 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: logger.info("effect=<%s>, tool=<%s> | applied chaos post-hook", type(effect).__name__, tool_name) # ----------------------------------------------------------------------- - # Model output chaos hook + # Model output chaos hooks # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] - def message_added(self, event: MessageAddedEvent) -> None: + def before_model_invocation(self, event: BeforeModelCallEvent) -> None: + """Intercept model calls to inject pre-hook effects (FullRefusal). + + For pre-hook effects, cancels the model call by setting event.cancel + to the effect's cancel_message. The SDK builds an assistant message + from the cancel text, sets stop_reason="end_turn", and ends the cycle. + + No role/toolUse guard is needed here — there is no message yet at pre time. + """ + chaos_case = _current_chaos_case.get() + if chaos_case is None or not chaos_case.model_effects: + return + + pre = [e for e in chaos_case.model_effects if e.hook == "pre"] + if not pre: + return + + # First pre effect wins (cancel short-circuits, only one can win) + first_pre = pre[0] + if hasattr(first_pre, "cancel_message"): + event.cancel = first_pre.cancel_message() + logger.info("effect=<%s> | injected model pre-hook (cancel)", type(first_pre).__name__) + + @hook # type: ignore[call-overload] + def after_model_invocation(self, event: MessageAddedEvent) -> None: """Intercept messages to corrupt the final assistant response. GUARD: corruption is applied ONLY when ALL conditions hold: @@ -181,13 +192,18 @@ def message_added(self, event: MessageAddedEvent) -> None: MessageAddedEvent fires BEFORE the agent extracts toolUse blocks for execution, so corrupting a tool_use message would break the agent loop. + Only post-hook effects are applied here. If no post effects exist (e.g. + only FullRefusal in model_effects), this returns early to prevent + double-corruption in mixed pre+post cases. + NOTE: stop_reason is NOT available on MessageAddedEvent (the event only carries `message: Message`). We use the toolUse-presence check as a proxy: messages with toolUse blocks are mid-turn tool_use messages; messages without are final end_turn responses. This is reliable because end_turn messages never contain toolUse blocks. """ - if self._model_effects is None: + chaos_case = _current_chaos_case.get() + if chaos_case is None or not chaos_case.model_effects: return message = event.message @@ -206,9 +222,20 @@ def message_added(self, event: MessageAddedEvent) -> None: if isinstance(block, dict) and "toolUse" in block: return + # If any pre effects exist, they already produced the turn — skip post + # to prevent double-corruption in mixed pre+post cases. + pre_effects = [e for e in chaos_case.model_effects if e.hook == "pre"] + if pre_effects: + return + + # Filter to post effects only + post_effects = [e for e in chaos_case.model_effects if e.hook == "post"] + if not post_effects: + return + # Separate SuccessFraming from primary effects (applied last) - primary_effects: list[ModelEffect] = [e for e in self._model_effects if not isinstance(e, SuccessFraming)] - framing_effects: list[ModelEffect] = [e for e in self._model_effects if isinstance(e, SuccessFraming)] + primary_effects: list = [e for e in post_effects if not isinstance(e, SuccessFraming)] + framing_effects: list = [e for e in post_effects if isinstance(e, SuccessFraming)] # Apply primary effects sequentially corrupted = content @@ -222,7 +249,7 @@ def message_added(self, event: MessageAddedEvent) -> None: # Mutate via dict assignment (NOT attribute assignment on the event) message["content"] = corrupted - effect_names = ", ".join(type(e).__name__ for e in self._model_effects) + effect_names = ", ".join(type(e).__name__ for e in post_effects) logger.info( "effects=<%s> | applied model output chaos to assistant message", effect_names, diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py index 113c6e46..306d035f 100644 --- a/tests/strands_evals/chaos/test_model_chaos.py +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -11,6 +11,8 @@ import copy from unittest.mock import MagicMock +from strands_evals.chaos._context import _current_chaos_case +from strands_evals.chaos.case import ChaosCase from strands_evals.chaos.effects import ( Confabulation, EmptyResponse, @@ -63,6 +65,17 @@ def _tool_result_message() -> dict: } +def _set_chaos_case(model_effects): + """Helper to set the _current_chaos_case ContextVar with given model_effects.""" + case = ChaosCase( + name="test_case", + input="test input", + model_effects=model_effects, + ) + _current_chaos_case.set(case) + return case + + # --------------------------------------------------------------------------- # Per-effect corruption tests on final end_turn assistant messages # --------------------------------------------------------------------------- @@ -72,74 +85,96 @@ class TestModelChaosFormatCorruptionMalformedJson: """MALFORMED_JSON effect corrupts the final assistant message.""" def test_malformed_json_corrupts_json_text(self): - plugin = ChaosPlugin(model_effects=[MalformedJson()]) + _set_chaos_case([MalformedJson()]) + plugin = ChaosPlugin() message = _final_assistant_message('{"key": "value", "nested": {"a": 1}}') event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) # Content should be corrupted (JSON truncated) result_text = message["content"][0]["text"] assert result_text != '{"key": "value", "nested": {"a": 1}}' + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosFormatCorruptionEmptyResponse: """EMPTY_RESPONSE effect empties the final assistant message content.""" def test_empty_response_on_final_message(self): - plugin = ChaosPlugin(model_effects=[EmptyResponse()]) + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() message = _final_assistant_message("Hello world") event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) assert message["content"] == [] + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosHallucination: """Hallucination effect corrupts the final assistant message text.""" def test_confabulation_injects_template(self): - plugin = ChaosPlugin(model_effects=[Confabulation()]) + _set_chaos_case([Confabulation()]) + plugin = ChaosPlugin() original_text = "The weather is sunny. It is warm outside. Birds are singing." message = _final_assistant_message(original_text) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) result_text = message["content"][0]["text"] assert result_text != original_text # Should contain original text fragments assert "sunny" in result_text or "warm" in result_text + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosRefusal: - """FULL_REFUSAL replaces the final assistant message content with a refusal.""" + """FULL_REFUSAL is a pre-hook effect — it uses before_model_invocation.""" - def test_refusal_replaces_content(self): - plugin = ChaosPlugin(model_effects=[FullRefusal()]) - message = _final_assistant_message("Here is the code you requested...") - event = _make_event(message) + def test_refusal_cancels_model_call(self): + _set_chaos_case([FullRefusal()]) + plugin = ChaosPlugin() + event = MagicMock() - plugin.message_added(event) + plugin.before_model_invocation(event) - # Content should be a single refusal text block - assert len(message["content"]) == 1 - assert message["content"][0]["text"] in FullRefusal._REFUSAL_TEMPLATES + # event.cancel should be set to a refusal template string + assert event.cancel in FullRefusal._REFUSAL_TEMPLATES + + def teardown_method(self): + _current_chaos_case.set(None) class TestModelChaosSuccessFraming: """Success framing prepends a confident prefix after corruption.""" - def test_success_framing_with_refusal(self): - plugin = ChaosPlugin(model_effects=[FullRefusal(), SuccessFraming()]) + def test_success_framing_with_empty_response(self): + # SuccessFraming is a post-hook composable effect + _set_chaos_case([EmptyResponse(), SuccessFraming()]) + plugin = ChaosPlugin() message = _final_assistant_message("Here is the code you requested...") event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) + # EmptyResponse clears content to [], then SuccessFraming prepends a + # prefix block (disguises the emptied response with confident framing) + assert len(message["content"]) == 1 result_text = message["content"][0]["text"] - assert any(result_text.startswith(prefix) for prefix in SuccessFraming._SUCCESS_PREFIXES) + assert result_text in SuccessFraming._SUCCESS_PREFIXES + + def teardown_method(self): + _current_chaos_case.set(None) # --------------------------------------------------------------------------- @@ -152,12 +187,13 @@ class TestModelChaosGuardToolUseMessage: def test_empty_response_on_tooluse_message_not_corrupted(self): """EMPTY_RESPONSE on a tool_use message passes through; toolUse intact.""" - plugin = ChaosPlugin(model_effects=[EmptyResponse()]) + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() message = _tooluse_assistant_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) # Content should be UNCHANGED — guard skipped corruption assert message["content"] == original_content @@ -166,24 +202,30 @@ def test_empty_response_on_tooluse_message_not_corrupted(self): assert len(tool_blocks) == 1 assert tool_blocks[0]["toolUse"]["name"] == "search" - def test_full_refusal_on_tooluse_message_not_corrupted(self): - """FULL_REFUSAL on a tool_use message passes through; toolUse intact.""" - plugin = ChaosPlugin(model_effects=[FullRefusal()]) + def test_full_refusal_does_not_affect_post_hook(self): + """FullRefusal is pre-hook only — after_model_invocation with toolUse still passes through.""" + _set_chaos_case([FullRefusal()]) + plugin = ChaosPlugin() message = _tooluse_assistant_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) + # FullRefusal is pre-hook, so post hook has no post effects to apply assert message["content"] == original_content + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosGuardStructuredOutputPath: """structured_output_model path fires MessageAddedEvent with toolUse — guard skips it.""" def test_structured_output_tool_message_not_corrupted(self): """A message containing the structured output tool call is NOT corrupted.""" - plugin = ChaosPlugin(model_effects=[EmptyResponse()]) + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() # Simulate the structured output tool invocation message message = { "role": "assistant", @@ -200,59 +242,137 @@ def test_structured_output_tool_message_not_corrupted(self): original_content = copy.deepcopy(message["content"]) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) # Guard should skip — toolUse block present assert message["content"] == original_content + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosGuardFinalMessage: """Final end_turn assistant message IS corrupted.""" def test_final_end_turn_message_corrupted(self): """An end_turn message with text only (no toolUse) gets corrupted.""" - plugin = ChaosPlugin(model_effects=[EmptyResponse()]) + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() message = _final_assistant_message("Hello world") event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) assert message["content"] == [] + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosGuardRoleFiltering: """User and tool result messages are NOT corrupted.""" def test_user_message_not_corrupted(self): - plugin = ChaosPlugin(model_effects=[EmptyResponse()]) + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() message = _user_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) assert message["content"] == original_content def test_tool_result_message_not_corrupted(self): - plugin = ChaosPlugin(model_effects=[EmptyResponse()]) + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() message = _tool_result_message() original_content = copy.deepcopy(message["content"]) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) assert message["content"] == original_content + def teardown_method(self): + _current_chaos_case.set(None) + class TestModelChaosPassthrough: """No corruption when no model_effects is set.""" def test_no_config_passes_through(self): - plugin = ChaosPlugin() # No model_effects + # No chaos case set — plugin should pass through + _current_chaos_case.set(None) + plugin = ChaosPlugin() message = _final_assistant_message("Hello world") original_content = copy.deepcopy(message["content"]) event = _make_event(message) - plugin.message_added(event) + plugin.after_model_invocation(event) assert message["content"] == original_content + + +# --------------------------------------------------------------------------- +# Pre-hook integration test +# --------------------------------------------------------------------------- + + +class TestModelChaosPreHookIntegration: + """Integration test: FullRefusal pre-hook produces one assistant turn.""" + + def test_full_refusal_produces_single_turn(self): + """FullRefusal cancels model call, SDK builds cancel message, run ends.""" + _set_chaos_case([FullRefusal()]) + plugin = ChaosPlugin() + + # Step 1: before_model_invocation fires + pre_event = MagicMock() + plugin.before_model_invocation(pre_event) + cancel_text = pre_event.cancel + assert cancel_text in FullRefusal._REFUSAL_TEMPLATES + + # Step 2: SDK builds the cancel message and fires MessageAddedEvent + cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} + post_event = _make_event(cancel_message) + plugin.after_model_invocation(post_event) + + # Step 3: verify the cancel message is unchanged (not double-corrupted) + assert cancel_message["content"] == [{"text": cancel_text}] + assert len(cancel_message["content"]) == 1 + + def teardown_method(self): + _current_chaos_case.set(None) + + +# --------------------------------------------------------------------------- +# Mixed pre+post case test +# --------------------------------------------------------------------------- + + +class TestModelChaosMixedCase: + """Mixed pre+post effects: pre wins, post does NOT double-corrupt.""" + + def test_pre_plus_post_produces_single_uncorrupted_turn(self): + """ChaosCase with FullRefusal + EmptyResponse: pre cancels, post skipped.""" + _set_chaos_case([FullRefusal(), EmptyResponse()]) + plugin = ChaosPlugin() + + # Pre-hook fires and cancels + pre_event = MagicMock() + plugin.before_model_invocation(pre_event) + cancel_text = pre_event.cancel + assert cancel_text in FullRefusal._REFUSAL_TEMPLATES + + # SDK builds cancel message, MessageAddedEvent fires + cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} + post_event = _make_event(cancel_message) + plugin.after_model_invocation(post_event) + + # Post effect (EmptyResponse) should NOT have emptied the content + assert cancel_message["content"] == [{"text": cancel_text}] + assert len(cancel_message["content"]) == 1 + + def teardown_method(self): + _current_chaos_case.set(None) From 670973c51646ae4d469708991fcb3f0514591a84 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Thu, 9 Jul 2026 07:05:16 +0000 Subject: [PATCH 5/8] - Change 1: Relax Guard 2 so MalformedJson reaches structured-output toolUse blocks (corrupts tool input JSON). Other post effects still skip toolUse messages. Guard 2 checks event.agent.tool_registry.dynamic_tools to identify structured-output toolUse blocks. Only those are corrupted by MalformedJson; ordinary mid-turn toolUse is left untouched. - Change 2: Move model_effects into keyed effects dict (effects={'model_effects': {'*': [...]}}) mirroring tool_effects. Remove flat sibling field. '*' wildcard resolves to all models. - Change 3: Rename _apply_to_blocks -> _apply_to_tool_blocks. Add _apply_to_model_blocks for model post effects. Delete the short _apply_text_to_blocks helper (folded into model blocks method). Added _apply_malformed_json_selective helper that skips toolUse blocks whose name is not in dynamic_tools. - Change 4: EmptyResponse becomes pre-hook using cancel=' ' (single space, truthy). Now two pre effects: FullRefusal and EmptyResponse. Test calibration: propagation assertion used (structured_output_model runs through normal event loop, fires MessageAddedEvent). Tests: plain-named toolUse asserted UNCHANGED; only SO toolUse corrupted; mixed message (regular + SO) selectively corrupts. --- src/strands_evals/chaos/case.py | 80 ++- src/strands_evals/chaos/effects.py | 32 +- src/strands_evals/chaos/plugin.py | 167 +++--- tests/strands_evals/chaos/test_model_chaos.py | 476 ++++++++++++------ 4 files changed, 468 insertions(+), 287 deletions(-) diff --git a/src/strands_evals/chaos/case.py b/src/strands_evals/chaos/case.py index 6ffc442c..185cf5a8 100644 --- a/src/strands_evals/chaos/case.py +++ b/src/strands_evals/chaos/case.py @@ -6,6 +6,7 @@ """ import uuid +from typing import cast from pydantic import Field, model_validator from typing_extensions import Generic @@ -14,6 +15,9 @@ from ..types.evaluation import InputT, OutputT from .effects import ModelEffectUnion, ToolEffectUnion +# Type alias for the effects dict structure +EffectsDict = dict[str, dict[str, list[ToolEffectUnion | ModelEffectUnion]]] + class ChaosCase(Case, Generic[InputT, OutputT]): """A test case with associated chaos effects. @@ -26,16 +30,26 @@ class ChaosCase(Case, Generic[InputT, OutputT]): ChaosExperiment. Attributes: - effects: A dict keyed by effect category. Currently supports - ``"tool_effects"`` mapping tool_name -> list of effects. + effects: A dict keyed by effect category. Supports ``"tool_effects"`` + mapping tool_name -> list of effects, and ``"model_effects"`` + mapping model_name (or ``"*"`` wildcard) -> list of effects. Example:: from strands_evals import Case from strands_evals.chaos import ChaosCase - from strands_evals.chaos.effects import Timeout, TruncateFields + from strands_evals.chaos.effects import FullRefusal, Timeout, TruncateFields + + # Direct construction with model effects + chaos_case = ChaosCase( + name="refusal_test", + input="Tell me something", + effects={ + "model_effects": {"*": [FullRefusal()]}, + }, + ) - # Direct construction + # Direct construction with tool effects chaos_case = ChaosCase( name="search_timeout", input="Find flights to Tokyo", @@ -55,41 +69,53 @@ class ChaosCase(Case, Generic[InputT, OutputT]): # Produces 6 ChaosCase objects: 2 cases × (2 effect maps + 1 baseline) """ - effects: dict[str, dict[str, list[ToolEffectUnion]]] = Field( + effects: EffectsDict = Field( default_factory=dict, - description="Effect categories. Currently supports 'tool_effects' mapping " - "tool_name -> list of effects. Empty dict means baseline (no chaos).", - ) - - model_effects: list[ModelEffectUnion] = Field( - default_factory=list, - description="List of model output effects to apply. FullRefusal is pre-hook " - "(cancels model call); others are post-hook (corrupt response).", + description="Effect categories. Supports 'tool_effects' mapping " + "tool_name -> list of effects, and 'model_effects' mapping " + "model_name (or '*' wildcard) -> list of effects. " + "Empty dict means baseline (no chaos).", ) @model_validator(mode="after") - def _validate_tool_effects(self) -> "ChaosCase": - """Validate tool effects configuration.""" - allowed_categories = {"tool_effects"} + def _validate_effects(self) -> "ChaosCase": + """Validate effects configuration structure.""" + allowed_categories = {"tool_effects", "model_effects"} unknown = set(self.effects.keys()) - allowed_categories if unknown: raise ValueError( f"Unknown effect categories: {sorted(unknown)}. Allowed categories: {sorted(allowed_categories)}." ) + # Validate tool_effects: dict[str, list[ToolEffectUnion]] for tool_name, effects_list in self.tool_effects.items(): if len(effects_list) > 1: raise ValueError( f"Tool '{tool_name}' has {len(effects_list)} effects — only 1 is allowed per " f"ChaosCase. Use separate ChaosCase instances to test effects independently." ) + + # Validate model_effects: dict[str, list[ModelEffectUnion]] + model_effects_map = self.effects.get("model_effects", {}) + if model_effects_map: + if not isinstance(model_effects_map, dict): + raise ValueError("'model_effects' must be a dict keyed by model name (or '*' wildcard).") + for model_name, effects_list in model_effects_map.items(): # type: ignore[assignment] + if not isinstance(model_name, str): + raise ValueError(f"model_effects keys must be strings, got {type(model_name).__name__}.") + if not isinstance(effects_list, list): + raise ValueError( + f"model_effects['{model_name}'] must be a list of model effects, " + f"got {type(effects_list).__name__}." + ) + return self @classmethod def expand( cls, cases: list[Case], - effect_maps: dict[str, dict[str, dict[str, list[ToolEffectUnion]]]], + effect_maps: dict[str, EffectsDict], include_no_effect_baseline: bool = False, ) -> list["ChaosCase"]: """Generate the Cartesian product of cases × named effect maps. @@ -102,14 +128,17 @@ def expand( cases: Base test cases to expand. effect_maps: Named effect configurations. Keys are short human-readable names (used in the composite case name); values are dicts keyed by - effect category (e.g. ``"tool_effects"``) mapping tool_name -> list - of effect instances. + effect category (e.g. ``"tool_effects"``, ``"model_effects"``) + mapping target -> list of effect instances. Example:: { "search_timeout": { "tool_effects": {"search_tool": [Timeout()]} }, + "refusal": { + "model_effects": {"*": [FullRefusal()]} + }, } include_no_effect_baseline: If True, includes a baseline (no chaos) variant for each case. Defaults to False. @@ -118,7 +147,7 @@ def expand( Flat list of ChaosCase objects with composite names like "flight_search|baseline" or "flight_search|search_timeout". """ - all_entries: list[tuple[str, dict[str, dict[str, list[ToolEffectUnion]]]]] = [] + all_entries: list[tuple[str, EffectsDict]] = [] if include_no_effect_baseline: all_entries.append(("baseline", {})) @@ -152,7 +181,16 @@ def expand( @property def tool_effects(self) -> dict[str, list[ToolEffectUnion]]: """Convenience accessor for effects['tool_effects'].""" - return self.effects.get("tool_effects", {}) + return cast(dict[str, list[ToolEffectUnion]], self.effects.get("tool_effects", {})) + + @property + def model_effects(self) -> list[ModelEffectUnion]: + """Resolve model effects. '*' wildcard applies to all models.""" + model_effects_map = self.effects.get("model_effects", {}) + if not model_effects_map: + return [] + # For now, resolve "*" (wildcard = applies to all models) + return cast(list[ModelEffectUnion], model_effects_map.get("*", [])) def __repr__(self) -> str: effects_str = ", ".join( diff --git a/src/strands_evals/chaos/effects.py b/src/strands_evals/chaos/effects.py index ca0997ad..421b3929 100644 --- a/src/strands_evals/chaos/effects.py +++ b/src/strands_evals/chaos/effects.py @@ -16,7 +16,7 @@ import random import re from abc import abstractmethod -from typing import Annotated, Any, Callable, ClassVar, Literal, Union +from typing import Annotated, Any, ClassVar, Literal, Union from pydantic import BaseModel, Discriminator, Field, Tag @@ -343,22 +343,6 @@ class ModelEffect(ChaosEffect): hook: ClassVar[Literal["pre", "post"]] = "post" -# --------------------------------------------------------------------------- -# Helper — module-level (used by text-transforming model effects) -# --------------------------------------------------------------------------- - - -def _apply_text_to_blocks(blocks: list, fn: Callable[[str], str]) -> list: - """Apply *fn* to every ``"text"`` value; leave other blocks untouched.""" - result = [] - for block in blocks: - block = dict(block) - if "text" in block and isinstance(block["text"], str): - block["text"] = fn(block["text"]) - result.append(block) - return result - - # --------------------------------------------------------------------------- # a) MalformedJson # --------------------------------------------------------------------------- @@ -410,9 +394,13 @@ def _malform_blocks(blocks: list) -> list: class EmptyResponse(ModelEffect): """Returns empty content.""" - hook: ClassVar[Literal["pre", "post"]] = "post" + hook: ClassVar[Literal["pre", "post"]] = "pre" effect_type: Literal["empty_response"] = "empty_response" + def cancel_message(self) -> str: + # Pre-cancel with single space (truthy) skips real model call — "model returned nothing". + return " " + def apply(self, content: Any = None) -> Any: if content is None: raise ValueError("EmptyResponse.apply() requires content") @@ -451,7 +439,13 @@ def apply(self, content: Any = None) -> Any: if isinstance(content, str): return self._confabulate(content) elif isinstance(content, list): - return _apply_text_to_blocks(content, self._confabulate) + result = [] + for block in content: + block = dict(block) + if "text" in block and isinstance(block["text"], str): + block["text"] = self._confabulate(block["text"]) + result.append(block) + return result raise ValueError(f"Confabulation.apply() received unsupported type {type(content).__name__}") def _confabulate(self, text: str) -> str: diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index c4d798d7..ec8077cd 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -5,25 +5,8 @@ - BeforeToolCallEvent: cancels tool calls for pre-hook effects (Timeout, etc.) - AfterToolCallEvent: corrupts tool responses for post-hook effects (TruncateFields, etc.) -- BeforeModelCallEvent: cancels model call for pre-hook effects (FullRefusal) -- MessageAddedEvent: corrupts model output for post-hook effects (EmptyResponse, etc.) - -Model output corruption uses dict mutation on event.message["content"]. This is -necessary because AfterModelCallEvent.stop_response is read-only (only `retry` -is writeable). Dict mutation bypasses _can_write (which only intercepts __setattr__). - -The post-model-chaos callback is GUARDED to only corrupt the FINAL agent response: -- role == "assistant" -- message content contains NO toolUse blocks - -This guard prevents destructive effects (EMPTY_RESPONSE, FULL_REFUSAL) from -deleting toolUse blocks on mid-turn tool_use messages, which would break the -agent loop. MessageAddedEvent fires BEFORE tool dispatch. - -The pre-model-chaos callback uses BeforeModelCallEvent.cancel to inject a refusal -message before the model is called. The cancel path builds an assistant message, -sets stop_reason="end_turn", and ends the agent cycle — no guard needed since -there is no existing message at pre time. +- BeforeModelCallEvent: cancels model call for pre-hook effects (FullRefusal, EmptyResponse) +- MessageAddedEvent: corrupts model output for post-hook effects (MalformedJson, Confabulation, etc.) """ import json @@ -40,6 +23,7 @@ from ._context import _current_chaos_case from .effects import ( ChaosEffect, + MalformedJson, SuccessFraming, TruncateFields, ) @@ -57,7 +41,7 @@ class ChaosPlugin(Plugin): - AfterToolCallEvent: corrupts tool responses for post-hook effects Model output chaos: - - BeforeModelCallEvent: cancels model call for pre-hook effects (FullRefusal) + - BeforeModelCallEvent: cancels model call for pre-hook effects (FullRefusal, EmptyResponse) - MessageAddedEvent: corrupts the final assistant response content (post effects) The active ChaosCase is managed via a ContextVar (set by ChaosExperiment). @@ -66,7 +50,8 @@ class ChaosPlugin(Plugin): Model output effects are configured via `model_effects` on the ChaosCase. Effects are applied sequentially. SuccessFraming is always applied LAST - (composable post-step). + (composable post-step). MalformedJson can reach structured-output toolUse + blocks; other post effects skip toolUse messages. Example:: @@ -77,17 +62,12 @@ class ChaosPlugin(Plugin): chaos_case = ChaosCase( name="refusal_test", input="Tell me about quantum physics", - model_effects=[FullRefusal()], + effects={ + "model_effects": {"*": [FullRefusal()]}, + }, ) chaos = ChaosPlugin() agent = Agent(model=my_model, tools=[...], plugins=[chaos]) - - NOTE on structured_output: When the agent uses structured_output_model, the - final response is a toolUse block (containing the structured output tool call). - The toolUse guard will skip these messages, so model chaos does NOT affect - structured_output responses. This is intentional — corrupting the structured - output tool call would break parsing. Future work may add a dedicated - structured_output chaos effect that corrupts the tool input fields specifically. """ name = "chaos-testing" @@ -100,9 +80,7 @@ class ChaosPlugin(Plugin): def before_tool_call(self, event: BeforeToolCallEvent) -> None: """Intercept tool calls to inject pre-hook (error) effects. - For pre-hook effects (Timeout, NetworkError, ExecutionError, - ValidationError), cancels the tool call with the effect's error_message - before the tool executes. + Cancels the tool call with the effect's error_message before execution. """ chaos_case = _current_chaos_case.get() if chaos_case is None or not chaos_case.tool_effects: @@ -124,8 +102,7 @@ def before_tool_call(self, event: BeforeToolCallEvent) -> None: def after_tool_call(self, event: AfterToolCallEvent) -> None: """Intercept tool results to inject post-hook (corruption) effects. - For corruption effects (TruncateFields, RemoveFields, CorruptValues), - applies effect.apply() to JSON content blocks in the tool response. + Applies corruption effects to JSON content blocks in the tool response. """ chaos_case = _current_chaos_case.get() if chaos_case is None or not chaos_case.tool_effects: @@ -148,7 +125,7 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: content = result.get("content") if isinstance(content, list): - result["content"] = self._apply_to_blocks(effect, content) # type: ignore[assignment] + result["content"] = self._apply_to_tool_blocks(effect, content) # type: ignore[assignment] logger.info("effect=<%s>, tool=<%s> | applied chaos post-hook", type(effect).__name__, tool_name) @@ -158,13 +135,10 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: @hook # type: ignore[call-overload] def before_model_invocation(self, event: BeforeModelCallEvent) -> None: - """Intercept model calls to inject pre-hook effects (FullRefusal). + """Intercept model calls to inject pre-hook effects (FullRefusal, EmptyResponse). - For pre-hook effects, cancels the model call by setting event.cancel - to the effect's cancel_message. The SDK builds an assistant message - from the cancel text, sets stop_reason="end_turn", and ends the cycle. - - No role/toolUse guard is needed here — there is no message yet at pre time. + Cancels the model call by setting event.cancel to the effect's cancel_message. + No role/toolUse guard is needed — there is no message yet at pre time. """ chaos_case = _current_chaos_case.get() if chaos_case is None or not chaos_case.model_effects: @@ -184,23 +158,8 @@ def before_model_invocation(self, event: BeforeModelCallEvent) -> None: def after_model_invocation(self, event: MessageAddedEvent) -> None: """Intercept messages to corrupt the final assistant response. - GUARD: corruption is applied ONLY when ALL conditions hold: - 1. message role == "assistant" - 2. message content contains NO toolUse blocks - - This prevents destructive effects from breaking mid-turn tool dispatch. - MessageAddedEvent fires BEFORE the agent extracts toolUse blocks for - execution, so corrupting a tool_use message would break the agent loop. - - Only post-hook effects are applied here. If no post effects exist (e.g. - only FullRefusal in model_effects), this returns early to prevent - double-corruption in mixed pre+post cases. - - NOTE: stop_reason is NOT available on MessageAddedEvent (the event only - carries `message: Message`). We use the toolUse-presence check as a proxy: - messages with toolUse blocks are mid-turn tool_use messages; messages - without are final end_turn responses. This is reliable because end_turn - messages never contain toolUse blocks. + Guards ensure corruption is only applied to appropriate messages. + Only post-hook effects are applied here. """ chaos_case = _current_chaos_case.get() if chaos_case is None or not chaos_case.model_effects: @@ -212,18 +171,11 @@ def after_model_invocation(self, event: MessageAddedEvent) -> None: if message.get("role") != "assistant": return - # Guard 2: skip messages with toolUse blocks (mid-turn tool dispatch) content = message.get("content") if content is None: return - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and "toolUse" in block: - return - # If any pre effects exist, they already produced the turn — skip post - # to prevent double-corruption in mixed pre+post cases. pre_effects = [e for e in chaos_case.model_effects if e.hook == "pre"] if pre_effects: return @@ -233,20 +185,27 @@ def after_model_invocation(self, event: MessageAddedEvent) -> None: if not post_effects: return - # Separate SuccessFraming from primary effects (applied last) - primary_effects: list = [e for e in post_effects if not isinstance(e, SuccessFraming)] - framing_effects: list = [e for e in post_effects if isinstance(e, SuccessFraming)] - - # Apply primary effects sequentially - corrupted = content - for effect in primary_effects: - corrupted = effect.apply(corrupted) - - # Apply success framing last (composable post-step) - for effect in framing_effects: - corrupted = effect.apply(corrupted) - - # Mutate via dict assignment (NOT attribute assignment on the event) + # Guard 2: skip toolUse messages (mid-turn dispatch); exception: MalformedJson on structured-output toolUse. + if isinstance(content, list): + has_tool_use = any(isinstance(block, dict) and "toolUse" in block for block in content) + if has_tool_use: + if not any(isinstance(e, MalformedJson) for e in post_effects): + return + # Only allow through if toolUse is a structured-output tool (dynamic_tools) + so_tool_names = set(event.agent.tool_registry.dynamic_tools.keys()) + tool_use_names = { + block["toolUse"]["name"] for block in content if isinstance(block, dict) and "toolUse" in block + } + if not tool_use_names & so_tool_names: + return # No structured-output toolUse — skip + + # Resolve structured-output tool names for _apply_to_model_blocks + so_tool_names_set: set[str] = set() + if isinstance(content, list) and any(isinstance(b, dict) and "toolUse" in b for b in content): + so_tool_names_set = set(event.agent.tool_registry.dynamic_tools.keys()) + + # Apply model post effects + corrupted = self._apply_to_model_blocks(post_effects, content, so_tool_names_set) message["content"] = corrupted effect_names = ", ".join(type(e).__name__ for e in post_effects) @@ -255,12 +214,58 @@ def after_model_invocation(self, event: MessageAddedEvent) -> None: effect_names, ) + # ----------------------------------------------------------------------- + # Model corruption helpers + # ----------------------------------------------------------------------- + + def _apply_to_model_blocks(self, post_effects: list, content: list, so_tool_names: set[str] | None = None) -> list: + """Apply model post effects to content blocks sequentially. + + Handles text blocks (Confabulation, SuccessFraming, MalformedJson on text) + and toolUse blocks (MalformedJson on structured-output tool input only). + Ordinary mid-turn toolUse blocks are left untouched. + """ + primary = [e for e in post_effects if not isinstance(e, SuccessFraming)] + framing = [e for e in post_effects if isinstance(e, SuccessFraming)] + + corrupted = content + for effect in primary: + if isinstance(effect, MalformedJson) and so_tool_names: + corrupted = self._apply_malformed_json_selective(effect, corrupted, so_tool_names) + else: + corrupted = effect.apply(corrupted) + for effect in framing: + corrupted = effect.apply(corrupted) + return corrupted + + def _apply_malformed_json_selective(self, effect: MalformedJson, blocks: list, so_tool_names: set[str]) -> list: + """Apply MalformedJson only to structured-output toolUse blocks and text blocks. + + Ordinary mid-turn toolUse blocks are left untouched. + """ + result = [] + for block in blocks: + block = dict(block) + if "toolUse" in block: + tool_name = block["toolUse"].get("name", "") + if tool_name in so_tool_names: + # Structured-output toolUse — corrupt the input JSON + tool_use = dict(block["toolUse"]) + raw = json.dumps(tool_use.get("input", {})) + tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" + block["toolUse"] = tool_use + # else: ordinary mid-turn toolUse — leave untouched + elif "text" in block and isinstance(block["text"], str): + block["text"] = MalformedJson._malform_text(block["text"]) + result.append(block) + return result + # ----------------------------------------------------------------------- # Tool corruption helpers # ----------------------------------------------------------------------- - def _apply_to_blocks(self, effect: ChaosEffect, blocks: list) -> list: - """Apply effect to text blocks in a content list.""" + def _apply_to_tool_blocks(self, effect: ChaosEffect, blocks: list) -> list: + """Apply effect to text blocks in a tool content list.""" corrupted_blocks = [] for block in blocks: if isinstance(block, dict) and "text" in block: diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py index 306d035f..4adc1609 100644 --- a/tests/strands_evals/chaos/test_model_chaos.py +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -1,11 +1,13 @@ -"""Unit tests for model output chaos via ChaosPlugin MessageAddedEvent callback. +"""Unit tests for model output chaos via ChaosPlugin two-hook architecture. Tests cover: -- Per-effect corruption on final assistant messages (end_turn, no toolUse) -- Guard: toolUse-carrying messages are NOT corrupted -- Guard: user/tool messages are NOT corrupted -- Guard: passthrough when no config set -- structured_output_model path: messages with toolUse are skipped (deferred scope) +- 6.1: Effects constructed via keyed dict {"model_effects": {"*": [...]}} +- 6.2: EmptyResponse as pre-hook: model not called, turn is single space +- 6.3: FullRefusal as pre-hook: model not called, turn is refusal text +- 6.4: MalformedJson on structured-output toolUse: toolUse input corrupted +- 6.5: Post effects (Confabulation, MalformedJson-on-text, SuccessFraming) work +- 6.6: Mixed pre+post still produces one turn (pre wins) +- 6.7: MalformedJson DOES reach/corrupt structured-output toolUse """ import copy @@ -22,11 +24,22 @@ ) from strands_evals.chaos.plugin import ChaosPlugin +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_event(message: dict, dynamic_tools: dict | None = None) -> MagicMock: + """Create a mock MessageAddedEvent with the given message. -def _make_event(message: dict) -> MagicMock: - """Create a mock MessageAddedEvent with the given message.""" + Args: + message: The message dict. + dynamic_tools: Optional dict of dynamic tool names -> tools (structured-output tools). + If None, defaults to empty dict (no structured-output tools registered). + """ event = MagicMock() event.message = message + event.agent.tool_registry.dynamic_tools = dynamic_tools or {} return event @@ -66,61 +79,222 @@ def _tool_result_message() -> dict: def _set_chaos_case(model_effects): - """Helper to set the _current_chaos_case ContextVar with given model_effects.""" + """Helper to set the _current_chaos_case ContextVar with given model_effects. + + Uses keyed dict form: effects={"model_effects": {"*": model_effects}} + """ case = ChaosCase( name="test_case", input="test input", - model_effects=model_effects, + effects={"model_effects": {"*": model_effects}}, ) _current_chaos_case.set(case) return case # --------------------------------------------------------------------------- -# Per-effect corruption tests on final end_turn assistant messages +# 6.1: Effects constructed via keyed dict # --------------------------------------------------------------------------- -class TestModelChaosFormatCorruptionMalformedJson: - """MALFORMED_JSON effect corrupts the final assistant message.""" +class TestKeyedDictConstruction: + """Effects are constructed via keyed dict form.""" - def test_malformed_json_corrupts_json_text(self): - _set_chaos_case([MalformedJson()]) + def test_keyed_dict_form_is_valid(self): + """ChaosCase accepts effects={"model_effects": {"*": [...]}}.""" + case = ChaosCase( + name="keyed", + input="test", + effects={"model_effects": {"*": [MalformedJson()]}}, + ) + assert case.model_effects == [MalformedJson()] + + def test_wildcard_resolver(self): + """model_effects property resolves '*' wildcard to flat list.""" + case = ChaosCase( + name="wildcard", + input="test", + effects={"model_effects": {"*": [FullRefusal(), MalformedJson()]}}, + ) + assert len(case.model_effects) == 2 + assert isinstance(case.model_effects[0], FullRefusal) + assert isinstance(case.model_effects[1], MalformedJson) + + def test_empty_effects_baseline(self): + """Empty effects dict produces no model_effects.""" + case = ChaosCase(name="baseline", input="test", effects={}) + assert case.model_effects == [] + + +# --------------------------------------------------------------------------- +# 6.2: EmptyResponse as pre-hook +# --------------------------------------------------------------------------- + + +class TestEmptyResponsePreHook: + """EmptyResponse is a pre-hook effect — cancels model call with single space.""" + + def test_empty_response_cancels_with_single_space(self): + """before_model_invocation sets event.cancel to ' ' (single space).""" + _set_chaos_case([EmptyResponse()]) plugin = ChaosPlugin() - message = _final_assistant_message('{"key": "value", "nested": {"a": 1}}') - event = _make_event(message) + event = MagicMock() - plugin.after_model_invocation(event) + plugin.before_model_invocation(event) - # Content should be corrupted (JSON truncated) - result_text = message["content"][0]["text"] - assert result_text != '{"key": "value", "nested": {"a": 1}}' + # event.cancel should be set to single space + assert event.cancel == " " + + def test_empty_response_model_not_called(self): + """When EmptyResponse fires as pre-hook, post-hook does not apply effects.""" + _set_chaos_case([EmptyResponse()]) + plugin = ChaosPlugin() + + # Pre-hook fires + pre_event = MagicMock() + plugin.before_model_invocation(pre_event) + assert pre_event.cancel == " " + + # SDK builds cancel message, MessageAddedEvent fires + cancel_message = {"role": "assistant", "content": [{"text": " "}]} + post_event = _make_event(cancel_message) + plugin.after_model_invocation(post_event) + + # Content should be unchanged (pre effects skip post processing) + assert cancel_message["content"] == [{"text": " "}] def teardown_method(self): _current_chaos_case.set(None) -class TestModelChaosFormatCorruptionEmptyResponse: - """EMPTY_RESPONSE effect empties the final assistant message content.""" +# --------------------------------------------------------------------------- +# 6.3: FullRefusal as pre-hook (unchanged behavior) +# --------------------------------------------------------------------------- + + +class TestFullRefusalPreHook: + """FullRefusal is a pre-hook effect — cancels model call with refusal text.""" - def test_empty_response_on_final_message(self): - _set_chaos_case([EmptyResponse()]) + def test_full_refusal_cancels_model_call(self): + """before_model_invocation sets event.cancel to a refusal template.""" + _set_chaos_case([FullRefusal()]) plugin = ChaosPlugin() - message = _final_assistant_message("Hello world") - event = _make_event(message) + event = MagicMock() + + plugin.before_model_invocation(event) + + assert event.cancel in FullRefusal._REFUSAL_TEMPLATES + + def test_full_refusal_produces_single_turn(self): + """FullRefusal cancels model call, SDK builds cancel message, run ends.""" + _set_chaos_case([FullRefusal()]) + plugin = ChaosPlugin() + + # Step 1: before_model_invocation fires + pre_event = MagicMock() + plugin.before_model_invocation(pre_event) + cancel_text = pre_event.cancel + assert cancel_text in FullRefusal._REFUSAL_TEMPLATES + + # Step 2: SDK builds the cancel message and fires MessageAddedEvent + cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} + post_event = _make_event(cancel_message) + plugin.after_model_invocation(post_event) + + # Step 3: verify the cancel message is unchanged (not double-corrupted) + assert cancel_message["content"] == [{"text": cancel_text}] + assert len(cancel_message["content"]) == 1 + + def teardown_method(self): + _current_chaos_case.set(None) + + +# --------------------------------------------------------------------------- +# 6.4: MalformedJson on structured-output toolUse +# --------------------------------------------------------------------------- + + +class TestMalformedJsonStructuredOutput: + """MalformedJson DOES reach and corrupt structured-output toolUse blocks only.""" + + def test_malformed_json_corrupts_structured_output_tooluse(self): + """MalformedJson corrupts toolUse input when tool is in dynamic_tools (structured-output).""" + _set_chaos_case([MalformedJson()]) + plugin = ChaosPlugin() + message = { + "role": "assistant", + "content": [ + {"toolUse": {"toolUseId": "so_1", "name": "MyModel", "input": {"field1": "value1"}}}, + ], + } + # Register "MyModel" as a structured-output dynamic tool + event = _make_event(message, dynamic_tools={"MyModel": MagicMock()}) plugin.after_model_invocation(event) - assert message["content"] == [] + # toolUse input should be corrupted — now a truncated JSON string + tool_use_block = message["content"][0]["toolUse"] + corrupted_input = tool_use_block["input"] + assert isinstance(corrupted_input, str) + assert not corrupted_input.endswith("}") + + def test_plain_tooluse_not_corrupted_even_with_malformed_json(self): + """A plain mid-turn toolUse (not in dynamic_tools) is NOT corrupted by MalformedJson.""" + _set_chaos_case([MalformedJson()]) + plugin = ChaosPlugin() + message = { + "role": "assistant", + "content": [ + {"toolUse": {"toolUseId": "tu_1", "name": "search", "input": {"query": "test"}}}, + ], + } + original_content = copy.deepcopy(message["content"]) + # "search" is NOT in dynamic_tools — it's a regular tool + event = _make_event(message, dynamic_tools={}) + + plugin.after_model_invocation(event) + + # Content should be UNCHANGED — Guard 2 rejects (no structured-output tool found) + assert message["content"] == original_content + + def test_mixed_tooluse_only_structured_output_corrupted(self): + """In a message with both regular and structured-output toolUse, only SO is corrupted.""" + _set_chaos_case([MalformedJson()]) + plugin = ChaosPlugin() + message = { + "role": "assistant", + "content": [ + {"toolUse": {"toolUseId": "tu_1", "name": "search", "input": {"query": "test"}}}, + {"toolUse": {"toolUseId": "so_1", "name": "MyModel", "input": {"field1": "value1"}}}, + ], + } + # Only "MyModel" is structured-output + event = _make_event(message, dynamic_tools={"MyModel": MagicMock()}) + + plugin.after_model_invocation(event) + + # "search" toolUse should be UNCHANGED + search_block = message["content"][0]["toolUse"] + assert search_block["input"] == {"query": "test"} + # "MyModel" toolUse should be CORRUPTED + so_block = message["content"][1]["toolUse"] + assert isinstance(so_block["input"], str) + assert not so_block["input"].endswith("}") def teardown_method(self): _current_chaos_case.set(None) -class TestModelChaosHallucination: - """Hallucination effect corrupts the final assistant message text.""" +# --------------------------------------------------------------------------- +# 6.5: Post effects still work on text-only messages +# --------------------------------------------------------------------------- + + +class TestPostEffectsOnText: + """Post effects (Confabulation, MalformedJson-on-text, SuccessFraming) work.""" def test_confabulation_injects_template(self): + """Confabulation injects fabricated citations into text content.""" _set_chaos_case([Confabulation()]) plugin = ChaosPlugin() original_text = "The weather is sunny. It is warm outside. Birds are singing." @@ -134,146 +308,165 @@ def test_confabulation_injects_template(self): # Should contain original text fragments assert "sunny" in result_text or "warm" in result_text - def teardown_method(self): - _current_chaos_case.set(None) - - -class TestModelChaosRefusal: - """FULL_REFUSAL is a pre-hook effect — it uses before_model_invocation.""" - - def test_refusal_cancels_model_call(self): - _set_chaos_case([FullRefusal()]) + def test_malformed_json_on_text(self): + """MalformedJson truncates JSON-like text content.""" + _set_chaos_case([MalformedJson()]) plugin = ChaosPlugin() - event = MagicMock() - - plugin.before_model_invocation(event) + message = _final_assistant_message('{"key": "value", "nested": {"a": 1}}') + event = _make_event(message) - # event.cancel should be set to a refusal template string - assert event.cancel in FullRefusal._REFUSAL_TEMPLATES + plugin.after_model_invocation(event) - def teardown_method(self): - _current_chaos_case.set(None) + result_text = message["content"][0]["text"] + assert result_text != '{"key": "value", "nested": {"a": 1}}' + # Should be truncated (roughly half the original) + assert len(result_text) < len('{"key": "value", "nested": {"a": 1}}') + def test_success_framing_prepends_prefix(self): + """SuccessFraming prepends a confident prefix to text content.""" + _set_chaos_case([SuccessFraming()]) + plugin = ChaosPlugin() + message = _final_assistant_message("Here is the result.") + event = _make_event(message) -class TestModelChaosSuccessFraming: - """Success framing prepends a confident prefix after corruption.""" + plugin.after_model_invocation(event) - def test_success_framing_with_empty_response(self): - # SuccessFraming is a post-hook composable effect - _set_chaos_case([EmptyResponse(), SuccessFraming()]) + result_text = message["content"][0]["text"] + # Should start with one of the success prefixes + has_prefix = any(result_text.startswith(p) for p in SuccessFraming._SUCCESS_PREFIXES) + assert has_prefix + # Original text should still be present + assert "Here is the result." in result_text + + def test_confabulation_plus_success_framing(self): + """Confabulation + SuccessFraming compose: citation injected, then prefix prepended.""" + _set_chaos_case([Confabulation(), SuccessFraming()]) plugin = ChaosPlugin() - message = _final_assistant_message("Here is the code you requested...") + original_text = "The weather is sunny. It is warm outside. Birds are singing." + message = _final_assistant_message(original_text) event = _make_event(message) plugin.after_model_invocation(event) - # EmptyResponse clears content to [], then SuccessFraming prepends a - # prefix block (disguises the emptied response with confident framing) - assert len(message["content"]) == 1 result_text = message["content"][0]["text"] - assert result_text in SuccessFraming._SUCCESS_PREFIXES + # Should start with a success prefix (SuccessFraming applied last) + has_prefix = any(result_text.startswith(p) for p in SuccessFraming._SUCCESS_PREFIXES) + assert has_prefix def teardown_method(self): _current_chaos_case.set(None) # --------------------------------------------------------------------------- -# Guard tests +# 6.6: Mixed pre+post still produces one turn (pre wins) # --------------------------------------------------------------------------- -class TestModelChaosGuardToolUseMessage: - """Messages with toolUse blocks are NOT corrupted (guard skips them).""" +class TestMixedPrePostCase: + """Mixed pre+post effects: pre wins, post does NOT double-corrupt.""" - def test_empty_response_on_tooluse_message_not_corrupted(self): - """EMPTY_RESPONSE on a tool_use message passes through; toolUse intact.""" - _set_chaos_case([EmptyResponse()]) + def test_full_refusal_plus_malformed_json(self): + """FullRefusal (pre) + MalformedJson (post): pre cancels, post skipped.""" + _set_chaos_case([FullRefusal(), MalformedJson()]) plugin = ChaosPlugin() - message = _tooluse_assistant_message() - original_content = copy.deepcopy(message["content"]) - event = _make_event(message) - plugin.after_model_invocation(event) + # Pre-hook fires and cancels + pre_event = MagicMock() + plugin.before_model_invocation(pre_event) + cancel_text = pre_event.cancel + assert cancel_text in FullRefusal._REFUSAL_TEMPLATES - # Content should be UNCHANGED — guard skipped corruption - assert message["content"] == original_content - # toolUse blocks should be intact - tool_blocks = [b for b in message["content"] if "toolUse" in b] - assert len(tool_blocks) == 1 - assert tool_blocks[0]["toolUse"]["name"] == "search" + # SDK builds cancel message, MessageAddedEvent fires + cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} + post_event = _make_event(cancel_message) + plugin.after_model_invocation(post_event) - def test_full_refusal_does_not_affect_post_hook(self): - """FullRefusal is pre-hook only — after_model_invocation with toolUse still passes through.""" - _set_chaos_case([FullRefusal()]) + # Post effect (MalformedJson) should NOT have corrupted the content + assert cancel_message["content"] == [{"text": cancel_text}] + assert len(cancel_message["content"]) == 1 + + def test_empty_response_plus_success_framing(self): + """EmptyResponse (pre) + SuccessFraming (post): pre cancels, post skipped.""" + _set_chaos_case([EmptyResponse(), SuccessFraming()]) plugin = ChaosPlugin() - message = _tooluse_assistant_message() - original_content = copy.deepcopy(message["content"]) - event = _make_event(message) - plugin.after_model_invocation(event) + # Pre-hook fires + pre_event = MagicMock() + plugin.before_model_invocation(pre_event) + assert pre_event.cancel == " " - # FullRefusal is pre-hook, so post hook has no post effects to apply - assert message["content"] == original_content + # SDK builds cancel message with single space + cancel_message = {"role": "assistant", "content": [{"text": " "}]} + post_event = _make_event(cancel_message) + plugin.after_model_invocation(post_event) + + # SuccessFraming (post) should NOT have been applied + assert cancel_message["content"] == [{"text": " "}] def teardown_method(self): _current_chaos_case.set(None) -class TestModelChaosGuardStructuredOutputPath: - """structured_output_model path fires MessageAddedEvent with toolUse — guard skips it.""" +# --------------------------------------------------------------------------- +# 6.7: MalformedJson DOES reach structured-output toolUse (replaces old guard test) +# --------------------------------------------------------------------------- + - def test_structured_output_tool_message_not_corrupted(self): - """A message containing the structured output tool call is NOT corrupted.""" - _set_chaos_case([EmptyResponse()]) +class TestMalformedJsonReachesStructuredOutput: + """MalformedJson reaches structured-output toolUse (Guard 2 relaxed for it).""" + + def test_malformed_json_corrupts_structured_output_tooluse(self): + """MalformedJson DOES corrupt a structured-output toolUse block.""" + _set_chaos_case([MalformedJson()]) plugin = ChaosPlugin() - # Simulate the structured output tool invocation message message = { "role": "assistant", "content": [ { "toolUse": { "toolUseId": "so_1", - "name": "structured_output__MyModel", + "name": "MyModel", "input": {"field1": "value1"}, } }, ], } - original_content = copy.deepcopy(message["content"]) - event = _make_event(message) + event = _make_event(message, dynamic_tools={"MyModel": MagicMock()}) plugin.after_model_invocation(event) - # Guard should skip — toolUse block present - assert message["content"] == original_content - - def teardown_method(self): - _current_chaos_case.set(None) - + # The toolUse input should now be a corrupted string, not a dict + tool_use_block = message["content"][0]["toolUse"] + assert isinstance(tool_use_block["input"], str) + assert not tool_use_block["input"].endswith("}") -class TestModelChaosGuardFinalMessage: - """Final end_turn assistant message IS corrupted.""" - - def test_final_end_turn_message_corrupted(self): - """An end_turn message with text only (no toolUse) gets corrupted.""" - _set_chaos_case([EmptyResponse()]) + def test_other_post_effects_still_skip_tooluse(self): + """Confabulation on a toolUse message is skipped (Guard 2 only relaxed for MalformedJson).""" + _set_chaos_case([Confabulation()]) plugin = ChaosPlugin() - message = _final_assistant_message("Hello world") + message = _tooluse_assistant_message() + original_content = copy.deepcopy(message["content"]) event = _make_event(message) plugin.after_model_invocation(event) - assert message["content"] == [] + # Content should be UNCHANGED — guard skipped corruption + assert message["content"] == original_content def teardown_method(self): _current_chaos_case.set(None) -class TestModelChaosGuardRoleFiltering: +# --------------------------------------------------------------------------- +# Guard tests (role filtering, passthrough) +# --------------------------------------------------------------------------- + + +class TestGuardRoleFiltering: """User and tool result messages are NOT corrupted.""" def test_user_message_not_corrupted(self): - _set_chaos_case([EmptyResponse()]) + _set_chaos_case([Confabulation()]) plugin = ChaosPlugin() message = _user_message() original_content = copy.deepcopy(message["content"]) @@ -284,7 +477,7 @@ def test_user_message_not_corrupted(self): assert message["content"] == original_content def test_tool_result_message_not_corrupted(self): - _set_chaos_case([EmptyResponse()]) + _set_chaos_case([Confabulation()]) plugin = ChaosPlugin() message = _tool_result_message() original_content = copy.deepcopy(message["content"]) @@ -298,11 +491,10 @@ def teardown_method(self): _current_chaos_case.set(None) -class TestModelChaosPassthrough: +class TestPassthrough: """No corruption when no model_effects is set.""" def test_no_config_passes_through(self): - # No chaos case set — plugin should pass through _current_chaos_case.set(None) plugin = ChaosPlugin() message = _final_assistant_message("Hello world") @@ -313,66 +505,18 @@ def test_no_config_passes_through(self): assert message["content"] == original_content - -# --------------------------------------------------------------------------- -# Pre-hook integration test -# --------------------------------------------------------------------------- - - -class TestModelChaosPreHookIntegration: - """Integration test: FullRefusal pre-hook produces one assistant turn.""" - - def test_full_refusal_produces_single_turn(self): - """FullRefusal cancels model call, SDK builds cancel message, run ends.""" - _set_chaos_case([FullRefusal()]) - plugin = ChaosPlugin() - - # Step 1: before_model_invocation fires - pre_event = MagicMock() - plugin.before_model_invocation(pre_event) - cancel_text = pre_event.cancel - assert cancel_text in FullRefusal._REFUSAL_TEMPLATES - - # Step 2: SDK builds the cancel message and fires MessageAddedEvent - cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} - post_event = _make_event(cancel_message) - plugin.after_model_invocation(post_event) - - # Step 3: verify the cancel message is unchanged (not double-corrupted) - assert cancel_message["content"] == [{"text": cancel_text}] - assert len(cancel_message["content"]) == 1 - - def teardown_method(self): - _current_chaos_case.set(None) - - -# --------------------------------------------------------------------------- -# Mixed pre+post case test -# --------------------------------------------------------------------------- - - -class TestModelChaosMixedCase: - """Mixed pre+post effects: pre wins, post does NOT double-corrupt.""" - - def test_pre_plus_post_produces_single_uncorrupted_turn(self): - """ChaosCase with FullRefusal + EmptyResponse: pre cancels, post skipped.""" - _set_chaos_case([FullRefusal(), EmptyResponse()]) + def test_empty_effects_passes_through(self): + """ChaosCase with empty effects dict does not corrupt.""" + case = ChaosCase(name="baseline", input="test", effects={}) + _current_chaos_case.set(case) plugin = ChaosPlugin() + message = _final_assistant_message("Hello world") + original_content = copy.deepcopy(message["content"]) + event = _make_event(message) - # Pre-hook fires and cancels - pre_event = MagicMock() - plugin.before_model_invocation(pre_event) - cancel_text = pre_event.cancel - assert cancel_text in FullRefusal._REFUSAL_TEMPLATES - - # SDK builds cancel message, MessageAddedEvent fires - cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} - post_event = _make_event(cancel_message) - plugin.after_model_invocation(post_event) + plugin.after_model_invocation(event) - # Post effect (EmptyResponse) should NOT have emptied the content - assert cancel_message["content"] == [{"text": cancel_text}] - assert len(cancel_message["content"]) == 1 + assert message["content"] == original_content def teardown_method(self): _current_chaos_case.set(None) From dc98bbab7b14a2e635b5a9a07dc8fe9a47b00af6 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Wed, 12 Aug 2026 08:17:08 +0000 Subject: [PATCH 6/8] fix(chaos): enforce effect-family validation, reject unsupported model keys, semantic SO-tool detection --- pyproject.toml | 2 +- src/strands_evals/chaos/case.py | 22 +++- src/strands_evals/chaos/effects.py | 9 ++ src/strands_evals/chaos/plugin.py | 94 ++++++++------ tests/strands_evals/chaos/test_model_chaos.py | 122 ++++++++++++++++-- 5 files changed, 194 insertions(+), 55 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 80265ff0..5c3490b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ authors = [ dependencies = [ "pydantic>=2.4.0,<3.0.0", "rich>=14.0.0,<15.0.0", - "strands-agents>=1.42.0", + "strands-agents>=1.45.0", "strands-agents-tools>=0.1.0,<1.0.0", "typing-extensions>=4.13.2,<5.0.0", "opentelemetry-api>=1.20.0", diff --git a/src/strands_evals/chaos/case.py b/src/strands_evals/chaos/case.py index 185cf5a8..dd782a23 100644 --- a/src/strands_evals/chaos/case.py +++ b/src/strands_evals/chaos/case.py @@ -13,7 +13,7 @@ from ..case import Case from ..types.evaluation import InputT, OutputT -from .effects import ModelEffectUnion, ToolEffectUnion +from .effects import ModelEffect, ModelEffectUnion, ToolEffect, ToolEffectUnion # Type alias for the effects dict structure EffectsDict = dict[str, dict[str, list[ToolEffectUnion | ModelEffectUnion]]] @@ -94,12 +94,25 @@ def _validate_effects(self) -> "ChaosCase": f"Tool '{tool_name}' has {len(effects_list)} effects — only 1 is allowed per " f"ChaosCase. Use separate ChaosCase instances to test effects independently." ) + # Fix A: enforce effect-family membership + for effect in effects_list: + if not isinstance(effect, ToolEffect): + raise ValueError( + f"Effect {type(effect).__name__} in tool_effects['{tool_name}'] is not a {ToolEffect.__name__}" + ) # Validate model_effects: dict[str, list[ModelEffectUnion]] model_effects_map = self.effects.get("model_effects", {}) if model_effects_map: if not isinstance(model_effects_map, dict): raise ValueError("'model_effects' must be a dict keyed by model name (or '*' wildcard).") + # Fix B: reject non-"*" keys + for model_name in model_effects_map: + if model_name != "*": + raise ValueError( + f"model_effects key '{model_name}' is not supported; " + f"model targeting not yet implemented. Use '*' for all models." + ) for model_name, effects_list in model_effects_map.items(): # type: ignore[assignment] if not isinstance(model_name, str): raise ValueError(f"model_effects keys must be strings, got {type(model_name).__name__}.") @@ -108,6 +121,13 @@ def _validate_effects(self) -> "ChaosCase": f"model_effects['{model_name}'] must be a list of model effects, " f"got {type(effects_list).__name__}." ) + # Fix A: enforce effect-family membership + for effect in effects_list: + if not isinstance(effect, ModelEffect): + raise ValueError( + f"Effect {type(effect).__name__} in model_effects['{model_name}'] " + f"is not a {ModelEffect.__name__}" + ) return self diff --git a/src/strands_evals/chaos/effects.py b/src/strands_evals/chaos/effects.py index 421b3929..48390653 100644 --- a/src/strands_evals/chaos/effects.py +++ b/src/strands_evals/chaos/effects.py @@ -385,6 +385,15 @@ def _malform_blocks(blocks: list) -> list: result.append(block) return result + def malform_tool_use_block(self, block: dict) -> dict: + """Corrupt a single toolUse block's input JSON.""" + block = dict(block) + tool_use = dict(block["toolUse"]) + raw = json.dumps(tool_use.get("input", {})) + tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" + block["toolUse"] = tool_use + return block + # --------------------------------------------------------------------------- # b) EmptyResponse diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index ec8077cd..a904763e 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -166,21 +166,16 @@ def after_model_invocation(self, event: MessageAddedEvent) -> None: return message = event.message - - # Guard 1: only assistant messages - if message.get("role") != "assistant": + if not self._is_final_model_output(message): return content = message.get("content") - if content is None: - return - # If any pre effects exist, they already produced the turn — skip post + # Pre effects already produced the turn — skip post pre_effects = [e for e in chaos_case.model_effects if e.hook == "pre"] if pre_effects: return - # Filter to post effects only post_effects = [e for e in chaos_case.model_effects if e.hook == "post"] if not post_effects: return @@ -191,34 +186,52 @@ def after_model_invocation(self, event: MessageAddedEvent) -> None: if has_tool_use: if not any(isinstance(e, MalformedJson) for e in post_effects): return - # Only allow through if toolUse is a structured-output tool (dynamic_tools) - so_tool_names = set(event.agent.tool_registry.dynamic_tools.keys()) - tool_use_names = { - block["toolUse"]["name"] for block in content if isinstance(block, dict) and "toolUse" in block - } - if not tool_use_names & so_tool_names: - return # No structured-output toolUse — skip - - # Resolve structured-output tool names for _apply_to_model_blocks - so_tool_names_set: set[str] = set() - if isinstance(content, list) and any(isinstance(b, dict) and "toolUse" in b for b in content): - so_tool_names_set = set(event.agent.tool_registry.dynamic_tools.keys()) - - # Apply model post effects - corrupted = self._apply_to_model_blocks(post_effects, content, so_tool_names_set) + structured_output_tool_names = self._get_structured_output_tool_names(event.agent) + if not self._has_structured_output_tool_use(content, structured_output_tool_names): + return + else: + structured_output_tool_names = set() + else: + structured_output_tool_names = set() + + corrupted = self._apply_to_model_blocks(post_effects, content, structured_output_tool_names) message["content"] = corrupted effect_names = ", ".join(type(e).__name__ for e in post_effects) - logger.info( - "effects=<%s> | applied model output chaos to assistant message", - effect_names, + logger.info("effects=<%s> | applied model output chaos", effect_names) + + # ----------------------------------------------------------------------- + # Model output helper methods + # ----------------------------------------------------------------------- + + def _is_final_model_output(self, message) -> bool: + """Check if message is a final assistant model output (role=assistant, has content).""" + return message.get("role") == "assistant" and message.get("content") is not None + + def _get_structured_output_tool_names(self, agent) -> set[str]: # type: ignore[type-arg] + """Identify structured-output tools via isinstance(tool, StructuredOutputTool).""" + from strands.tools.structured_output.structured_output_tool import StructuredOutputTool + + return { + name for name, tool in agent.tool_registry.dynamic_tools.items() if isinstance(tool, StructuredOutputTool) + } + + def _has_structured_output_tool_use(self, content: list, structured_output_tool_names: set[str]) -> bool: + """Check if content has any toolUse block matching a structured-output tool.""" + return any( + isinstance(block, dict) + and "toolUse" in block + and block["toolUse"].get("name", "") in structured_output_tool_names + for block in content ) # ----------------------------------------------------------------------- # Model corruption helpers # ----------------------------------------------------------------------- - def _apply_to_model_blocks(self, post_effects: list, content: list, so_tool_names: set[str] | None = None) -> list: + def _apply_to_model_blocks( + self, post_effects: list, content: list, structured_output_tool_names: set[str] | None = None + ) -> list: """Apply model post effects to content blocks sequentially. Handles text blocks (Confabulation, SuccessFraming, MalformedJson on text) @@ -230,32 +243,27 @@ def _apply_to_model_blocks(self, post_effects: list, content: list, so_tool_name corrupted = content for effect in primary: - if isinstance(effect, MalformedJson) and so_tool_names: - corrupted = self._apply_malformed_json_selective(effect, corrupted, so_tool_names) + if isinstance(effect, MalformedJson) and structured_output_tool_names: + corrupted = self._apply_malformed_json_selective(effect, corrupted, structured_output_tool_names) else: corrupted = effect.apply(corrupted) for effect in framing: corrupted = effect.apply(corrupted) return corrupted - def _apply_malformed_json_selective(self, effect: MalformedJson, blocks: list, so_tool_names: set[str]) -> list: - """Apply MalformedJson only to structured-output toolUse blocks and text blocks. - - Ordinary mid-turn toolUse blocks are left untouched. - """ + def _apply_malformed_json_selective( + self, effect: MalformedJson, blocks: list, structured_output_tool_names: set[str] + ) -> list: + """Apply MalformedJson: text blocks get malformed; only SO toolUse blocks get corrupted.""" result = [] for block in blocks: - block = dict(block) - if "toolUse" in block: + if isinstance(block, dict) and "toolUse" in block: tool_name = block["toolUse"].get("name", "") - if tool_name in so_tool_names: - # Structured-output toolUse — corrupt the input JSON - tool_use = dict(block["toolUse"]) - raw = json.dumps(tool_use.get("input", {})) - tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" - block["toolUse"] = tool_use - # else: ordinary mid-turn toolUse — leave untouched - elif "text" in block and isinstance(block["text"], str): + if tool_name in structured_output_tool_names: + block = effect.malform_tool_use_block(block) + # else: ordinary toolUse — leave untouched + elif isinstance(block, dict) and "text" in block and isinstance(block["text"], str): + block = dict(block) block["text"] = MalformedJson._malform_text(block["text"]) result.append(block) return result diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py index 4adc1609..c866ae18 100644 --- a/tests/strands_evals/chaos/test_model_chaos.py +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -8,11 +8,19 @@ - 6.5: Post effects (Confabulation, MalformedJson-on-text, SuccessFraming) work - 6.6: Mixed pre+post still produces one turn (pre wins) - 6.7: MalformedJson DOES reach/corrupt structured-output toolUse +- Effect family validation (Fix A): wrong-category effects rejected +- Wildcard rejection (Fix B): non-'*' model_effects keys rejected +- Ordinary dynamic tool not corrupted (Fix C): isinstance-based detection """ import copy from unittest.mock import MagicMock +import pytest +from pydantic import ValidationError as PydanticValidationError +from strands.hooks import BeforeModelCallEvent +from strands.tools.structured_output.structured_output_tool import StructuredOutputTool + from strands_evals.chaos._context import _current_chaos_case from strands_evals.chaos.case import ChaosCase from strands_evals.chaos.effects import ( @@ -21,6 +29,7 @@ FullRefusal, MalformedJson, SuccessFraming, + Timeout, ) from strands_evals.chaos.plugin import ChaosPlugin @@ -138,7 +147,7 @@ def test_empty_response_cancels_with_single_space(self): """before_model_invocation sets event.cancel to ' ' (single space).""" _set_chaos_case([EmptyResponse()]) plugin = ChaosPlugin() - event = MagicMock() + event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(event) @@ -151,7 +160,7 @@ def test_empty_response_model_not_called(self): plugin = ChaosPlugin() # Pre-hook fires - pre_event = MagicMock() + pre_event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(pre_event) assert pre_event.cancel == " " @@ -179,7 +188,7 @@ def test_full_refusal_cancels_model_call(self): """before_model_invocation sets event.cancel to a refusal template.""" _set_chaos_case([FullRefusal()]) plugin = ChaosPlugin() - event = MagicMock() + event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(event) @@ -191,7 +200,7 @@ def test_full_refusal_produces_single_turn(self): plugin = ChaosPlugin() # Step 1: before_model_invocation fires - pre_event = MagicMock() + pre_event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(pre_event) cancel_text = pre_event.cancel assert cancel_text in FullRefusal._REFUSAL_TEMPLATES @@ -218,7 +227,7 @@ class TestMalformedJsonStructuredOutput: """MalformedJson DOES reach and corrupt structured-output toolUse blocks only.""" def test_malformed_json_corrupts_structured_output_tooluse(self): - """MalformedJson corrupts toolUse input when tool is in dynamic_tools (structured-output).""" + """MalformedJson corrupts toolUse input when tool is a StructuredOutputTool.""" _set_chaos_case([MalformedJson()]) plugin = ChaosPlugin() message = { @@ -228,7 +237,8 @@ def test_malformed_json_corrupts_structured_output_tooluse(self): ], } # Register "MyModel" as a structured-output dynamic tool - event = _make_event(message, dynamic_tools={"MyModel": MagicMock()}) + mock_so_tool = MagicMock(spec=StructuredOutputTool) + event = _make_event(message, dynamic_tools={"MyModel": mock_so_tool}) plugin.after_model_invocation(event) @@ -269,7 +279,8 @@ def test_mixed_tooluse_only_structured_output_corrupted(self): ], } # Only "MyModel" is structured-output - event = _make_event(message, dynamic_tools={"MyModel": MagicMock()}) + mock_so_tool = MagicMock(spec=StructuredOutputTool) + event = _make_event(message, dynamic_tools={"MyModel": mock_so_tool}) plugin.after_model_invocation(event) @@ -371,7 +382,7 @@ def test_full_refusal_plus_malformed_json(self): plugin = ChaosPlugin() # Pre-hook fires and cancels - pre_event = MagicMock() + pre_event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(pre_event) cancel_text = pre_event.cancel assert cancel_text in FullRefusal._REFUSAL_TEMPLATES @@ -391,7 +402,7 @@ def test_empty_response_plus_success_framing(self): plugin = ChaosPlugin() # Pre-hook fires - pre_event = MagicMock() + pre_event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(pre_event) assert pre_event.cancel == " " @@ -431,7 +442,8 @@ def test_malformed_json_corrupts_structured_output_tooluse(self): }, ], } - event = _make_event(message, dynamic_tools={"MyModel": MagicMock()}) + mock_so_tool = MagicMock(spec=StructuredOutputTool) + event = _make_event(message, dynamic_tools={"MyModel": mock_so_tool}) plugin.after_model_invocation(event) @@ -457,6 +469,96 @@ def teardown_method(self): _current_chaos_case.set(None) +# --------------------------------------------------------------------------- +# Effect family validation (Fix A) +# --------------------------------------------------------------------------- + + +class TestEffectFamilyValidation: + """Effects placed in the wrong category are rejected.""" + + def test_tool_effect_in_model_effects_rejected(self): + """A ToolEffect under model_effects raises ValueError.""" + with pytest.raises(ValueError, match="is not a ModelEffect"): + ChaosCase( + name="bad", + input="test", + effects={"model_effects": {"*": [Timeout()]}}, + ) + + def test_model_effect_in_tool_effects_rejected(self): + """A ModelEffect under tool_effects raises ValueError.""" + with pytest.raises(ValueError, match="is not a ToolEffect"): + ChaosCase( + name="bad", + input="test", + effects={"tool_effects": {"search": [FullRefusal()]}}, + ) + + def test_model_effect_in_tool_effects_rejected_via_model_validate(self): + """A ModelEffect under tool_effects is rejected on the model_validate (dict) path.""" + with pytest.raises(PydanticValidationError, match="is not a ToolEffect"): + ChaosCase.model_validate( + { + "name": "bad_tool", + "input": "test", + "effects": {"tool_effects": {"search": [{"effect_type": "full_refusal"}]}}, + } + ) + + def test_tool_effect_in_model_effects_rejected_via_model_validate(self): + """A ToolEffect under model_effects is rejected on the model_validate (dict) path.""" + with pytest.raises(PydanticValidationError, match="is not a ModelEffect"): + ChaosCase.model_validate( + { + "name": "bad_model", + "input": "test", + "effects": {"model_effects": {"*": [{"effect_type": "timeout"}]}}, + } + ) + + def test_named_model_key_rejected(self): + """A non-'*' key in model_effects is rejected.""" + with pytest.raises(ValueError, match="model targeting not yet"): + ChaosCase( + name="bad", + input="test", + effects={"model_effects": {"claude-sonnet": [MalformedJson()]}}, + ) + + +# --------------------------------------------------------------------------- +# Ordinary dynamic tool not corrupted (Fix C) +# --------------------------------------------------------------------------- + + +class TestOrdinaryDynamicToolNotCorrupted: + """An ordinary dynamic tool (not StructuredOutputTool) is NOT corrupted.""" + + def test_ordinary_dynamic_tool_unchanged(self): + """MalformedJson does NOT corrupt a regular dynamic tool's toolUse.""" + _set_chaos_case([MalformedJson()]) + plugin = ChaosPlugin() + message = { + "role": "assistant", + "content": [ + {"toolUse": {"toolUseId": "dt_1", "name": "my_dynamic_tool", "input": {"key": "val"}}}, + ], + } + original_content = copy.deepcopy(message["content"]) + # Register as a plain MagicMock (NOT spec'd to StructuredOutputTool) + mock_tool = MagicMock() + event = _make_event(message, dynamic_tools={"my_dynamic_tool": mock_tool}) + + plugin.after_model_invocation(event) + + # Should be UNCHANGED — ordinary dynamic tool, not structured-output + assert message["content"] == original_content + + def teardown_method(self): + _current_chaos_case.set(None) + + # --------------------------------------------------------------------------- # Guard tests (role filtering, passthrough) # --------------------------------------------------------------------------- From 7eb597b3399a0ed644153c855263742a9990e733 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Thu, 13 Aug 2026 06:22:49 +0000 Subject: [PATCH 7/8] refactor(chaos): TypedDict effects schema, message classification, single malform impl --- src/strands_evals/chaos/case.py | 84 ++++-------- src/strands_evals/chaos/effects.py | 81 +++--------- src/strands_evals/chaos/plugin.py | 109 ++++++++-------- tests/strands_evals/chaos/test_model_chaos.py | 121 +++++------------- 4 files changed, 129 insertions(+), 266 deletions(-) diff --git a/src/strands_evals/chaos/case.py b/src/strands_evals/chaos/case.py index dd782a23..2d237d5d 100644 --- a/src/strands_evals/chaos/case.py +++ b/src/strands_evals/chaos/case.py @@ -6,17 +6,23 @@ """ import uuid -from typing import cast +from typing import Literal, TypedDict -from pydantic import Field, model_validator +from pydantic import ConfigDict, Field, model_validator from typing_extensions import Generic from ..case import Case from ..types.evaluation import InputT, OutputT -from .effects import ModelEffect, ModelEffectUnion, ToolEffect, ToolEffectUnion +from .effects import ModelEffectUnion, ToolEffectUnion -# Type alias for the effects dict structure -EffectsDict = dict[str, dict[str, list[ToolEffectUnion | ModelEffectUnion]]] + +class ChaosEffects(TypedDict, total=False): + """Typed schema for chaos effects configuration.""" + + __pydantic_config__ = ConfigDict(extra="forbid") # type: ignore[misc] + + tool_effects: dict[str, list[ToolEffectUnion]] + model_effects: dict[Literal["*"], list[ModelEffectUnion]] class ChaosCase(Case, Generic[InputT, OutputT]): @@ -32,7 +38,7 @@ class ChaosCase(Case, Generic[InputT, OutputT]): Attributes: effects: A dict keyed by effect category. Supports ``"tool_effects"`` mapping tool_name -> list of effects, and ``"model_effects"`` - mapping model_name (or ``"*"`` wildcard) -> list of effects. + mapping ``"*"`` wildcard -> list of effects. Example:: @@ -69,73 +75,34 @@ class ChaosCase(Case, Generic[InputT, OutputT]): # Produces 6 ChaosCase objects: 2 cases × (2 effect maps + 1 baseline) """ - effects: EffectsDict = Field( - default_factory=dict, + effects: ChaosEffects = Field( + default_factory=ChaosEffects, description="Effect categories. Supports 'tool_effects' mapping " "tool_name -> list of effects, and 'model_effects' mapping " - "model_name (or '*' wildcard) -> list of effects. " + "'*' wildcard -> list of effects. " "Empty dict means baseline (no chaos).", ) @model_validator(mode="after") def _validate_effects(self) -> "ChaosCase": - """Validate effects configuration structure.""" - allowed_categories = {"tool_effects", "model_effects"} - unknown = set(self.effects.keys()) - allowed_categories - if unknown: - raise ValueError( - f"Unknown effect categories: {sorted(unknown)}. Allowed categories: {sorted(allowed_categories)}." - ) - - # Validate tool_effects: dict[str, list[ToolEffectUnion]] + """Validate behavioral constraints the type system cannot express.""" + self._validate_tool_effects() + return self + + def _validate_tool_effects(self) -> None: + """At most one effect per tool.""" for tool_name, effects_list in self.tool_effects.items(): if len(effects_list) > 1: raise ValueError( f"Tool '{tool_name}' has {len(effects_list)} effects — only 1 is allowed per " f"ChaosCase. Use separate ChaosCase instances to test effects independently." ) - # Fix A: enforce effect-family membership - for effect in effects_list: - if not isinstance(effect, ToolEffect): - raise ValueError( - f"Effect {type(effect).__name__} in tool_effects['{tool_name}'] is not a {ToolEffect.__name__}" - ) - - # Validate model_effects: dict[str, list[ModelEffectUnion]] - model_effects_map = self.effects.get("model_effects", {}) - if model_effects_map: - if not isinstance(model_effects_map, dict): - raise ValueError("'model_effects' must be a dict keyed by model name (or '*' wildcard).") - # Fix B: reject non-"*" keys - for model_name in model_effects_map: - if model_name != "*": - raise ValueError( - f"model_effects key '{model_name}' is not supported; " - f"model targeting not yet implemented. Use '*' for all models." - ) - for model_name, effects_list in model_effects_map.items(): # type: ignore[assignment] - if not isinstance(model_name, str): - raise ValueError(f"model_effects keys must be strings, got {type(model_name).__name__}.") - if not isinstance(effects_list, list): - raise ValueError( - f"model_effects['{model_name}'] must be a list of model effects, " - f"got {type(effects_list).__name__}." - ) - # Fix A: enforce effect-family membership - for effect in effects_list: - if not isinstance(effect, ModelEffect): - raise ValueError( - f"Effect {type(effect).__name__} in model_effects['{model_name}'] " - f"is not a {ModelEffect.__name__}" - ) - - return self @classmethod def expand( cls, cases: list[Case], - effect_maps: dict[str, EffectsDict], + effect_maps: dict[str, ChaosEffects], include_no_effect_baseline: bool = False, ) -> list["ChaosCase"]: """Generate the Cartesian product of cases × named effect maps. @@ -167,7 +134,7 @@ def expand( Flat list of ChaosCase objects with composite names like "flight_search|baseline" or "flight_search|search_timeout". """ - all_entries: list[tuple[str, EffectsDict]] = [] + all_entries: list[tuple[str, ChaosEffects]] = [] if include_no_effect_baseline: all_entries.append(("baseline", {})) @@ -201,7 +168,7 @@ def expand( @property def tool_effects(self) -> dict[str, list[ToolEffectUnion]]: """Convenience accessor for effects['tool_effects'].""" - return cast(dict[str, list[ToolEffectUnion]], self.effects.get("tool_effects", {})) + return self.effects.get("tool_effects", {}) @property def model_effects(self) -> list[ModelEffectUnion]: @@ -209,8 +176,7 @@ def model_effects(self) -> list[ModelEffectUnion]: model_effects_map = self.effects.get("model_effects", {}) if not model_effects_map: return [] - # For now, resolve "*" (wildcard = applies to all models) - return cast(list[ModelEffectUnion], model_effects_map.get("*", [])) + return model_effects_map.get("*", []) def __repr__(self) -> str: effects_str = ", ".join( diff --git a/src/strands_evals/chaos/effects.py b/src/strands_evals/chaos/effects.py index 48390653..c3cb547d 100644 --- a/src/strands_evals/chaos/effects.py +++ b/src/strands_evals/chaos/effects.py @@ -45,11 +45,6 @@ class ToolEffect(ChaosEffect): """ -# --------------------------------------------------------------------------- -# Pre-hook effects: cancel the tool call before execution -# --------------------------------------------------------------------------- - - class Timeout(ToolEffect): """Simulates a tool call timeout. @@ -154,11 +149,6 @@ def apply(self, context: Any = None) -> str: return self.error_message -# --------------------------------------------------------------------------- -# Post-hook effects: corrupt the tool response after execution -# --------------------------------------------------------------------------- - - class TruncateFields(ToolEffect): """Truncates string values in the tool response. @@ -310,10 +300,6 @@ def apply(self, response: Any = None) -> Any: return result -# --------------------------------------------------------------------------- -# Discriminated union type for Pydantic serialization -# --------------------------------------------------------------------------- - ToolEffectUnion = Annotated[ Union[ Annotated[Timeout, Tag("timeout")], @@ -343,11 +329,6 @@ class ModelEffect(ChaosEffect): hook: ClassVar[Literal["pre", "post"]] = "post" -# --------------------------------------------------------------------------- -# a) MalformedJson -# --------------------------------------------------------------------------- - - class MalformedJson(ModelEffect): """Corrupts JSON structures in model output.""" @@ -358,35 +339,22 @@ def apply(self, content: Any = None) -> Any: if content is None: raise ValueError("MalformedJson.apply() requires content") if isinstance(content, str): - return self._malform_text(content) + return self.malform_text(content) elif isinstance(content, list): return self._malform_blocks(content) raise ValueError(f"MalformedJson.apply() received unsupported type {type(content).__name__}") @staticmethod - def _malform_text(text: str) -> str: + def malform_text(text: str) -> str: + """Corrupt JSON-like text — the ONE place text malformation lives.""" stripped = text.strip() if stripped.startswith("{") or stripped.startswith("["): return stripped[: len(stripped) // 2] return text @staticmethod - def _malform_blocks(blocks: list) -> list: - result = [] - for block in blocks: - block = dict(block) - if "toolUse" in block: - tool_use = dict(block["toolUse"]) - raw = json.dumps(tool_use.get("input", {})) - tool_use["input"] = raw[:-1] if raw.endswith("}") else raw + "{{{" - block["toolUse"] = tool_use - elif "text" in block and isinstance(block["text"], str): - block["text"] = MalformedJson._malform_text(block["text"]) - result.append(block) - return result - - def malform_tool_use_block(self, block: dict) -> dict: - """Corrupt a single toolUse block's input JSON.""" + def malform_tool_use_block(block: dict) -> dict: + """Corrupt a single toolUse block's input JSON — the ONE place this logic lives.""" block = dict(block) tool_use = dict(block["toolUse"]) raw = json.dumps(tool_use.get("input", {})) @@ -394,10 +362,18 @@ def malform_tool_use_block(self, block: dict) -> dict: block["toolUse"] = tool_use return block - -# --------------------------------------------------------------------------- -# b) EmptyResponse -# --------------------------------------------------------------------------- + @staticmethod + def _malform_blocks(blocks: list) -> list: + """Apply malformation to all blocks — delegates toolUse corruption to malform_tool_use_block.""" + result = [] + for block in blocks: + if isinstance(block, dict) and "toolUse" in block: + block = MalformedJson.malform_tool_use_block(block) + elif isinstance(block, dict) and "text" in block and isinstance(block["text"], str): + block = dict(block) + block["text"] = MalformedJson.malform_text(block["text"]) + result.append(block) + return result class EmptyResponse(ModelEffect): @@ -407,7 +383,7 @@ class EmptyResponse(ModelEffect): effect_type: Literal["empty_response"] = "empty_response" def cancel_message(self) -> str: - # Pre-cancel with single space (truthy) skips real model call — "model returned nothing". + """Pre-cancel with single space (truthy) skips real model call.""" return " " def apply(self, content: Any = None) -> Any: @@ -420,11 +396,6 @@ def apply(self, content: Any = None) -> Any: raise ValueError(f"EmptyResponse.apply() received unsupported type {type(content).__name__}") -# --------------------------------------------------------------------------- -# c) Confabulation -# --------------------------------------------------------------------------- - - class Confabulation(ModelEffect): """Injects fabricated citations into model output text.""" @@ -469,11 +440,6 @@ def _confabulate(self, text: str) -> str: return " ".join(sentences) -# --------------------------------------------------------------------------- -# d) FullRefusal -# --------------------------------------------------------------------------- - - class FullRefusal(ModelEffect): """Replaces model output with a refusal message.""" @@ -504,11 +470,6 @@ def apply(self, content: Any = None) -> Any: raise ValueError(f"FullRefusal.apply() received unsupported type {type(content).__name__}") -# --------------------------------------------------------------------------- -# e) SuccessFraming (composable post-step) -# --------------------------------------------------------------------------- - - class SuccessFraming(ModelEffect): """Prepends a confident success prefix to content. @@ -521,7 +482,7 @@ class SuccessFraming(ModelEffect): _SUCCESS_PREFIXES: ClassVar[list[str]] = [ "Successfully completed the requested operation.", "Done! Here are the results you asked for.", - "Great news \u2014 everything worked as expected.", + "Great news — everything worked as expected.", "Operation finished successfully. Here's what I found:", "All done! The task has been completed without issues.", "I've successfully processed your request. Here's the output:", @@ -545,10 +506,6 @@ def apply(self, content: Any = None) -> Any: raise ValueError(f"SuccessFraming.apply() received unsupported type {type(content).__name__}") -# --------------------------------------------------------------------------- -# ModelEffectUnion — discriminated union for Pydantic deserialization -# --------------------------------------------------------------------------- - ModelEffectUnion = Annotated[ Union[ Annotated[MalformedJson, Tag("malformed_json")], diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index a904763e..7a7d275c 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -11,6 +11,7 @@ import json import logging +from enum import Enum, auto from strands.hooks import ( AfterToolCallEvent, @@ -31,6 +32,15 @@ logger = logging.getLogger(__name__) +class MessageKind(Enum): + """Classification of messages for model output chaos routing.""" + + IRRELEVANT = auto() + ORDINARY_TOOL_USE = auto() + STRUCTURED_OUTPUT = auto() + FINAL_TEXT = auto() + + class ChaosPlugin(Plugin): """Strands Plugin that injects deterministic chaos based on configuration. @@ -72,9 +82,7 @@ class ChaosPlugin(Plugin): name = "chaos-testing" - # ----------------------------------------------------------------------- # Tool chaos hooks - # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] def before_tool_call(self, event: BeforeToolCallEvent) -> None: @@ -129,9 +137,7 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: logger.info("effect=<%s>, tool=<%s> | applied chaos post-hook", type(effect).__name__, tool_name) - # ----------------------------------------------------------------------- # Model output chaos hooks - # ----------------------------------------------------------------------- @hook # type: ignore[call-overload] def before_model_invocation(self, event: BeforeModelCallEvent) -> None: @@ -156,57 +162,67 @@ def before_model_invocation(self, event: BeforeModelCallEvent) -> None: @hook # type: ignore[call-overload] def after_model_invocation(self, event: MessageAddedEvent) -> None: - """Intercept messages to corrupt the final assistant response. - - Guards ensure corruption is only applied to appropriate messages. - Only post-hook effects are applied here. - """ + """Apply post-hook model effects based on message classification.""" chaos_case = _current_chaos_case.get() if chaos_case is None or not chaos_case.model_effects: return - message = event.message - if not self._is_final_model_output(message): - return - - content = message.get("content") - # Pre effects already produced the turn — skip post - pre_effects = [e for e in chaos_case.model_effects if e.hook == "pre"] - if pre_effects: + if any(e.hook == "pre" for e in chaos_case.model_effects): return post_effects = [e for e in chaos_case.model_effects if e.hook == "post"] if not post_effects: return - # Guard 2: skip toolUse messages (mid-turn dispatch); exception: MalformedJson on structured-output toolUse. - if isinstance(content, list): - has_tool_use = any(isinstance(block, dict) and "toolUse" in block for block in content) - if has_tool_use: - if not any(isinstance(e, MalformedJson) for e in post_effects): - return - structured_output_tool_names = self._get_structured_output_tool_names(event.agent) - if not self._has_structured_output_tool_use(content, structured_output_tool_names): - return - else: - structured_output_tool_names = set() - else: - structured_output_tool_names = set() + kind, content, so_tool_names = self._classify_model_message(event) - corrupted = self._apply_to_model_blocks(post_effects, content, structured_output_tool_names) - message["content"] = corrupted + if kind == MessageKind.IRRELEVANT: + return + elif kind == MessageKind.ORDINARY_TOOL_USE: + return + elif kind == MessageKind.STRUCTURED_OUTPUT: + # Only MalformedJson reaches structured-output toolUse + if not any(isinstance(e, MalformedJson) for e in post_effects): + return + assert content is not None # guaranteed by classifier for STRUCTURED_OUTPUT + corrupted = self._apply_to_model_blocks(post_effects, content, so_tool_names) + else: # FINAL_TEXT + assert content is not None # guaranteed by classifier for FINAL_TEXT + corrupted = self._apply_to_model_blocks(post_effects, content, set()) + event.message["content"] = corrupted effect_names = ", ".join(type(e).__name__ for e in post_effects) logger.info("effects=<%s> | applied model output chaos", effect_names) - # ----------------------------------------------------------------------- - # Model output helper methods - # ----------------------------------------------------------------------- + # Message classification - def _is_final_model_output(self, message) -> bool: - """Check if message is a final assistant model output (role=assistant, has content).""" - return message.get("role") == "assistant" and message.get("content") is not None + def _classify_model_message(self, event: MessageAddedEvent) -> tuple[MessageKind, list | None, set[str]]: + """Classify a message for chaos routing. Computed once per hook invocation.""" + message = event.message + if message.get("role") != "assistant": + return MessageKind.IRRELEVANT, None, set() + content = message.get("content") + if content is None: + return MessageKind.IRRELEVANT, None, set() + if not isinstance(content, list): + return MessageKind.FINAL_TEXT, content, set() + + has_tool_use = any(isinstance(block, dict) and "toolUse" in block for block in content) + if not has_tool_use: + return MessageKind.FINAL_TEXT, content, set() + + # Has toolUse — determine if it's structured-output + structured_output_tool_names = self._get_structured_output_tool_names(event.agent) + has_so = any( + isinstance(block, dict) + and "toolUse" in block + and block["toolUse"].get("name", "") in structured_output_tool_names + for block in content + ) + if has_so: + return MessageKind.STRUCTURED_OUTPUT, content, structured_output_tool_names + return MessageKind.ORDINARY_TOOL_USE, content, set() def _get_structured_output_tool_names(self, agent) -> set[str]: # type: ignore[type-arg] """Identify structured-output tools via isinstance(tool, StructuredOutputTool).""" @@ -216,18 +232,7 @@ def _get_structured_output_tool_names(self, agent) -> set[str]: # type: ignore[ name for name, tool in agent.tool_registry.dynamic_tools.items() if isinstance(tool, StructuredOutputTool) } - def _has_structured_output_tool_use(self, content: list, structured_output_tool_names: set[str]) -> bool: - """Check if content has any toolUse block matching a structured-output tool.""" - return any( - isinstance(block, dict) - and "toolUse" in block - and block["toolUse"].get("name", "") in structured_output_tool_names - for block in content - ) - - # ----------------------------------------------------------------------- # Model corruption helpers - # ----------------------------------------------------------------------- def _apply_to_model_blocks( self, post_effects: list, content: list, structured_output_tool_names: set[str] | None = None @@ -260,17 +265,15 @@ def _apply_malformed_json_selective( if isinstance(block, dict) and "toolUse" in block: tool_name = block["toolUse"].get("name", "") if tool_name in structured_output_tool_names: - block = effect.malform_tool_use_block(block) + block = MalformedJson.malform_tool_use_block(block) # else: ordinary toolUse — leave untouched elif isinstance(block, dict) and "text" in block and isinstance(block["text"], str): block = dict(block) - block["text"] = MalformedJson._malform_text(block["text"]) + block["text"] = MalformedJson.malform_text(block["text"]) result.append(block) return result - # ----------------------------------------------------------------------- # Tool corruption helpers - # ----------------------------------------------------------------------- def _apply_to_tool_blocks(self, effect: ChaosEffect, blocks: list) -> list: """Apply effect to text blocks in a tool content list.""" diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py index c866ae18..174ead02 100644 --- a/tests/strands_evals/chaos/test_model_chaos.py +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -1,16 +1,16 @@ """Unit tests for model output chaos via ChaosPlugin two-hook architecture. Tests cover: -- 6.1: Effects constructed via keyed dict {"model_effects": {"*": [...]}} -- 6.2: EmptyResponse as pre-hook: model not called, turn is single space -- 6.3: FullRefusal as pre-hook: model not called, turn is refusal text -- 6.4: MalformedJson on structured-output toolUse: toolUse input corrupted -- 6.5: Post effects (Confabulation, MalformedJson-on-text, SuccessFraming) work -- 6.6: Mixed pre+post still produces one turn (pre wins) -- 6.7: MalformedJson DOES reach/corrupt structured-output toolUse -- Effect family validation (Fix A): wrong-category effects rejected -- Wildcard rejection (Fix B): non-'*' model_effects keys rejected -- Ordinary dynamic tool not corrupted (Fix C): isinstance-based detection +- Effects constructed via keyed dict {"model_effects": {"*": [...]}} +- EmptyResponse as pre-hook: model not called, turn is single space +- FullRefusal as pre-hook: model not called, turn is refusal text +- MalformedJson on structured-output toolUse: toolUse input corrupted +- Post effects (Confabulation, MalformedJson-on-text, SuccessFraming) work +- Mixed pre+post still produces one turn (pre wins) +- MalformedJson DOES reach/corrupt structured-output toolUse +- Effect family validation: wrong-category effects rejected +- Wildcard rejection: non-'*' model_effects keys rejected +- Ordinary dynamic tool not corrupted: isinstance-based detection """ import copy @@ -33,10 +33,6 @@ ) from strands_evals.chaos.plugin import ChaosPlugin -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - def _make_event(message: dict, dynamic_tools: dict | None = None) -> MagicMock: """Create a mock MessageAddedEvent with the given message. @@ -101,11 +97,6 @@ def _set_chaos_case(model_effects): return case -# --------------------------------------------------------------------------- -# 6.1: Effects constructed via keyed dict -# --------------------------------------------------------------------------- - - class TestKeyedDictConstruction: """Effects are constructed via keyed dict form.""" @@ -135,11 +126,6 @@ def test_empty_effects_baseline(self): assert case.model_effects == [] -# --------------------------------------------------------------------------- -# 6.2: EmptyResponse as pre-hook -# --------------------------------------------------------------------------- - - class TestEmptyResponsePreHook: """EmptyResponse is a pre-hook effect — cancels model call with single space.""" @@ -151,7 +137,6 @@ def test_empty_response_cancels_with_single_space(self): plugin.before_model_invocation(event) - # event.cancel should be set to single space assert event.cancel == " " def test_empty_response_model_not_called(self): @@ -176,11 +161,6 @@ def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# 6.3: FullRefusal as pre-hook (unchanged behavior) -# --------------------------------------------------------------------------- - - class TestFullRefusalPreHook: """FullRefusal is a pre-hook effect — cancels model call with refusal text.""" @@ -218,11 +198,6 @@ def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# 6.4: MalformedJson on structured-output toolUse -# --------------------------------------------------------------------------- - - class TestMalformedJsonStructuredOutput: """MalformedJson DOES reach and corrupt structured-output toolUse blocks only.""" @@ -236,13 +211,11 @@ def test_malformed_json_corrupts_structured_output_tooluse(self): {"toolUse": {"toolUseId": "so_1", "name": "MyModel", "input": {"field1": "value1"}}}, ], } - # Register "MyModel" as a structured-output dynamic tool mock_so_tool = MagicMock(spec=StructuredOutputTool) event = _make_event(message, dynamic_tools={"MyModel": mock_so_tool}) plugin.after_model_invocation(event) - # toolUse input should be corrupted — now a truncated JSON string tool_use_block = message["content"][0]["toolUse"] corrupted_input = tool_use_block["input"] assert isinstance(corrupted_input, str) @@ -259,12 +232,10 @@ def test_plain_tooluse_not_corrupted_even_with_malformed_json(self): ], } original_content = copy.deepcopy(message["content"]) - # "search" is NOT in dynamic_tools — it's a regular tool event = _make_event(message, dynamic_tools={}) plugin.after_model_invocation(event) - # Content should be UNCHANGED — Guard 2 rejects (no structured-output tool found) assert message["content"] == original_content def test_mixed_tooluse_only_structured_output_corrupted(self): @@ -278,7 +249,6 @@ def test_mixed_tooluse_only_structured_output_corrupted(self): {"toolUse": {"toolUseId": "so_1", "name": "MyModel", "input": {"field1": "value1"}}}, ], } - # Only "MyModel" is structured-output mock_so_tool = MagicMock(spec=StructuredOutputTool) event = _make_event(message, dynamic_tools={"MyModel": mock_so_tool}) @@ -296,11 +266,6 @@ def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# 6.5: Post effects still work on text-only messages -# --------------------------------------------------------------------------- - - class TestPostEffectsOnText: """Post effects (Confabulation, MalformedJson-on-text, SuccessFraming) work.""" @@ -316,7 +281,6 @@ def test_confabulation_injects_template(self): result_text = message["content"][0]["text"] assert result_text != original_text - # Should contain original text fragments assert "sunny" in result_text or "warm" in result_text def test_malformed_json_on_text(self): @@ -330,7 +294,6 @@ def test_malformed_json_on_text(self): result_text = message["content"][0]["text"] assert result_text != '{"key": "value", "nested": {"a": 1}}' - # Should be truncated (roughly half the original) assert len(result_text) < len('{"key": "value", "nested": {"a": 1}}') def test_success_framing_prepends_prefix(self): @@ -343,10 +306,8 @@ def test_success_framing_prepends_prefix(self): plugin.after_model_invocation(event) result_text = message["content"][0]["text"] - # Should start with one of the success prefixes has_prefix = any(result_text.startswith(p) for p in SuccessFraming._SUCCESS_PREFIXES) assert has_prefix - # Original text should still be present assert "Here is the result." in result_text def test_confabulation_plus_success_framing(self): @@ -360,7 +321,6 @@ def test_confabulation_plus_success_framing(self): plugin.after_model_invocation(event) result_text = message["content"][0]["text"] - # Should start with a success prefix (SuccessFraming applied last) has_prefix = any(result_text.startswith(p) for p in SuccessFraming._SUCCESS_PREFIXES) assert has_prefix @@ -368,11 +328,6 @@ def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# 6.6: Mixed pre+post still produces one turn (pre wins) -# --------------------------------------------------------------------------- - - class TestMixedPrePostCase: """Mixed pre+post effects: pre wins, post does NOT double-corrupt.""" @@ -381,13 +336,11 @@ def test_full_refusal_plus_malformed_json(self): _set_chaos_case([FullRefusal(), MalformedJson()]) plugin = ChaosPlugin() - # Pre-hook fires and cancels pre_event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(pre_event) cancel_text = pre_event.cancel assert cancel_text in FullRefusal._REFUSAL_TEMPLATES - # SDK builds cancel message, MessageAddedEvent fires cancel_message = {"role": "assistant", "content": [{"text": cancel_text}]} post_event = _make_event(cancel_message) plugin.after_model_invocation(post_event) @@ -401,12 +354,10 @@ def test_empty_response_plus_success_framing(self): _set_chaos_case([EmptyResponse(), SuccessFraming()]) plugin = ChaosPlugin() - # Pre-hook fires pre_event = BeforeModelCallEvent(agent=MagicMock()) plugin.before_model_invocation(pre_event) assert pre_event.cancel == " " - # SDK builds cancel message with single space cancel_message = {"role": "assistant", "content": [{"text": " "}]} post_event = _make_event(cancel_message) plugin.after_model_invocation(post_event) @@ -418,13 +369,8 @@ def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# 6.7: MalformedJson DOES reach structured-output toolUse (replaces old guard test) -# --------------------------------------------------------------------------- - - class TestMalformedJsonReachesStructuredOutput: - """MalformedJson reaches structured-output toolUse (Guard 2 relaxed for it).""" + """MalformedJson reaches structured-output toolUse (relaxed for it).""" def test_malformed_json_corrupts_structured_output_tooluse(self): """MalformedJson DOES corrupt a structured-output toolUse block.""" @@ -447,13 +393,12 @@ def test_malformed_json_corrupts_structured_output_tooluse(self): plugin.after_model_invocation(event) - # The toolUse input should now be a corrupted string, not a dict tool_use_block = message["content"][0]["toolUse"] assert isinstance(tool_use_block["input"], str) assert not tool_use_block["input"].endswith("}") def test_other_post_effects_still_skip_tooluse(self): - """Confabulation on a toolUse message is skipped (Guard 2 only relaxed for MalformedJson).""" + """Confabulation on a toolUse message is skipped (only relaxed for MalformedJson).""" _set_chaos_case([Confabulation()]) plugin = ChaosPlugin() message = _tooluse_assistant_message() @@ -462,24 +407,18 @@ def test_other_post_effects_still_skip_tooluse(self): plugin.after_model_invocation(event) - # Content should be UNCHANGED — guard skipped corruption assert message["content"] == original_content def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# Effect family validation (Fix A) -# --------------------------------------------------------------------------- - - class TestEffectFamilyValidation: - """Effects placed in the wrong category are rejected.""" + """Effects placed in the wrong category are rejected structurally by Pydantic.""" def test_tool_effect_in_model_effects_rejected(self): - """A ToolEffect under model_effects raises ValueError.""" - with pytest.raises(ValueError, match="is not a ModelEffect"): + """A ToolEffect under model_effects is rejected by discriminated union.""" + with pytest.raises(PydanticValidationError, match="union_tag_invalid"): ChaosCase( name="bad", input="test", @@ -487,8 +426,8 @@ def test_tool_effect_in_model_effects_rejected(self): ) def test_model_effect_in_tool_effects_rejected(self): - """A ModelEffect under tool_effects raises ValueError.""" - with pytest.raises(ValueError, match="is not a ToolEffect"): + """A ModelEffect under tool_effects is rejected by discriminated union.""" + with pytest.raises(PydanticValidationError, match="union_tag_invalid"): ChaosCase( name="bad", input="test", @@ -497,7 +436,7 @@ def test_model_effect_in_tool_effects_rejected(self): def test_model_effect_in_tool_effects_rejected_via_model_validate(self): """A ModelEffect under tool_effects is rejected on the model_validate (dict) path.""" - with pytest.raises(PydanticValidationError, match="is not a ToolEffect"): + with pytest.raises(PydanticValidationError, match="union_tag_invalid"): ChaosCase.model_validate( { "name": "bad_tool", @@ -508,7 +447,7 @@ def test_model_effect_in_tool_effects_rejected_via_model_validate(self): def test_tool_effect_in_model_effects_rejected_via_model_validate(self): """A ToolEffect under model_effects is rejected on the model_validate (dict) path.""" - with pytest.raises(PydanticValidationError, match="is not a ModelEffect"): + with pytest.raises(PydanticValidationError, match="union_tag_invalid"): ChaosCase.model_validate( { "name": "bad_model", @@ -518,18 +457,22 @@ def test_tool_effect_in_model_effects_rejected_via_model_validate(self): ) def test_named_model_key_rejected(self): - """A non-'*' key in model_effects is rejected.""" - with pytest.raises(ValueError, match="model targeting not yet"): + """A non-'*' key in model_effects is rejected by Literal constraint.""" + with pytest.raises(PydanticValidationError, match="literal_error"): ChaosCase( name="bad", input="test", effects={"model_effects": {"claude-sonnet": [MalformedJson()]}}, ) - -# --------------------------------------------------------------------------- -# Ordinary dynamic tool not corrupted (Fix C) -# --------------------------------------------------------------------------- + def test_bogus_category_rejected(self): + """An unknown effects category is rejected by extra='forbid'.""" + with pytest.raises(PydanticValidationError, match="extra_forbidden"): + ChaosCase( + name="bad", + input="test", + effects={"bogus": {"x": []}}, + ) class TestOrdinaryDynamicToolNotCorrupted: @@ -552,18 +495,12 @@ def test_ordinary_dynamic_tool_unchanged(self): plugin.after_model_invocation(event) - # Should be UNCHANGED — ordinary dynamic tool, not structured-output assert message["content"] == original_content def teardown_method(self): _current_chaos_case.set(None) -# --------------------------------------------------------------------------- -# Guard tests (role filtering, passthrough) -# --------------------------------------------------------------------------- - - class TestGuardRoleFiltering: """User and tool result messages are NOT corrupted.""" From d354eabb4ac4577707e8896e15d01f1da8473e18 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Oruganti Date: Thu, 13 Aug 2026 20:19:58 +0000 Subject: [PATCH 8/8] refactor(chaos): reject multi-pre configs, orchestrate hooks over named operations, trim public exports --- src/strands_evals/chaos/__init__.py | 18 +-- src/strands_evals/chaos/case.py | 12 ++ src/strands_evals/chaos/plugin.py | 138 +++++++++--------- tests/strands_evals/chaos/test_case.py | 4 +- tests/strands_evals/chaos/test_model_chaos.py | 56 +++++++ 5 files changed, 147 insertions(+), 81 deletions(-) diff --git a/src/strands_evals/chaos/__init__.py b/src/strands_evals/chaos/__init__.py index 156e3f55..cfd78b4d 100644 --- a/src/strands_evals/chaos/__init__.py +++ b/src/strands_evals/chaos/__init__.py @@ -4,23 +4,18 @@ under tool failures and response corruption scenarios. """ -from .case import ChaosCase +from .case import ChaosCase, ChaosEffects from .effects import ( - ChaosEffect, Confabulation, CorruptValues, EmptyResponse, ExecutionError, FullRefusal, MalformedJson, - ModelEffect, - ModelEffectUnion, NetworkError, RemoveFields, SuccessFraming, Timeout, - ToolEffect, - ToolEffectUnion, TruncateFields, ValidationError, ) @@ -30,23 +25,18 @@ __all__ = [ # Core classes "ChaosCase", + "ChaosEffects", "ChaosExperiment", "ChaosPlugin", - # Effect hierarchy - "ChaosEffect", - "ToolEffect", - "ToolEffectUnion", - # Pre-hook effects (tool call failures) + # Tool effects "Timeout", "NetworkError", "ExecutionError", "ValidationError", - # Post-hook effects (response corruption) "TruncateFields", "RemoveFields", "CorruptValues", - "ModelEffect", - "ModelEffectUnion", + # Model effects "MalformedJson", "EmptyResponse", "Confabulation", diff --git a/src/strands_evals/chaos/case.py b/src/strands_evals/chaos/case.py index 2d237d5d..82a0f785 100644 --- a/src/strands_evals/chaos/case.py +++ b/src/strands_evals/chaos/case.py @@ -87,6 +87,7 @@ class ChaosCase(Case, Generic[InputT, OutputT]): def _validate_effects(self) -> "ChaosCase": """Validate behavioral constraints the type system cannot express.""" self._validate_tool_effects() + self._validate_pre_model_effects() return self def _validate_tool_effects(self) -> None: @@ -98,6 +99,17 @@ def _validate_tool_effects(self) -> None: f"ChaosCase. Use separate ChaosCase instances to test effects independently." ) + def _validate_pre_model_effects(self) -> None: + """At most one pre-hook model effect: pre effects cancel the model call, so only one can win.""" + pre_effects = [e for e in self.model_effects if e.hook == "pre"] + if len(pre_effects) > 1: + names = ", ".join(type(e).__name__ for e in pre_effects) + raise ValueError( + f"model_effects has {len(pre_effects)} pre-hook effects ({names}) — only 1 is allowed per " + f"ChaosCase. Pre-hook effects cancel the model call, so only one can take effect. " + f"Use separate ChaosCase instances to test them independently." + ) + @classmethod def expand( cls, diff --git a/src/strands_evals/chaos/plugin.py b/src/strands_evals/chaos/plugin.py index 7a7d275c..6aab1cb9 100644 --- a/src/strands_evals/chaos/plugin.py +++ b/src/strands_evals/chaos/plugin.py @@ -12,6 +12,7 @@ import json import logging from enum import Enum, auto +from typing import NamedTuple, Protocol, cast from strands.hooks import ( AfterToolCallEvent, @@ -33,14 +34,26 @@ class MessageKind(Enum): - """Classification of messages for model output chaos routing.""" + """Kind of corruptible model output.""" - IRRELEVANT = auto() - ORDINARY_TOOL_USE = auto() STRUCTURED_OUTPUT = auto() FINAL_TEXT = auto() +class ModelOutputTarget(NamedTuple): + """A model output eligible for corruption.""" + + kind: MessageKind + content: list + structured_output_tool_names: set[str] + + +class PreModelEffect(Protocol): + """A model effect that cancels the model call with a message.""" + + def cancel_message(self) -> str: ... + + class ChaosPlugin(Plugin): """Strands Plugin that injects deterministic chaos based on configuration. @@ -141,88 +154,75 @@ def after_tool_call(self, event: AfterToolCallEvent) -> None: @hook # type: ignore[call-overload] def before_model_invocation(self, event: BeforeModelCallEvent) -> None: - """Intercept model calls to inject pre-hook effects (FullRefusal, EmptyResponse). - - Cancels the model call by setting event.cancel to the effect's cancel_message. - No role/toolUse guard is needed — there is no message yet at pre time. - """ - chaos_case = _current_chaos_case.get() - if chaos_case is None or not chaos_case.model_effects: - return - - pre = [e for e in chaos_case.model_effects if e.hook == "pre"] - if not pre: + """Cancel the model call when a pre-hook model effect is configured.""" + effect = self._select_pre_model_effect() + if effect is None: return - - # First pre effect wins (cancel short-circuits, only one can win) - first_pre = pre[0] - if hasattr(first_pre, "cancel_message"): - event.cancel = first_pre.cancel_message() - logger.info("effect=<%s> | injected model pre-hook (cancel)", type(first_pre).__name__) + event.cancel = effect.cancel_message() @hook # type: ignore[call-overload] def after_model_invocation(self, event: MessageAddedEvent) -> None: - """Apply post-hook model effects based on message classification.""" - chaos_case = _current_chaos_case.get() - if chaos_case is None or not chaos_case.model_effects: + """Corrupt eligible model output with the configured post-hook model effects.""" + effects = self._get_post_model_effects() + target = self._classify_model_output(event) + if target is None or not effects: return + event.message["content"] = self._apply_model_effects(effects, target) - # Pre effects already produced the turn — skip post - if any(e.hook == "pre" for e in chaos_case.model_effects): - return + def _select_pre_model_effect(self) -> PreModelEffect | None: + """Return the single configured pre-hook model effect, or None. - post_effects = [e for e in chaos_case.model_effects if e.hook == "post"] - if not post_effects: - return - - kind, content, so_tool_names = self._classify_model_message(event) + ChaosCase validation guarantees at most one pre effect, so no ordering policy is needed. + """ + chaos_case = _current_chaos_case.get() + if chaos_case is None: + return None + for effect in chaos_case.model_effects: + if effect.hook == "pre": + return cast(PreModelEffect, effect) + return None - if kind == MessageKind.IRRELEVANT: - return - elif kind == MessageKind.ORDINARY_TOOL_USE: - return - elif kind == MessageKind.STRUCTURED_OUTPUT: - # Only MalformedJson reaches structured-output toolUse - if not any(isinstance(e, MalformedJson) for e in post_effects): - return - assert content is not None # guaranteed by classifier for STRUCTURED_OUTPUT - corrupted = self._apply_to_model_blocks(post_effects, content, so_tool_names) - else: # FINAL_TEXT - assert content is not None # guaranteed by classifier for FINAL_TEXT - corrupted = self._apply_to_model_blocks(post_effects, content, set()) + def _get_post_model_effects(self) -> list: + """Return the configured post-hook model effects. - event.message["content"] = corrupted - effect_names = ", ".join(type(e).__name__ for e in post_effects) - logger.info("effects=<%s> | applied model output chaos", effect_names) + Empty when a pre effect is configured: the pre effect already produced the turn, + so applying post effects would corrupt it twice. + """ + chaos_case = _current_chaos_case.get() + if chaos_case is None: + return [] + if any(e.hook == "pre" for e in chaos_case.model_effects): + return [] + return [e for e in chaos_case.model_effects if e.hook == "post"] - # Message classification + def _classify_model_output(self, event: MessageAddedEvent) -> ModelOutputTarget | None: + """Return the corruptible target for this message, or None if it must be left alone. - def _classify_model_message(self, event: MessageAddedEvent) -> tuple[MessageKind, list | None, set[str]]: - """Classify a message for chaos routing. Computed once per hook invocation.""" + Ordinary tool dispatch is excluded: MessageAddedEvent fires before dispatch, so + corrupting those blocks breaks the agent loop. + """ message = event.message if message.get("role") != "assistant": - return MessageKind.IRRELEVANT, None, set() + return None content = message.get("content") if content is None: - return MessageKind.IRRELEVANT, None, set() + return None if not isinstance(content, list): - return MessageKind.FINAL_TEXT, content, set() + return ModelOutputTarget(MessageKind.FINAL_TEXT, content, set()) - has_tool_use = any(isinstance(block, dict) and "toolUse" in block for block in content) - if not has_tool_use: - return MessageKind.FINAL_TEXT, content, set() + if not any(isinstance(block, dict) and "toolUse" in block for block in content): + return ModelOutputTarget(MessageKind.FINAL_TEXT, content, set()) - # Has toolUse — determine if it's structured-output structured_output_tool_names = self._get_structured_output_tool_names(event.agent) - has_so = any( + targets_structured_output = any( isinstance(block, dict) and "toolUse" in block and block["toolUse"].get("name", "") in structured_output_tool_names for block in content ) - if has_so: - return MessageKind.STRUCTURED_OUTPUT, content, structured_output_tool_names - return MessageKind.ORDINARY_TOOL_USE, content, set() + if targets_structured_output: + return ModelOutputTarget(MessageKind.STRUCTURED_OUTPUT, content, structured_output_tool_names) + return None def _get_structured_output_tool_names(self, agent) -> set[str]: # type: ignore[type-arg] """Identify structured-output tools via isinstance(tool, StructuredOutputTool).""" @@ -232,16 +232,24 @@ def _get_structured_output_tool_names(self, agent) -> set[str]: # type: ignore[ name for name, tool in agent.tool_registry.dynamic_tools.items() if isinstance(tool, StructuredOutputTool) } - # Model corruption helpers + def _apply_model_effects(self, effects: list, target: ModelOutputTarget) -> list: + """Corrupt the target content with the given post-hook model effects. + + Structured-output toolUse is reachable only by MalformedJson; any other effect + would break the structured-output contract, so it is skipped for that target. + """ + if target.kind is MessageKind.STRUCTURED_OUTPUT: + effects = [e for e in effects if isinstance(e, MalformedJson)] + if not effects: + return target.content + return self._apply_to_model_blocks(effects, target.content, target.structured_output_tool_names) def _apply_to_model_blocks( self, post_effects: list, content: list, structured_output_tool_names: set[str] | None = None ) -> list: """Apply model post effects to content blocks sequentially. - Handles text blocks (Confabulation, SuccessFraming, MalformedJson on text) - and toolUse blocks (MalformedJson on structured-output tool input only). - Ordinary mid-turn toolUse blocks are left untouched. + SuccessFraming runs last so it frames whatever the other effects produced. """ primary = [e for e in post_effects if not isinstance(e, SuccessFraming)] framing = [e for e in post_effects if isinstance(e, SuccessFraming)] diff --git a/tests/strands_evals/chaos/test_case.py b/tests/strands_evals/chaos/test_case.py index 644112c8..e90d494f 100644 --- a/tests/strands_evals/chaos/test_case.py +++ b/tests/strands_evals/chaos/test_case.py @@ -53,8 +53,8 @@ def test_case_with_multiple_effects_per_tool(self): ) def test_unknown_effect_category_raises(self): - """Unknown effect category keys should be rejected.""" - with pytest.raises(ValueError, match="Unknown effect categories"): + """Unknown effect category keys should be rejected by the ChaosEffects schema.""" + with pytest.raises(ValueError, match="extra_forbidden"): ChaosCase( name="bad_category", input="hello", diff --git a/tests/strands_evals/chaos/test_model_chaos.py b/tests/strands_evals/chaos/test_model_chaos.py index 174ead02..a7137c3c 100644 --- a/tests/strands_evals/chaos/test_model_chaos.py +++ b/tests/strands_evals/chaos/test_model_chaos.py @@ -475,6 +475,62 @@ def test_bogus_category_rejected(self): ) +class TestSinglePreModelEffect: + """At most one pre-hook model effect per case — pre effects cancel the model call.""" + + def test_two_pre_effects_rejected(self): + """FullRefusal + EmptyResponse (both pre) is rejected, naming both effects.""" + with pytest.raises(PydanticValidationError, match="only 1 is allowed"): + ChaosCase( + name="two_pre", + input="test", + effects={"model_effects": {"*": [FullRefusal(), EmptyResponse()]}}, + ) + + def test_two_pre_effects_rejected_via_model_validate(self): + """Two pre effects are rejected on the model_validate (dict) path.""" + with pytest.raises(PydanticValidationError, match="only 1 is allowed"): + ChaosCase.model_validate( + { + "name": "two_pre", + "input": "test", + "effects": { + "model_effects": {"*": [{"effect_type": "full_refusal"}, {"effect_type": "empty_response"}]} + }, + } + ) + + def test_rejection_names_both_effects(self): + """The error message identifies both offending pre effects.""" + with pytest.raises(PydanticValidationError) as exc_info: + ChaosCase( + name="two_pre", + input="test", + effects={"model_effects": {"*": [FullRefusal(), EmptyResponse()]}}, + ) + message = str(exc_info.value) + assert "FullRefusal" in message + assert "EmptyResponse" in message + + def test_single_pre_effect_accepted(self): + """One pre effect alone is valid.""" + case = ChaosCase( + name="one_pre", + input="test", + effects={"model_effects": {"*": [FullRefusal()]}}, + ) + assert len(case.model_effects) == 1 + + def test_pre_plus_post_mix_accepted(self): + """A pre + post mix is valid — only multiple pre effects are rejected.""" + case = ChaosCase( + name="mixed", + input="test", + effects={"model_effects": {"*": [FullRefusal(), MalformedJson()]}}, + ) + assert len(case.model_effects) == 2 + + class TestOrdinaryDynamicToolNotCorrupted: """An ordinary dynamic tool (not StructuredOutputTool) is NOT corrupted."""