FIX: keep template conditions until their parameters are rendered

pyrit/models/seeds/seed.py

@@ -9,6 +9,7 @@
 
 from __future__ import annotations
 
+import functools
 import logging
 import re
 import uuid
@@ -23,7 +24,7 @@
 from pyrit.models.seeds.seed_origin import SeedOrigin
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Callable, Iterator
     from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -81,25 +82,66 @@ def __repr__(self) -> str:
         """
         return f"{{{{ {self._undefined_name} }}}}" if self._undefined_name else ""
 
+    # A placeholder cannot decide a branch or a loop: answering now would drop the
+    # {% if %} or {% for %} tags, and the later render could not decide again.
     def __iter__(self) -> Iterator[object]:
         """
-        Return an empty iterator to prevent iteration over undefined variables.
+        Defer rendering instead of iterating over an unresolved variable.
 
-        Returns:
-            Iterator[object]: Empty iterator.
+        Raises:
+            _DeferRenderError: Always.
 
         """
-        return iter([])
+        raise _DeferRenderError(self._undefined_name)
 
     def __bool__(self) -> bool:
         """
-        Evaluate as truthy to avoid falsey-branch side effects.
+        Defer rendering instead of testing an unresolved variable.
 
-        Returns:
-            bool: Always True.
+        Raises:
+            _DeferRenderError: Always.
+
+        """
+        raise _DeferRenderError(self._undefined_name)
 
+    def __eq__(self, other: object) -> bool:
         """
-        return True  # Ensures it doesn't evaluate to False
+        Defer rendering instead of comparing an unresolved variable.
+
+        Raises:
+            _DeferRenderError: Always.
+
+        """
+        raise _DeferRenderError(self._undefined_name)
+
+    def __ne__(self, other: object) -> bool:
+        """
+        Defer rendering instead of comparing an unresolved variable.
+
+        Raises:
+            _DeferRenderError: Always.
+
+        """
+        raise _DeferRenderError(self._undefined_name)
+
+    __hash__ = Undefined.__hash__
+
+
+class _DeferRenderError(Exception):
+    """Raised when an unresolved variable would decide a branch or a loop."""
+
+
+def _deferring(function: Callable[..., Any]) -> Callable[..., Any]:
+    # Jinja tests such as `is defined` and the `default` filter check the value's type, not its truth.
+    # functools.wraps keeps Jinja's pass_environment marker, so the value may not be the first argument.
+    @functools.wraps(function)
+    def deferring(*args: Any, **kwargs: Any) -> Any:
+        for arg in args:
+            if isinstance(arg, PartialUndefined):
+                raise _DeferRenderError(arg._undefined_name)
+        return function(*args, **kwargs)
+
+    return deferring
 
 
 class Seed(BaseModel):
@@ -223,11 +265,16 @@ def render_template_value_silent(self, **kwargs: Any) -> str:
 
         # Create a Jinja template with PartialUndefined placeholders
         env = SandboxedEnvironment(undefined=PartialUndefined)
+        env.tests = {name: _deferring(test) for name, test in env.tests.items()}
+        env.filters["default"] = env.filters["d"] = _deferring(env.filters["default"])
         is_jinja_template = env.from_string(self.value)
 
         try:
             # Render the template with the provided kwargs
             return is_jinja_template.render(**kwargs)
+        except _DeferRenderError:
+            # A missing parameter decides a branch or a loop - preserve the template as-is
+            return self.value
         except Exception as e:
             logger.error("Error rendering template: %s", e)
             return self.value

tests/unit/executor/attack/multi_turn/test_crescendo.py

@@ -710,6 +710,7 @@ async def test_setup_sets_adversarial_chat_system_prompt(
         call_args = mock_adversarial_chat.set_system_prompt_async.call_args
         assert "Test objective" in call_args.kwargs["system_prompt"]
         assert "15" in call_args.kwargs["system_prompt"]  # Check for the max_turns value
+        assert "Prior Conversation Context" not in call_args.kwargs["system_prompt"]
         assert call_args.kwargs["conversation_id"] == basic_context.session.adversarial_chat_conversation_id
 
     async def test_setup_handles_prepended_conversation_with_refusal(

tests/unit/models/test_seed.py

@@ -9,6 +9,9 @@
 
 import numpy as np
 import pytest
+import yaml
+from jinja2 import StrictUndefined
+from jinja2.sandbox import SandboxedEnvironment
 from PIL import Image
 from scipy.io import wavfile
 
@@ -204,6 +207,76 @@ def test_render_template_value_silent_blocks_ssti_via_endraw_injection():
     assert "__class__" not in result or result == raw_wrapped
 
 
+@pytest.mark.parametrize(
+    ("template_value", "parameters", "expected"),
+    [
+        ("{% if flag %}A{% else %}B{% endif %}", {"flag": True}, "A"),
+        ("{% if flag %}A{% else %}B{% endif %}", {"flag": False}, "B"),
+        ("{% if flag %}A{% else %}B{% endif %}", {"flag": None}, "B"),
+        ("{% if other %}A{% elif flag %}B{% else %}C{% endif %}", {"other": False, "flag": False}, "C"),
+        ('{{ "A" if flag else "B" }}', {"flag": False}, "B"),
+        ("{% set local = flag %}{% if local %}A{% else %}B{% endif %}", {"flag": False}, "B"),
+        ("{% if flag is defined %}A{% else %}B{% endif %}", {}, "B"),
+        ("{{ flag | default('B') }}", {}, "B"),
+        ("{% if flag == 'a' %}A{% else %}B{% endif %}", {"flag": "b"}, "B"),
+        ("{% if flag is filter %}A{% else %}B{% endif %}", {"flag": "upper"}, "A"),
+        ("{% if flag is test %}A{% else %}B{% endif %}", {"flag": "nope"}, "B"),
+        ("{% for item in ['A', 'B'] if flag %}{{ item }}{% endfor %}", {"flag": False}, ""),
+        (
+            "{% macro show(rows) %}{% for row in rows %}[{{ row }}]{% endfor %}{% endmacro %}{{ show(items) }}",
+            {"items": ["x", "y"]},
+            "[x][y]",
+        ),
+    ],
+)
+def test_seed_prompt_keeps_template_whose_missing_parameter_decides_a_branch(template_value, parameters, expected):
+    template = SeedPrompt(value=template_value, data_type="text", is_jinja_template=True)
+
+    assert template.value == template_value
+    assert template.render_template_value(**parameters) == expected
+
+
+def test_render_template_value_silent_renders_condition_once_its_parameters_are_provided():
+    template = SeedPrompt(
+        value="{% if flag %}{{ datasets_path }} {{ prompt }}{% endif %}",
+        data_type="text",
+        is_jinja_template=True,
+    )
+
+    assert template.render_template_value_silent(flag=True, datasets_path="data") == "data {{ prompt }}"
+
+
+def test_render_template_value_silent_renders_if_guard_on_loop_variable():
+    seed = SeedPrompt(
+        value="{% for item in items %}{% if item %}[{{ item }}]{% endif %}{% endfor %}",
+        data_type="text",
+        is_jinja_template=True,
+    )
+
+    assert seed.render_template_value_silent(items=["a", "", "b"]) == "[a][b]"
+
+
+_CONVERSATION_CONTEXT_TEMPLATES = sorted(
+    path
+    for path in pathlib.Path(DATASETS_PATH, "executors").rglob("*.yaml")
+    if "{% if conversation_context %}" in path.read_text(encoding="utf-8")
+)
+
+
+@pytest.mark.parametrize("template_path", _CONVERSATION_CONTEXT_TEMPLATES, ids=lambda path: path.stem)
+@pytest.mark.parametrize("conversation_context", [None, "<context>"])
+def test_loaded_template_renders_like_its_source(template_path, conversation_context):
+    seed_prompt = SeedPrompt.from_yaml_file(template_path)
+    parameters = {name: f"<{name}>" for name in seed_prompt.parameters or []}
+    parameters["conversation_context"] = conversation_context
+    source = yaml.safe_load(template_path.read_text(encoding="utf-8"))["value"]
+
+    expected = SandboxedEnvironment(undefined=StrictUndefined).from_string(source).render(**parameters)
+
+    assert seed_prompt.render_template_value(**parameters) == expected
+    assert ("<context>" in expected) == (conversation_context is not None)
+
+
 def test_seed_group_untrusted_auto_escapes():
     group = SeedGroup(seeds=[{"value": '{{ "".__class__ }}', "data_type": "text"}])
     seed = group.prompts[0]