FEAT: add raise-on-empty variants to true/false score aggregators (#2984)

Co-authored-by: Roman Lutz <romanlutz13@gmail.com>
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: 3 people

pyrit/score/true_false/true_false_composite_scorer.py

@@ -39,8 +39,9 @@ class TrueFalseCompositeScorer(TrueFalseScorer):
     Children are true/false scorers of any evidence kind, so a scorer over a message can be
     composed with one over evidence that is not a message at all.
 
-    Built-in AND, OR, and MAJORITY aggregators opt into order-independent evaluation
-    identity. Duplicates remain significant; custom aggregators and execution stay ordered.
+    Built-in AND, OR, and MAJORITY aggregators, including their raise-on-empty variants,
+    opt into order-independent evaluation identity. Duplicates remain significant;
+    custom aggregators and execution stay ordered.
     """
 
     def __init__(
@@ -87,6 +88,9 @@ def _build_identifier(self) -> ComponentIdentifier:
                 TrueFalseScoreAggregator.AND,
                 TrueFalseScoreAggregator.OR,
                 TrueFalseScoreAggregator.MAJORITY,
+                TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY,
+                TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY,
+                TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY,
             )
         )
         return self._create_identifier(

pyrit/score/true_false/true_false_score_aggregator.py

@@ -98,6 +98,7 @@ def _create_aggregator(
     result_func: Callable[[list[bool | None]], bool | None],
     true_msg: str,
     false_msg: str,
+    raise_on_empty: bool = False,
 ) -> TrueFalseAggregatorFunc:
     """
     Create a True/False aggregator using a result function over boolean values.
@@ -109,6 +110,7 @@ def _create_aggregator(
             constituent scores, and a ``None`` result means no verdict was reachable.
         true_msg (str): Description to use when the result is True.
         false_msg (str): Description to use when the result is False.
+        raise_on_empty (bool): Whether to raise ValueError when no scores are provided. Defaults to False.
 
     Returns:
         TrueFalseAggregatorFunc: Aggregator function that reduces a sequence of true/false Scores
@@ -125,6 +127,8 @@ def aggregator(scores: Iterable[Score]) -> ScoreAggregatorResult:
                 raise ValueError("All scores must be of type 'true_false'.")
 
         if not scores_list:
+            if raise_on_empty:
+                raise ValueError("No scores available for aggregation")
             # No scores; return a neutral result
             return ScoreAggregatorResult(
                 value=False,
@@ -167,6 +171,7 @@ def _create_binary_aggregator(
     op: BinaryBoolOp,
     true_msg: str,
     false_msg: str,
+    raise_on_empty: bool = False,
 ) -> TrueFalseAggregatorFunc:
     """
     Turn a binary operator over verdicts (e.g. ``_and``) into an aggregation function.
@@ -176,6 +181,7 @@ def _create_binary_aggregator(
         op (BinaryBoolOp): Binary three-valued operator to apply.
         true_msg (str): Description to use when the result is True.
         false_msg (str): Description to use when the result is False.
+        raise_on_empty (bool): Whether to raise ValueError when no scores are provided. Defaults to False.
 
     Returns:
         TrueFalseAggregatorFunc: Aggregator function that reduces scores using the binary operator.
@@ -185,6 +191,7 @@ def _create_binary_aggregator(
         result_func=lambda bs, _op=op: functools.reduce(_op, bs),
         true_msg=true_msg,
         false_msg=false_msg,
+        raise_on_empty=raise_on_empty,
     )
 
 
@@ -229,3 +236,27 @@ class TrueFalseScoreAggregator:
         true_msg="A strict majority of constituent scorers returned True in a MAJORITY composite scorer.",
         false_msg="A strict majority of constituent scorers did not return True in a MAJORITY composite scorer.",
     )
+
+    AND_RAISE_ON_EMPTY: TrueFalseAggregatorFunc = _create_binary_aggregator(
+        "AND_RAISE_ON_EMPTY",
+        _and,
+        "All constituent scorers returned True in an AND composite scorer.",
+        "At least one constituent scorer returned False in an AND composite scorer.",
+        raise_on_empty=True,
+    )
+
+    OR_RAISE_ON_EMPTY: TrueFalseAggregatorFunc = _create_binary_aggregator(
+        "OR_RAISE_ON_EMPTY",
+        _or,
+        "At least one constituent scorer returned True in an OR composite scorer.",
+        "All constituent scorers returned False in an OR composite scorer.",
+        raise_on_empty=True,
+    )
+
+    MAJORITY_RAISE_ON_EMPTY: TrueFalseAggregatorFunc = _create_aggregator(
+        "MAJORITY_RAISE_ON_EMPTY",
+        result_func=_majority,
+        true_msg="A strict majority of constituent scorers returned True in a MAJORITY composite scorer.",
+        false_msg="A strict majority of constituent scorers did not return True in a MAJORITY composite scorer.",
+        raise_on_empty=True,
+    )

tests/unit/score/test_scorer_evaluation_identifier.py

@@ -101,7 +101,14 @@ def test_eval_hash_matches_free_function(self):
 class TestCompositeEvaluationOrder:
     @pytest.mark.parametrize(
         "aggregator",
-        [TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND, TrueFalseScoreAggregator.MAJORITY],
+        [
+            TrueFalseScoreAggregator.OR,
+            TrueFalseScoreAggregator.AND,
+            TrueFalseScoreAggregator.MAJORITY,
+            TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY,
+            TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY,
+            TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY,
+        ],
     )
     def test_permutations_preserve_eval_identity_and_content_order(self, aggregator: TrueFalseAggregatorFunc) -> None:
         children = [SubStringScorer(substring=value) for value in ("a", "b", "c")]
@@ -120,15 +127,25 @@ def test_permutations_preserve_eval_identity_and_content_order(self, aggregator:
             assert restored.hash == identifier.hash
             assert ScorerEvaluationIdentifier(restored).eval_hash == identifier.eval_hash
 
-    def test_nested_permutations_preserve_eval_identity(self) -> None:
+    @pytest.mark.parametrize(
+        "outer_aggregator, inner_aggregator",
+        [
+            (TrueFalseScoreAggregator.OR, TrueFalseScoreAggregator.AND),
+            (TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY),
+            (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY),
+        ],
+    )
+    def test_nested_permutations_preserve_eval_identity(
+        self, *, outer_aggregator: TrueFalseAggregatorFunc, inner_aggregator: TrueFalseAggregatorFunc
+    ) -> None:
         a, b, c = [SubStringScorer(substring=value) for value in ("a", "b", "c")]
         first = TrueFalseCompositeScorer(
-            aggregator=TrueFalseScoreAggregator.OR,
-            scorers=[TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.AND, scorers=[a, b]), c],
+            aggregator=outer_aggregator,
+            scorers=[TrueFalseCompositeScorer(aggregator=inner_aggregator, scorers=[a, b]), c],
         )
         second = TrueFalseCompositeScorer(
-            aggregator=TrueFalseScoreAggregator.OR,
-            scorers=[c, TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.AND, scorers=[b, a])],
+            aggregator=outer_aggregator,
+            scorers=[c, TrueFalseCompositeScorer(aggregator=inner_aggregator, scorers=[b, a])],
         )
         assert first.get_identifier().eval_hash == second.get_identifier().eval_hash
         assert first.get_identifier().hash != second.get_identifier().hash
@@ -142,22 +159,39 @@ def test_configuration_aggregator_and_multiplicity_remain_distinct(self) -> None
             (TrueFalseScoreAggregator.MAJORITY, [a, b]),
             (TrueFalseScoreAggregator.MAJORITY, [a, a, b]),
             (TrueFalseScoreAggregator.MAJORITY, [a, b, b]),
+            (TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, [a, b]),
+            (TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY, [a, c]),
+            (TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY, [a, b]),
+            (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, [a, b]),
+            (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, [a, a, b]),
+            (TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY, [a, b, b]),
         ]
         hashes = {
             TrueFalseCompositeScorer(aggregator=aggregator, scorers=scorers).get_identifier().eval_hash
             for aggregator, scorers in configurations
         }
         assert len(hashes) == len(configurations)
 
-    def test_custom_aggregator_with_builtin_name_remains_ordered(self) -> None:
+    @pytest.mark.parametrize(
+        "aggregator",
+        [
+            TrueFalseScoreAggregator.OR,
+            TrueFalseScoreAggregator.AND,
+            TrueFalseScoreAggregator.MAJORITY,
+            TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY,
+            TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY,
+            TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY,
+        ],
+    )
+    def test_custom_aggregator_with_builtin_name_remains_ordered(self, aggregator: TrueFalseAggregatorFunc) -> None:
         def first_score(scores: Iterable[Score]) -> ScoreAggregatorResult:
             return TrueFalseScoreAggregator.OR([next(iter(scores))])
 
-        first_score.__name__ = TrueFalseScoreAggregator.OR.__name__
+        first_score.__name__ = aggregator.__name__
         a, b = [SubStringScorer(substring=value) for value in ("a", "b")]
         first = TrueFalseCompositeScorer(aggregator=first_score, scorers=[a, b]).get_identifier()
         second = TrueFalseCompositeScorer(aggregator=first_score, scorers=[b, a]).get_identifier()
-        builtin = TrueFalseCompositeScorer(aggregator=TrueFalseScoreAggregator.OR, scorers=[a, b]).get_identifier()
+        builtin = TrueFalseCompositeScorer(aggregator=aggregator, scorers=[a, b]).get_identifier()
 
         assert "sub_scorers_order_independent" not in first.params
         assert first.eval_hash != second.eval_hash

tests/unit/score/test_true_false_score_aggregator.py

@@ -276,3 +276,40 @@ def test_generator_of_wrong_type_still_raises():
     )
     with pytest.raises(ValueError, match="must be of type 'true_false'"):
         TrueFalseScoreAggregator.OR(s for s in [bad])
+
+
+# Tests for raise_on_empty behavior
+def test_and_raise_on_empty_with_scores():
+    """Test that AND_RAISE_ON_EMPTY works normally when scores are present."""
+    scores = [_mk_score(True, prr_id="1"), _mk_score(False, prr_id="1")]
+    res = TrueFalseScoreAggregator.AND_RAISE_ON_EMPTY(scores)
+    assert res.value is False
+
+
+def test_or_raise_on_empty_with_scores():
+    """Test that OR_RAISE_ON_EMPTY works normally when scores are present."""
+    scores = [_mk_score(True, prr_id="1"), _mk_score(False, prr_id="1")]
+    res = TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY(scores)
+    assert res.value is True
+
+
+def test_majority_raise_on_empty_with_scores():
+    """Test that MAJORITY_RAISE_ON_EMPTY works normally when scores are present."""
+    scores = [_mk_score(True, prr_id="1"), _mk_score(True, prr_id="1"), _mk_score(False, prr_id="1")]
+    res = TrueFalseScoreAggregator.MAJORITY_RAISE_ON_EMPTY(scores)
+    assert res.value is True
+
+
+@pytest.mark.parametrize("aggregator_name", ["AND_RAISE_ON_EMPTY", "OR_RAISE_ON_EMPTY", "MAJORITY_RAISE_ON_EMPTY"])
+def test_raise_on_empty_aggregators_raise_with_no_scores(aggregator_name):
+    """The strict-empty variants must not answer an empty input with a verdict."""
+    aggregator = getattr(TrueFalseScoreAggregator, aggregator_name)
+    with pytest.raises(ValueError, match="No scores available for aggregation"):
+        aggregator([])
+
+
+def test_raise_on_empty_aggregator_accepts_generators():
+    """A generator with scores must not trip the empty-input guard."""
+    values = [True, False, True]
+    res = TrueFalseScoreAggregator.OR_RAISE_ON_EMPTY(_mk_score(v, prr_id="1") for v in values)
+    assert res.value is True