Fix harm category normalization in scorer evaluation

Author: San-exe

pyrit/models/harm_category.py

@@ -100,6 +100,7 @@ def _initialize_aliases(cls) -> None:
             {
                 "ableism": [cls.REPRESENTATIONAL, cls.HATE_SPEECH],
                 "bias": [cls.REPRESENTATIONAL, cls.HATE_SPEECH],
+                "fairness_bias": [cls.REPRESENTATIONAL],
                 "bullying": [cls.HARASSMENT],
                 "chemistry": [cls.CBRN],
                 "controlled/regulated substances": [cls.DRUG_USE],

pyrit/score/scorer_evaluation/scorer_evaluator.py

@@ -237,6 +237,7 @@ async def run_evaluation_async(
         # Run evaluation
         metrics = await self.evaluate_dataset_async(
             labeled_dataset=combined_dataset,
+            harm_category=dataset_files.harm_category,
             num_scorer_trials=num_scorer_trials,
             max_concurrency=max_concurrency,
         )
@@ -352,6 +353,7 @@ def _should_skip_evaluation(
     async def evaluate_dataset_async(
         self,
         labeled_dataset: HumanLabeledDataset,
+        harm_category: str | None = None,
         num_scorer_trials: int = 1,
         max_concurrency: int = 10,
     ) -> ScorerMetrics:
@@ -364,6 +366,7 @@ async def evaluate_dataset_async(
 
         Args:
             labeled_dataset (HumanLabeledDataset): The HumanLabeledDataset to evaluate the scorer against.
+            harm_category (str | None): The configured harm category for the evaluation.
             num_scorer_trials (int): The number of trials to run the scorer on all responses.
             max_concurrency (int): Maximum number of concurrent scoring requests. Defaults to 10.
 
@@ -386,16 +389,17 @@ async def evaluate_dataset_async(
         # Transpose human scores so each row is a complete set of scores across all responses
         all_human_scores = np.array(human_scores_list).T
 
-        harm_category = None
+        dataset_harm_category = None
         if labeled_dataset.metrics_type == MetricsType.HARM and labeled_dataset.entries:
             first_entry = labeled_dataset.entries[0]
             if isinstance(first_entry, HarmHumanLabeledEntry):
-                harm_category = first_entry.harm_category
-            if harm_category is None:
+                dataset_harm_category = first_entry.harm_category
+            if dataset_harm_category is None:
                 raise ValueError(
                     "harm_category must be set in HarmHumanLabeledEntry for HARM datasets. "
                     "Ensure all entries have a valid harm_category."
                 )
+        evaluation_harm_category = harm_category or dataset_harm_category
 
         # Run scoring trials and measure timing
         all_model_scores_list = []
@@ -414,7 +418,10 @@ async def evaluate_dataset_async(
             total_scored_items += len(score_groups)
             score_values: list[bool | float] = []
             for index, scores in enumerate(score_groups):
-                score = self._select_evaluation_score(scores=scores, harm_category=harm_category)
+                score = self._select_evaluation_score(
+                    scores=scores,
+                    harm_category=evaluation_harm_category,
+                )
                 if score is None:
                     determined_responses[index] = False
                     score_values.append(False if self.expected_metrics_type == MetricsType.OBJECTIVE else 0.0)

tests/unit/score/test_scorer_evaluator.py

@@ -958,6 +958,25 @@ class TestSelectEvaluationScore:
     def _score(*, category: list[str] | None) -> Score:
         return Score(score_type="float_scale", score_value="0.5", score_category=category)
 
+    def test_accepts_fairness_bias_score_for_representational_evaluation(self) -> None:
+        score = self._score(category=["fairness_bias"])
+        assert (
+            ScorerEvaluator._select_evaluation_score(
+                scores=[score],
+                harm_category="REPRESENTATIONAL",
+            )
+            is score
+        )
+
+    def test_accepts_sexual_score_for_sexual_content_evaluation(self) -> None:
+        score = self._score(category=["sexual"])
+        assert (
+            ScorerEvaluator._select_evaluation_score(
+                scores=[score],
+                harm_category="SEXUAL_CONTENT",
+            )
+            is score
+        )
     @pytest.mark.parametrize("category", list(AzureContentFilterScorer._CATEGORY_EVAL_FILES))
     @pytest.mark.parametrize("multiple_scores", [False, True])
     def test_azure_categories_match_registered_evaluation(self, category: TextCategory, multiple_scores: bool) -> None: