FIX: Krippendorff alpha returns 1.0 instead of NaN for unpairable identical ratings

The num_categories == 1 early return reported perfect agreement before
the total_n == 0 degenerate check, so data where every item is rated by
a single rater (no pairable coincidences) produced alpha = 1.0 whenever
all ratings happened to be identical. The statistic is undefined in that
case; the docstring already promises NaN for undefined input.

Move the single-category early return after the coincidence construction
so unpairable data reaches the existing total_n == 0 NaN path while
pairable identical ratings still return 1.0.

Author: ManoharPaturi

pyrit/score/scorer_evaluation/krippendorff.py

@@ -238,10 +238,11 @@ def krippendorff_alpha(
 
     Returns:
         float: Krippendorff's alpha, where 1.0 indicates perfect agreement. The
-        value can be below 0 when agreement is worse than chance. ``np.nan`` is
-        returned when the statistic is undefined (e.g., fewer than two usable
-        ratings or zero expected disagreement with non-zero observed
-        disagreement).
+            value can be below 0 when agreement is worse than chance. ``np.nan`` is
+            returned when the statistic is undefined (e.g., fewer than two usable
+            ratings, no item with two or more ratings to form a pairable
+            coincidence, or zero expected disagreement with non-zero observed
+            disagreement).
 
     Raises:
         ValueError: If ``level_of_measurement`` is not ``"ordinal"``.
@@ -254,11 +255,6 @@ def krippendorff_alpha(
     if len(valid_ratings) < 2:
         return np.nan
 
-    # All ratings identical - perfect agreement
-    num_categories = len(categories)
-    if num_categories == 1:
-        return 1.0
-
     # Build value counts matrix
     value_counts = _build_value_counts_matrix(data, valid_mask, categories)
 
@@ -268,10 +264,19 @@ def krippendorff_alpha(
     # Build expected coincidence matrix
     expected_matrix, n_v, total_n = _build_expected_matrix(coincidence_matrix)
 
-    # Check for degenerate case
+    # Check for degenerate case. This also covers the single-category case where
+    # no item has two or more ratings: with nothing pairable, alpha is undefined
+    # even when every rating happens to be identical, so it must not report 1.0.
     if total_n == 0:
         return np.nan
 
+    # All ratings identical and pairable - perfect agreement. The general path
+    # below would also return 1.0 (every ordinal distance is zero); the early
+    # return just skips building the all-zero distance matrix.
+    num_categories = len(categories)
+    if num_categories == 1:
+        return 1.0
+
     # Build ordinal distance matrix
     distance_matrix = _build_ordinal_distance_matrix(num_categories, n_v)
 

tests/unit/score/test_krippendorff.py

@@ -69,6 +69,31 @@ def test_insufficient_data_returns_nan(self):
         result = krippendorff_alpha(data, level_of_measurement="ordinal")
         assert np.isnan(result)
 
+    def test_unpairable_identical_ratings_returns_nan(self):
+        """Test that identical ratings with no pairable item return NaN, not 1.0.
+
+        Every item is rated by exactly one rater, so no coincidence can be
+        formed and alpha is undefined, even though all ratings happen to be
+        identical. Reporting 1.0 would claim perfect reliability from data that
+        cannot measure reliability at all.
+        """
+        data = np.array([[1, np.nan], [np.nan, 1]])
+        result = krippendorff_alpha(data, level_of_measurement="ordinal")
+        assert np.isnan(result)
+
+    def test_unpairable_identical_ratings_mixed_with_pairable_returns_value(self):
+        """Test that one pairable item is enough to define alpha."""
+        # Item 0 is rated by two raters (pairable); item 1 by a single rater.
+        data = np.array([[1, 2], [1, np.nan]])
+        result = krippendorff_alpha(data, level_of_measurement="ordinal")
+        assert not np.isnan(result)
+
+    def test_pairable_identical_ratings_still_return_one(self):
+        """Test that identical ratings with a pairable item still return 1.0."""
+        data = np.array([[2, 2], [2, np.nan]])
+        result = krippendorff_alpha(data, level_of_measurement="ordinal")
+        assert result == 1.0
+
     def test_all_missing_returns_nan(self):
         """Test that all missing values returns NaN."""
         data = np.array([[np.nan, np.nan], [np.nan, np.nan]])