FIX Seed removal list filters match whole elements instead of JSON substrings (#2811)

Co-authored-by: Roman Lutz <romanlutz13@gmail.com>
Co-authored-by: hannahwestra25 <hannahwestra@microsoft.com>

Author: 3 people

doc/code/memory/8_seed_database.ipynb

@@ -286,7 +286,7 @@
     "\n",
     "For the most part these are user errors, but when you want to remove whole groups rather than individual seeds, use `remove_seed_groups_from_memory`. It applies the same filters, but removes every seed that shares a `prompt_group_id` with any match, so groups are never left partial. Note that it only affects seeds that belong to a group: a matching seed added individually (with no `prompt_group_id`) is skipped, so use `remove_seeds_from_memory` for those.\n",
     "\n",
-    "> **Note on deleting by `value`.** For the remove methods, the `value` filter defaults to full-string equality (`exact=True`), so `remove_seeds_from_memory(value=\"the\")` deletes only seeds whose value is exactly `\"the\"` — not everything containing it. This differs from `get_seeds`, which always matches `value` by substring. Pass `exact=False` to opt into substring deletion when you really want it. As a general rule, preview with the same filters via `get_seeds(...)` first and prefer a specific filter (such as `dataset_name` or `value_sha256`) for deletion.\n",
+    "> **Note on deleting by `value`.** For the remove methods, the `value` filter defaults to full-string equality (`exact=True`), so `remove_seeds_from_memory(value=\"the\")` deletes only seeds whose value is exactly `\"the\"` — not everything containing it. This differs from `get_seeds`, which always matches `value` by substring. The same applies to the list filters: `harm_categories`, `authors`, `groups` and `parameters` must match whole list elements (case-insensitive), so `remove_seeds_from_memory(harm_categories=[\"hate\"])` does not also remove seeds tagged `\"hate_speech\"`. Pass `exact=False` to opt into substring deletion when you really want it. As a general rule, preview with the same filters via `get_seeds(...)` first and prefer a specific filter (such as `dataset_name` or `value_sha256`) for deletion. Because `get_seeds` always matches by substring, the preview is a superset of what the remove methods delete: some previewed seeds will survive the deletion, which is expected.\n",
     "\n",
     "> **Note on file-backed seeds.** For `image_path`, `audio_path`, and `video_path` seeds, removal deletes only the database record; the serialized file on disk is left in place. Delete those files separately if they are no longer needed."
    ]

doc/code/memory/8_seed_database.py

@@ -145,6 +145,6 @@ def print_group(seed_group):
 #
 # For the most part these are user errors, but when you want to remove whole groups rather than individual seeds, use `remove_seed_groups_from_memory`. It applies the same filters, but removes every seed that shares a `prompt_group_id` with any match, so groups are never left partial. Note that it only affects seeds that belong to a group: a matching seed added individually (with no `prompt_group_id`) is skipped, so use `remove_seeds_from_memory` for those.
 #
-# > **Note on deleting by `value`.** For the remove methods, the `value` filter defaults to full-string equality (`exact=True`), so `remove_seeds_from_memory(value="the")` deletes only seeds whose value is exactly `"the"` — not everything containing it. This differs from `get_seeds`, which always matches `value` by substring. Pass `exact=False` to opt into substring deletion when you really want it. As a general rule, preview with the same filters via `get_seeds(...)` first and prefer a specific filter (such as `dataset_name` or `value_sha256`) for deletion.
+# > **Note on deleting by `value`.** For the remove methods, the `value` filter defaults to full-string equality (`exact=True`), so `remove_seeds_from_memory(value="the")` deletes only seeds whose value is exactly `"the"` — not everything containing it. This differs from `get_seeds`, which always matches `value` by substring. The same applies to the list filters: `harm_categories`, `authors`, `groups` and `parameters` must match whole list elements (case-insensitive), so `remove_seeds_from_memory(harm_categories=["hate"])` does not also remove seeds tagged `"hate_speech"`. Pass `exact=False` to opt into substring deletion when you really want it. As a general rule, preview with the same filters via `get_seeds(...)` first and prefer a specific filter (such as `dataset_name` or `value_sha256`) for deletion. Because `get_seeds` always matches by substring, the preview is a superset of what the remove methods delete: some previewed seeds will survive the deletion, which is expected.
 #
 # > **Note on file-backed seeds.** For `image_path`, `audio_path`, and `video_path` seeds, removal deletes only the database record; the serialized file on disk is left in place. Delete those files separately if they are no longer needed.

pyrit/memory/memory_interface.py

@@ -3047,8 +3047,10 @@ def _build_seed_filter_conditions(
         Args:
             value (str): The value to match. By default this matches by substring; pass exact=True to
                 require full-string equality instead. If None, all values are returned.
-            exact (bool): When True, ``value`` is matched by full-string equality rather than substring.
-                Has no effect unless ``value`` is provided. Defaults to False (substring matching).
+            exact (bool): When True, ``value`` is matched by full-string equality rather than substring,
+                and ``harm_categories``, ``authors``, ``groups`` and ``parameters`` must match whole list
+                elements (case-insensitive) rather than substrings of the stored list. Defaults to False
+                (substring matching).
             value_sha256 (Sequence[str] | None): A list of SHA256 hashes of values to match.
                 If None, all values are returned.
             dataset_name (str): The dataset name to match exactly. If None, all dataset names are considered.
@@ -3104,12 +3106,14 @@ def _build_seed_filter_conditions(
         elif seed_type is not None:
             conditions.append(SeedEntry.seed_type == seed_type)
 
-        self._add_list_conditions(field=SeedEntry.harm_categories, values=harm_categories, conditions=conditions)
-        self._add_list_conditions(field=SeedEntry.authors, values=authors, conditions=conditions)
-        self._add_list_conditions(field=SeedEntry.groups, values=groups, conditions=conditions)
+        self._add_list_conditions(
+            field=SeedEntry.harm_categories, values=harm_categories, conditions=conditions, exact=exact
+        )
+        self._add_list_conditions(field=SeedEntry.authors, values=authors, conditions=conditions, exact=exact)
+        self._add_list_conditions(field=SeedEntry.groups, values=groups, conditions=conditions, exact=exact)
 
         if parameters:
-            self._add_list_conditions(field=SeedEntry.parameters, values=parameters, conditions=conditions)
+            self._add_list_conditions(field=SeedEntry.parameters, values=parameters, conditions=conditions, exact=exact)
 
         if metadata:
             conditions.append(self._get_seed_metadata_conditions(metadata=metadata))
@@ -3229,10 +3233,11 @@ def remove_seeds_from_memory(
             value (str): The value to match. For the remove methods this defaults to full-string equality
                 (exact=True) so a short or common value does not delete far more seeds than intended; pass
                 exact=False to match by substring instead. If None, all values are considered.
-            exact (bool): When True, ``value`` is matched by full-string equality rather than substring.
-                Has no effect unless ``value`` is provided. Defaults to True for the remove methods (the
-                safer choice for deletion). Note this differs from get_seeds, which always matches ``value``
-                by substring.
+            exact (bool): When True, ``value`` is matched by full-string equality rather than substring, and
+                ``harm_categories``, ``authors``, ``groups`` and ``parameters`` must match whole list elements
+                (case-insensitive), so ``harm_categories=["hate"]`` does not also remove seeds tagged
+                ``"hate_speech"``. Defaults to True for the remove methods (the safer choice for deletion).
+                Note this differs from get_seeds, which always matches these filters by substring.
             value_sha256 (Sequence[str] | None): A list of SHA256 hashes of values to match.
                 If None, all values are considered.
             dataset_name (str): The dataset name to match exactly. If None, all dataset names are considered.
@@ -3246,9 +3251,9 @@ def remove_seeds_from_memory(
             all harm categories are considered.
                 Specifying multiple harm categories matches only prompts that are marked with all harm categories.
             added_by (str): The user who added the prompts.
-            authors (Sequence[str]): A list of authors to filter by.
-                Note that this filters by substring, so a query for "Adam Jones" may not return results if the record
-                is "A. Jones", "Jones, Adam", etc. If None, all authors are considered.
+            authors (Sequence[str]): A list of authors to filter by. With exact=True (the default) each author
+                must match a stored author exactly (case-insensitive); with exact=False this filters by substring.
+                If None, all authors are considered.
             groups (Sequence[str]): A list of groups to filter by. If None, all groups are considered.
             source (str): The source to filter by. If None, all sources are considered.
             seed_type (SeedType): The type of seed to filter by ("prompt", "objective", or
@@ -3344,10 +3349,11 @@ def remove_seed_groups_from_memory(
             value (str): The value to match. For the remove methods this defaults to full-string equality
                 (exact=True) so a short or common value does not delete far more seeds than intended; pass
                 exact=False to match by substring instead. If None, all values are considered.
-            exact (bool): When True, ``value`` is matched by full-string equality rather than substring.
-                Has no effect unless ``value`` is provided. Defaults to True for the remove methods (the
-                safer choice for deletion). Note this differs from get_seeds, which always matches ``value``
-                by substring.
+            exact (bool): When True, ``value`` is matched by full-string equality rather than substring, and
+                ``harm_categories``, ``authors``, ``groups`` and ``parameters`` must match whole list elements
+                (case-insensitive), so ``harm_categories=["hate"]`` does not also remove seeds tagged
+                ``"hate_speech"``. Defaults to True for the remove methods (the safer choice for deletion).
+                Note this differs from get_seeds, which always matches these filters by substring.
             value_sha256 (Sequence[str] | None): A list of SHA256 hashes of values to match.
                 If None, all values are considered.
             dataset_name (str): The dataset name to match exactly. If None, all dataset names are considered.
@@ -3361,9 +3367,9 @@ def remove_seed_groups_from_memory(
             all harm categories are considered.
                 Specifying multiple harm categories matches only prompts that are marked with all harm categories.
             added_by (str): The user who added the prompts.
-            authors (Sequence[str]): A list of authors to filter by.
-                Note that this filters by substring, so a query for "Adam Jones" may not return results if the record
-                is "A. Jones", "Jones, Adam", etc. If None, all authors are considered.
+            authors (Sequence[str]): A list of authors to filter by. With exact=True (the default) each author
+                must match a stored author exactly (case-insensitive); with exact=False this filters by substring.
+                If None, all authors are considered.
             groups (Sequence[str]): A list of groups to filter by. If None, all groups are considered.
             source (str): The source to filter by. If None, all sources are considered.
             seed_type (SeedType): The type of seed to filter by ("prompt", "objective", or
@@ -3431,11 +3437,25 @@ def remove_seed_groups_from_memory(
 
     def _add_list_conditions(
         self,
+        *,
         field: InstrumentedAttribute[Any],
         conditions: "list[ColumnElement[bool]]",
         values: Sequence[str] | None = None,
+        exact: bool = False,
     ) -> None:
-        if values:
+        if not values:
+            return
+        if exact:
+            # Match whole list elements (case-insensitive) so "hate" does not match "hate_speech" or "whatever".
+            conditions.append(
+                self._get_condition_json_array_match(
+                    json_column=field,
+                    property_path="$",
+                    array_to_match=list(values),
+                    match_mode="all",
+                )
+            )
+        else:
             conditions.extend(field.contains(value) for value in values)
 
     async def _serialize_seed_value_async(self, prompt: Seed) -> str:

tests/unit/memory/memory_interface/test_interface_remove_seeds.py

@@ -249,6 +249,55 @@ async def test_remove_seeds_by_value_exact_is_narrow(sqlite_instance: MemoryInte
     assert remaining[0].value == "the lazy dog"
 
 
+async def test_remove_seeds_by_harm_categories_matches_whole_elements(sqlite_instance: MemoryInterface):
+    seed_prompts = [
+        SeedPrompt(value="exact", harm_categories=["hate"], data_type="text"),
+        SeedPrompt(value="longer_label", harm_categories=["hate_speech"], data_type="text"),
+        SeedPrompt(value="inner_substring", harm_categories=["whatever"], data_type="text"),
+        SeedPrompt(value="wildcard", harm_categories=["selfXharm"], data_type="text"),
+    ]
+    await sqlite_instance.add_seeds_to_memory_async(seeds=seed_prompts, added_by="test")
+
+    assert sqlite_instance.remove_seeds_from_memory(harm_categories=["hate"]) == 1
+    assert sqlite_instance.remove_seeds_from_memory(harm_categories=["self_harm"]) == 0
+
+    remaining = sorted(seed.value for seed in sqlite_instance.get_seeds())
+    assert remaining == ["inner_substring", "longer_label", "wildcard"]
+
+
+async def test_remove_seeds_by_list_filters_is_case_insensitive(sqlite_instance: MemoryInterface):
+    seed_prompts = [
+        SeedPrompt(value="p1", harm_categories=["Violence"], data_type="text"),
+        SeedPrompt(value="p2", harm_categories=["fraud"], data_type="text"),
+    ]
+    await sqlite_instance.add_seeds_to_memory_async(seeds=seed_prompts, added_by="test")
+
+    assert sqlite_instance.remove_seeds_from_memory(harm_categories=["violence"]) == 1
+
+
+async def test_remove_seeds_by_groups_authors_parameters_match_whole_elements(sqlite_instance: MemoryInterface):
+    seed_prompts = [
+        SeedPrompt(value="p1", groups=["team"], authors=["Ann"], parameters=["goal"], data_type="text"),
+        SeedPrompt(value="p2", groups=["team_b"], authors=["Annabel"], parameters=["goal_2"], data_type="text"),
+    ]
+    await sqlite_instance.add_seeds_to_memory_async(seeds=seed_prompts, added_by="test")
+
+    assert sqlite_instance.remove_seeds_from_memory(groups=["team"], authors=["Ann"], parameters=["goal"]) == 1
+
+    remaining = sqlite_instance.get_seeds()
+    assert [seed.value for seed in remaining] == ["p2"]
+
+
+async def test_remove_seeds_by_harm_categories_substring_when_not_exact(sqlite_instance: MemoryInterface):
+    seed_prompts = [
+        SeedPrompt(value="p1", harm_categories=["hate"], data_type="text"),
+        SeedPrompt(value="p2", harm_categories=["hate_speech"], data_type="text"),
+    ]
+    await sqlite_instance.add_seeds_to_memory_async(seeds=seed_prompts, added_by="test")
+
+    assert sqlite_instance.remove_seeds_from_memory(harm_categories=["hate"], exact=False) == 2
+
+
 async def test_remove_seeds_multi_filter_narrowing(sqlite_instance: MemoryInterface):
     seed_prompts = [
         SeedPrompt(value="prompt1", dataset_name="ds1", added_by="user1", data_type="text"),
@@ -325,6 +374,23 @@ async def test_remove_seed_groups_removes_entire_group(sqlite_instance: MemoryIn
     assert remaining[0].value == "keep"
 
 
+async def test_remove_seed_groups_by_harm_categories_matches_whole_elements(sqlite_instance: MemoryInterface):
+    hate_group = SeedGroup(
+        seeds=[SeedPrompt(value="hate", harm_categories=["hate"], data_type="text", sequence=0, role="user")]
+    )
+    speech_group = SeedGroup(
+        seeds=[
+            SeedPrompt(value="hate_speech", harm_categories=["hate_speech"], data_type="text", sequence=0, role="user")
+        ]
+    )
+    await sqlite_instance.add_seed_groups_to_memory_async(prompt_groups=[hate_group, speech_group], added_by="test")
+
+    assert sqlite_instance.remove_seed_groups_from_memory(harm_categories=["hate"]) == 1
+
+    remaining = sqlite_instance.get_seeds()
+    assert [seed.value for seed in remaining] == ["hate_speech"]
+
+
 async def test_remove_seed_groups_spanning_multiple_datasets(sqlite_instance: MemoryInterface):
     group = SeedGroup(
         seeds=[