From 540e0f55953e5b5aec229623120559a9f32feba4 Mon Sep 17 00:00:00 2001 From: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:58:04 +0800 Subject: [PATCH 1/3] FIX: Include word-selection parameters in BinAsciiConverter identifiers Signed-off-by: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> --- pyrit/converter/bin_ascii_converter.py | 13 ++-- .../converter/test_bin_ascii_converter.py | 73 +++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/pyrit/converter/bin_ascii_converter.py b/pyrit/converter/bin_ascii_converter.py index 0c5084cc5b..027c261f0a 100644 --- a/pyrit/converter/bin_ascii_converter.py +++ b/pyrit/converter/bin_ascii_converter.py @@ -75,13 +75,12 @@ def _build_identifier(self) -> ComponentIdentifier: Returns: ComponentIdentifier: The identifier for this converter. """ - return self._create_identifier( - params={ - "word_selection_strategy": self._word_selection_strategy.__class__.__name__, - "word_split_separator": self._word_split_separator, - "encoding_func": self._encoding_func, - } - ) + params = dict(super()._build_identifier().params) + if not params["word_selection_strategy_params"]: + # Preserve existing identities for parameterless selection strategies. + del params["word_selection_strategy_params"] + params["encoding_func"] = self._encoding_func + return self._create_identifier(params=params) async def convert_word_async(self, word: str) -> str: """ diff --git a/tests/unit/converter/test_bin_ascii_converter.py b/tests/unit/converter/test_bin_ascii_converter.py index 50d0746c41..f76bccdf05 100644 --- a/tests/unit/converter/test_bin_ascii_converter.py +++ b/tests/unit/converter/test_bin_ascii_converter.py @@ -6,10 +6,14 @@ import pytest from pyrit.converter import ( + AllWordsSelectionStrategy, BinAsciiConverter, WordIndexSelectionStrategy, + WordKeywordSelectionStrategy, WordProportionSelectionStrategy, + WordSelectionStrategy, ) +from pyrit.registry import ConverterRegistry class TestBinAsciiConverterHex: @@ -221,3 +225,72 @@ async def test_none_separator_joins_with_encoded_space(self, encoding_func: str) space_result = await with_space.convert_async(prompt="alpha beta", input_type="text") assert none_result.output_text == space_result.output_text + + +@pytest.mark.parametrize( + ("first_strategy", "second_strategy"), + [ + (WordIndexSelectionStrategy(indices=[0]), WordIndexSelectionStrategy(indices=[1])), + (WordKeywordSelectionStrategy(keywords=["ab"]), WordKeywordSelectionStrategy(keywords=["cd"])), + ], +) +async def test_identifier_distinguishes_word_selection_async( + *, first_strategy: WordSelectionStrategy, second_strategy: WordSelectionStrategy +) -> None: + first = BinAsciiConverter(word_selection_strategy=first_strategy) + second = BinAsciiConverter(word_selection_strategy=second_strategy) + + first_result = await first.convert_async(prompt="ab cd", input_type="text") + second_result = await second.convert_async(prompt="ab cd", input_type="text") + + assert first_result.output_text == "6162 cd" + assert second_result.output_text == "ab 6364" + assert first.get_identifier().hash != second.get_identifier().hash + + +@pytest.mark.parametrize("encoding_func", ["hex", "quoted-printable", "UUencode"]) +def test_registry_accepts_distinct_word_selections(encoding_func: BinAsciiConverter.EncodingFunc) -> None: + first = BinAsciiConverter( + encoding_func=encoding_func, word_selection_strategy=WordIndexSelectionStrategy(indices=[0]) + ) + second = BinAsciiConverter( + encoding_func=encoding_func, word_selection_strategy=WordIndexSelectionStrategy(indices=[1]) + ) + instances = ConverterRegistry().instances + + instances.register(first) + instances.register(second) + + assert instances.get(first.get_identifier().unique_name) is first + assert instances.get(second.get_identifier().unique_name) is second + + +async def test_identifier_normalizes_equivalent_indices_async() -> None: + first = BinAsciiConverter(word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 1])) + second = BinAsciiConverter(word_selection_strategy=WordIndexSelectionStrategy(indices=[1, 0])) + + first_result = await first.convert_async(prompt="ab cd", input_type="text") + second_result = await second.convert_async(prompt="ab cd", input_type="text") + + assert first_result.output_text == second_result.output_text == "6162 6364" + assert first.get_identifier().hash == second.get_identifier().hash + + +@pytest.mark.parametrize("explicit_all_words", [False, True]) +@pytest.mark.parametrize( + ("encoding_func", "expected_hash"), + [ + ("hex", "997d88728fc485b88e30323fd1dc368292209d8ada8ed58b70e58aa8aadfe108"), + ("quoted-printable", "fc9df198b3e1199c620bccc6f86bbb73477641f85c76a62b7048cbf9e16aa170"), + ("UUencode", "20bc6dd2594880b93f9dab7c483ec5eee3fb8049993bb614922e59ef48a4b8e7"), + ], +) +def test_identifier_preserves_default_hash( + *, encoding_func: BinAsciiConverter.EncodingFunc, expected_hash: str, explicit_all_words: bool +) -> None: + converter = BinAsciiConverter( + encoding_func=encoding_func, + word_selection_strategy=AllWordsSelectionStrategy() if explicit_all_words else None, + ) + + assert converter.get_identifier().hash == expected_hash From 3f18baae0b77125da2926e8aaadf347adf71cede Mon Sep 17 00:00:00 2001 From: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:42:39 +0800 Subject: [PATCH 2/3] FIX: Preserve word-selection identity across converter overrides Signed-off-by: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> --- pyrit/converter/charswap_attack_converter.py | 7 ++ pyrit/converter/first_letter_converter.py | 12 +- pyrit/converter/leetspeak_converter.py | 8 +- .../unicode_replacement_converter.py | 12 +- pyrit/converter/zalgo_converter.py | 8 +- .../test_word_selection_identifiers.py | 110 ++++++++++++++++++ 6 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 tests/unit/converter/test_word_selection_identifiers.py diff --git a/pyrit/converter/charswap_attack_converter.py b/pyrit/converter/charswap_attack_converter.py index 70b6b82a4d..2270fef19f 100644 --- a/pyrit/converter/charswap_attack_converter.py +++ b/pyrit/converter/charswap_attack_converter.py @@ -58,8 +58,15 @@ def _build_identifier(self) -> ComponentIdentifier: Returns: ComponentIdentifier: The identifier for this converter. """ + selection_params = dict(super()._build_identifier().params) + # Preserve the legacy identity for the default 20% selection. + if type(self._word_selection_strategy) is WordProportionSelectionStrategy and selection_params[ + "word_selection_strategy_params" + ] == {"proportion": 0.2}: + selection_params = {} return self._create_identifier( params={ + **selection_params, "max_iterations": self._max_iterations, "seed": self._seed, }, diff --git a/pyrit/converter/first_letter_converter.py b/pyrit/converter/first_letter_converter.py index 0909db21ce..dd0e1d74ec 100644 --- a/pyrit/converter/first_letter_converter.py +++ b/pyrit/converter/first_letter_converter.py @@ -37,13 +37,11 @@ def _build_identifier(self) -> ComponentIdentifier: Returns: ComponentIdentifier: The identifier for this converter. """ - return self._create_identifier( - params={ - "word_selection_strategy": self._word_selection_strategy.__class__.__name__, - "word_split_separator": self._word_split_separator, - "letter_separator": self.letter_separator, - } - ) + params = dict(super()._build_identifier().params) + if not params["word_selection_strategy_params"]: + del params["word_selection_strategy_params"] + params["letter_separator"] = self.letter_separator + return self._create_identifier(params=params) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/converter/leetspeak_converter.py b/pyrit/converter/leetspeak_converter.py index 5880202f16..0076666129 100644 --- a/pyrit/converter/leetspeak_converter.py +++ b/pyrit/converter/leetspeak_converter.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from pyrit.converter.text_selection_strategy import WordSelectionStrategy +from pyrit.converter.text_selection_strategy import AllWordsSelectionStrategy, WordSelectionStrategy from pyrit.converter.word_level_converter import WordLevelConverter from pyrit.models import ComponentIdentifier @@ -65,8 +65,14 @@ def _build_identifier(self) -> ComponentIdentifier: substitutions_str = json.dumps(self._leet_substitutions, sort_keys=True) substitutions_hash = hashlib.sha256(substitutions_str.encode("utf-8")).hexdigest()[:16] + selection_params = ( + {} + if type(self._word_selection_strategy) is AllWordsSelectionStrategy + else super()._build_identifier().params + ) return self._create_identifier( params={ + **selection_params, "deterministic": self._deterministic, "custom_substitutions_hash": substitutions_hash, }, diff --git a/pyrit/converter/unicode_replacement_converter.py b/pyrit/converter/unicode_replacement_converter.py index 3741f45d6a..779c4ee308 100644 --- a/pyrit/converter/unicode_replacement_converter.py +++ b/pyrit/converter/unicode_replacement_converter.py @@ -36,13 +36,11 @@ def _build_identifier(self) -> ComponentIdentifier: Returns: ComponentIdentifier: The identifier for this converter. """ - return self._create_identifier( - params={ - "word_selection_strategy": self._word_selection_strategy.__class__.__name__, - "word_split_separator": self._word_split_separator, - "encode_spaces": self.encode_spaces, - } - ) + params = dict(super()._build_identifier().params) + if not params["word_selection_strategy_params"]: + del params["word_selection_strategy_params"] + params["encode_spaces"] = self.encode_spaces + return self._create_identifier(params=params) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/converter/zalgo_converter.py b/pyrit/converter/zalgo_converter.py index 96977c028b..1d18663722 100644 --- a/pyrit/converter/zalgo_converter.py +++ b/pyrit/converter/zalgo_converter.py @@ -4,7 +4,7 @@ import logging from typing import ClassVar -from pyrit.converter.text_selection_strategy import WordSelectionStrategy +from pyrit.converter.text_selection_strategy import AllWordsSelectionStrategy, WordSelectionStrategy from pyrit.converter.word_level_converter import WordLevelConverter from pyrit.models import ComponentIdentifier @@ -50,8 +50,14 @@ def _build_identifier(self) -> ComponentIdentifier: Returns: ComponentIdentifier: The identifier for this converter. """ + selection_params = ( + {} + if type(self._word_selection_strategy) is AllWordsSelectionStrategy + else super()._build_identifier().params + ) return self._create_identifier( params={ + **selection_params, "intensity": self._intensity, }, ) diff --git a/tests/unit/converter/test_word_selection_identifiers.py b/tests/unit/converter/test_word_selection_identifiers.py new file mode 100644 index 0000000000..c07cb5b9db --- /dev/null +++ b/tests/unit/converter/test_word_selection_identifiers.py @@ -0,0 +1,110 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest + +from pyrit.converter import ( + AllWordsSelectionStrategy, + CharSwapConverter, + FirstLetterConverter, + LeetspeakConverter, + UnicodeReplacementConverter, + WordIndexSelectionStrategy, + WordKeywordSelectionStrategy, + WordProportionSelectionStrategy, + WordSelectionStrategy, + ZalgoConverter, +) +from pyrit.converter.word_level_converter import WordLevelConverter +from pyrit.registry import ConverterRegistry + + +@pytest.fixture( + params=[CharSwapConverter, FirstLetterConverter, LeetspeakConverter, UnicodeReplacementConverter, ZalgoConverter], + ids=lambda converter: converter.__name__, +) +def converter_type(request: pytest.FixtureRequest) -> type[WordLevelConverter]: + return request.param + + +@pytest.mark.parametrize( + ("first_strategy", "second_strategy"), + [ + (WordIndexSelectionStrategy(indices=[0]), WordIndexSelectionStrategy(indices=[1])), + (WordKeywordSelectionStrategy(keywords=["alpha"]), WordKeywordSelectionStrategy(keywords=["beta"])), + ], + ids=["indices", "keywords"], +) +async def test_identifier_distinguishes_selected_words_async( + *, + converter_type: type[WordLevelConverter], + first_strategy: WordSelectionStrategy, + second_strategy: WordSelectionStrategy, +) -> None: + kwargs = {"max_iterations": 1} if converter_type is CharSwapConverter else {} + first = converter_type(word_selection_strategy=first_strategy, **kwargs) + second = converter_type(word_selection_strategy=second_strategy, **kwargs) + + first_result = await first.convert_async(prompt="alpha beta") + second_result = await second.convert_async(prompt="alpha beta") + + assert first_result.output_text.endswith(" beta") + assert second_result.output_text.startswith("alpha ") + assert first_result.output_text != second_result.output_text + assert first.get_identifier().hash != second.get_identifier().hash + + instances = ConverterRegistry().instances + instances.register(first) + instances.register(second) + assert instances.get(first.get_identifier().unique_name) is first + assert instances.get(second.get_identifier().unique_name) is second + + +@pytest.mark.parametrize( + ("first_strategy", "second_strategy"), + [ + (WordProportionSelectionStrategy(proportion=0.2), WordProportionSelectionStrategy(proportion=0.5)), + (WordProportionSelectionStrategy(proportion=0.2), WordProportionSelectionStrategy(proportion=0.2, seed=7)), + ( + WordKeywordSelectionStrategy(keywords=["alpha"], case_sensitive=True), + WordKeywordSelectionStrategy(keywords=["alpha"], case_sensitive=False), + ), + ], + ids=["proportion", "selection-seed", "case-sensitivity"], +) +def test_identifier_distinguishes_selection_settings( + *, + converter_type: type[WordLevelConverter], + first_strategy: WordSelectionStrategy, + second_strategy: WordSelectionStrategy, +) -> None: + first = converter_type(word_selection_strategy=first_strategy) + second = converter_type(word_selection_strategy=second_strategy) + + assert first.get_identifier().hash != second.get_identifier().hash + + +def test_identifier_normalizes_equivalent_indices(converter_type: type[WordLevelConverter]) -> None: + first = converter_type(word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 1])) + second = converter_type(word_selection_strategy=WordIndexSelectionStrategy(indices=[1, 0])) + + assert first.get_identifier().hash == second.get_identifier().hash + + +@pytest.mark.parametrize("explicit_default", [False, True]) +def test_identifier_preserves_default_hash(*, converter_type: type[WordLevelConverter], explicit_default: bool) -> None: + expected_hashes = { + CharSwapConverter: "826479a529db24708e0711f8d7fd6a3b79b82b1848e85e8b039933b0c2d19d4d", + FirstLetterConverter: "a0268c3263b42bad4bf082b273ac86180babd698cdd2bffcd5d9a3da26da0dae", + LeetspeakConverter: "ff4bc9afc79df2afbffd9d5bfe08f133339f8ec2f56e08d0fd3ee6f6af5b87cc", + UnicodeReplacementConverter: "173af73cd1eb54bcfaffb5dcc4eee0801325c36d73e4295c1e184b3fd1ec0e5c", + ZalgoConverter: "78d075f9172fd497bf741ddf148a6d43b4f4a9e31b59d21e692ab34bf1076f4e", + } + default_strategy = ( + WordProportionSelectionStrategy(proportion=0.2) + if converter_type is CharSwapConverter + else AllWordsSelectionStrategy() + ) + converter = converter_type(word_selection_strategy=default_strategy if explicit_default else None) + + assert converter.get_identifier().hash == expected_hashes[converter_type] From 01ca7ec0e3d8379763da803777e42a606cf866b7 Mon Sep 17 00:00:00 2001 From: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:01:22 +0800 Subject: [PATCH 3/3] Preserve word-index order in converter identifiers --- pyrit/converter/text_selection_strategy.py | 8 +-- .../converter/test_bin_ascii_converter.py | 4 +- .../converter/test_string_join_converter.py | 6 +-- .../converter/test_text_selection_strategy.py | 6 +-- .../test_word_selection_identifiers.py | 52 +++++++++++++++++-- 5 files changed, 61 insertions(+), 15 deletions(-) diff --git a/pyrit/converter/text_selection_strategy.py b/pyrit/converter/text_selection_strategy.py index 98129cfecf..6220dbeb93 100644 --- a/pyrit/converter/text_selection_strategy.py +++ b/pyrit/converter/text_selection_strategy.py @@ -551,7 +551,7 @@ def __init__(self, *, indices: list[int]) -> None: Initialize the word index selection strategy. Args: - indices (list[int]): The list of word indices to select. + indices (list[int]): The list of word indices in conversion order. Raises: ValueError: If an index is negative. @@ -562,12 +562,12 @@ def __init__(self, *, indices: list[int]) -> None: def get_identifier_params(self) -> dict[str, Any]: """ - Return the selected indices in sorted order. + Return the selected indices in conversion order. Returns: - dict[str, Any]: The sorted index list. + dict[str, Any]: The ordered index list. """ - return {"indices": sorted(self._indices)} + return {"indices": list(self._indices)} def select_words(self, *, words: list[str]) -> list[int]: """ diff --git a/tests/unit/converter/test_bin_ascii_converter.py b/tests/unit/converter/test_bin_ascii_converter.py index f76bccdf05..f2a2afae23 100644 --- a/tests/unit/converter/test_bin_ascii_converter.py +++ b/tests/unit/converter/test_bin_ascii_converter.py @@ -265,7 +265,7 @@ def test_registry_accepts_distinct_word_selections(encoding_func: BinAsciiConver assert instances.get(second.get_identifier().unique_name) is second -async def test_identifier_normalizes_equivalent_indices_async() -> None: +async def test_identifier_preserves_index_order_async() -> None: first = BinAsciiConverter(word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 1])) second = BinAsciiConverter(word_selection_strategy=WordIndexSelectionStrategy(indices=[1, 0])) @@ -273,7 +273,7 @@ async def test_identifier_normalizes_equivalent_indices_async() -> None: second_result = await second.convert_async(prompt="ab cd", input_type="text") assert first_result.output_text == second_result.output_text == "6162 6364" - assert first.get_identifier().hash == second.get_identifier().hash + assert first.get_identifier().hash != second.get_identifier().hash @pytest.mark.parametrize("explicit_all_words", [False, True]) diff --git a/tests/unit/converter/test_string_join_converter.py b/tests/unit/converter/test_string_join_converter.py index fd0245f98c..71e63d7548 100644 --- a/tests/unit/converter/test_string_join_converter.py +++ b/tests/unit/converter/test_string_join_converter.py @@ -78,7 +78,7 @@ def test_string_join_identifier_includes_selection_parameters() -> None: assert first.get_identifier().unique_name != second.get_identifier().unique_name -def test_string_join_identifier_normalizes_equivalent_index_sets() -> None: +def test_string_join_identifier_preserves_index_order() -> None: first = StringJoinConverter( word_selection_strategy=WordIndexSelectionStrategy(indices=[1, 0]), ) @@ -86,7 +86,7 @@ def test_string_join_identifier_normalizes_equivalent_index_sets() -> None: word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 1]), ) - assert first.get_identifier().hash == second.get_identifier().hash + assert first.get_identifier().hash != second.get_identifier().hash def test_string_join_registry_accepts_distinct_selection_configurations() -> None: @@ -113,7 +113,7 @@ def test_string_join_identifier_parameter_contents(*, indices: list[int], join_v "supported_input_types": ["text"], "supported_output_types": ["text"], "word_selection_strategy": "WordIndexSelectionStrategy", - "word_selection_strategy_params": {"indices": sorted(indices)}, + "word_selection_strategy_params": {"indices": indices}, "word_split_separator": " ", "join_value": join_value, } diff --git a/tests/unit/converter/test_text_selection_strategy.py b/tests/unit/converter/test_text_selection_strategy.py index b14fc810bd..8c713d3d8d 100644 --- a/tests/unit/converter/test_text_selection_strategy.py +++ b/tests/unit/converter/test_text_selection_strategy.py @@ -358,11 +358,11 @@ def test_select_range_converts_words_to_chars(self): # "quick brown" starts at index 4 and ends at index 15 assert result == (4, 15) - def test_identifier_params_are_order_independent(self): + def test_identifier_params_preserve_index_order(self): left = WordIndexSelectionStrategy(indices=[2, 0]) right = WordIndexSelectionStrategy(indices=[0, 2]) - assert left.get_identifier_params() == right.get_identifier_params() - assert left.get_identifier_params() == {"indices": [0, 2]} + assert left.get_identifier_params() == {"indices": [2, 0]} + assert right.get_identifier_params() == {"indices": [0, 2]} class TestWordKeywordSelectionStrategy: diff --git a/tests/unit/converter/test_word_selection_identifiers.py b/tests/unit/converter/test_word_selection_identifiers.py index c07cb5b9db..789dcf3a79 100644 --- a/tests/unit/converter/test_word_selection_identifiers.py +++ b/tests/unit/converter/test_word_selection_identifiers.py @@ -1,8 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from collections.abc import Callable, Iterator +from functools import partial + import pytest +from pyrit.common.random_context import configure_random_seed, get_configured_random_seed from pyrit.converter import ( AllWordsSelectionStrategy, CharSwapConverter, @@ -24,7 +28,20 @@ ids=lambda converter: converter.__name__, ) def converter_type(request: pytest.FixtureRequest) -> type[WordLevelConverter]: - return request.param + converter = request.param + assert isinstance(converter, type) + assert issubclass(converter, WordLevelConverter) + return converter + + +@pytest.fixture +def seeded_randomness() -> Iterator[None]: + previous_seed = get_configured_random_seed() + configure_random_seed(seed=123) + try: + yield + finally: + configure_random_seed(seed=previous_seed) @pytest.mark.parametrize( @@ -84,11 +101,40 @@ def test_identifier_distinguishes_selection_settings( assert first.get_identifier().hash != second.get_identifier().hash -def test_identifier_normalizes_equivalent_indices(converter_type: type[WordLevelConverter]) -> None: +def test_identifier_preserves_index_order(converter_type: type[WordLevelConverter]) -> None: first = converter_type(word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 1])) second = converter_type(word_selection_strategy=WordIndexSelectionStrategy(indices=[1, 0])) - assert first.get_identifier().hash == second.get_identifier().hash + assert first.get_identifier().hash != second.get_identifier().hash + + +@pytest.mark.usefixtures("seeded_randomness") +@pytest.mark.parametrize( + "converter_factory", + [ + pytest.param(partial(CharSwapConverter, max_iterations=1, seed=123), id="charswap"), + pytest.param(partial(ZalgoConverter, seed=123), id="zalgo"), + pytest.param(partial(LeetspeakConverter, deterministic=False), id="leetspeak"), + ], +) +async def test_reordered_indices_distinguish_stochastic_outputs_async( + converter_factory: Callable[..., WordLevelConverter], +) -> None: + first = converter_factory(word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 1])) + second = converter_factory(word_selection_strategy=WordIndexSelectionStrategy(indices=[1, 0])) + + first_result = await first.convert_async(prompt="alpha beta") + second_result = await second.convert_async(prompt="alpha beta") + assert (await first.convert_async(prompt="alpha beta")).output_text == first_result.output_text + assert (await second.convert_async(prompt="alpha beta")).output_text == second_result.output_text + assert first_result.output_text != second_result.output_text + assert first.get_identifier().hash != second.get_identifier().hash + + instances = ConverterRegistry().instances + instances.register(first) + instances.register(second) + assert instances.get(first.get_identifier().unique_name) is first + assert instances.get(second.get_identifier().unique_name) is second @pytest.mark.parametrize("explicit_default", [False, True])