From f942a7b94fcbad5806080fe01f076bf0469e723c Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 1 Sep 2026 15:55:24 -0700 Subject: [PATCH 1/3] FEAT: Add converter registry initializer Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 29476d2f-d5d9-4871-9edf-607a941b18f5 --- .pyrit_conf_example | 2 + doc/getting_started/pyrit_conf.md | 5 + pyrit/setup/initializers/__init__.py | 2 + pyrit/setup/initializers/converters.py | 72 ++++++++++ tests/unit/setup/test_configuration_loader.py | 2 +- .../unit/setup/test_converter_initializer.py | 133 ++++++++++++++++++ 6 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 pyrit/setup/initializers/converters.py create mode 100644 tests/unit/setup/test_converter_initializer.py diff --git a/.pyrit_conf_example b/.pyrit_conf_example index 978c8a293f..e148fda961 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -23,6 +23,7 @@ memory_db_type: sqlite # # Available initializers: # - target: Registers available prompt targets into the TargetRegistry +# - converter: Registers converters that use defaults or the adversarial_chat target # - scorer: Registers pre-configured scorers into the ScorerRegistry # - technique: Registers attack techniques into the AttackTechniqueRegistry # - load_default_datasets: Optionally preloads all registered datasets into memory @@ -49,6 +50,7 @@ initializers: tags: - default - scorer + - name: converter - name: scorer - name: technique # Optional full preload/cache warming for offline or shared environments. diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index fb498a9bec..925fe1686c 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -103,9 +103,12 @@ Most users should enable the following initializers. These are what the `.pyrit_ | Initializer | What It Registers | When You Need It | | --- | --- | --- | | `target` | Prompt targets (OpenAI, Azure, AML, etc.) into the `TargetRegistry` | Recommended for `pyrit_scan` and registry-based workflows | +| `converter` | Converters that use constructor defaults or only the registered `adversarial_chat` target | Recommended when attaching registered converters to techniques | | `scorer` | Scorers (refusal, content safety, harm-category, Likert, etc.) into the `ScorerRegistry` | Recommended for automated scoring and `pyrit_scan` evaluations | | `technique` | Attack techniques into the `AttackTechniqueRegistry` | Recommended for scenarios that select registered techniques | +The `converter` initializer removes the `Converter` suffix and uses snake case for instance names. For example, it registers `Base64Converter` as `base64`. + ```{note} **Execution order follows listing order.** Initializers execute in the order they appear in the config. Ensure dependencies are satisfied — for example, list `target` before `scorer` since scorers need targets to be registered first. ``` @@ -123,6 +126,7 @@ initializers: tags: - default - scorer + - name: converter - name: scorer - name: technique ``` @@ -404,6 +408,7 @@ initializers: tags: - default - scorer + - name: converter - name: scorer - name: technique # Optional full preload/cache warming; scenarios fetch requested datasets on demand. diff --git a/pyrit/setup/initializers/__init__.py b/pyrit/setup/initializers/__init__.py index ceb5c65bd8..e8ab3be972 100644 --- a/pyrit/setup/initializers/__init__.py +++ b/pyrit/setup/initializers/__init__.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from pyrit.models.parameter import Parameter + from pyrit.setup.initializers.converters import ConverterInitializer from pyrit.setup.initializers.load_default_datasets import LoadDefaultDatasets from pyrit.setup.initializers.preload_scenario_metadata import PreloadScenarioMetadata from pyrit.setup.initializers.refresh_datasets import RefreshDatasets @@ -21,6 +22,7 @@ _LAZY_EXPORTS: dict[str, str | tuple[str, str | None]] = { "Parameter": "pyrit.models.parameter", "PyRITInitializer": "pyrit.setup.pyrit_initializer", + "ConverterInitializer": "pyrit.setup.initializers.converters", "TechniqueInitializer": "pyrit.setup.initializers.techniques", "ScorerInitializer": "pyrit.setup.initializers.scorers", "TargetInitializer": "pyrit.setup.initializers.targets", diff --git a/pyrit/setup/initializers/converters.py b/pyrit/setup/initializers/converters.py new file mode 100644 index 0000000000..38c3c820fe --- /dev/null +++ b/pyrit/setup/initializers/converters.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Initializer for registering converters that need no user-supplied configuration.""" + +import logging + +from pyrit.models.identifiers.class_name_utils import class_name_to_snake_case +from pyrit.models.parameter import ComponentType +from pyrit.registry import ConverterRegistry, TargetRegistry +from pyrit.registry.components import ConverterMetadata +from pyrit.setup.pyrit_initializer import PyRITInitializer + +logger = logging.getLogger(__name__) + + +class ConverterInitializer(PyRITInitializer): + """ + Register converters that can be created from defaults or the adversarial chat target. + + A converter is eligible when it has no required constructor parameters. A converter + whose required parameters are all target references is also eligible when the + ``adversarial_chat`` target is registered. + """ + + _ADVERSARIAL_CHAT_TARGET = "adversarial_chat" + + async def initialize_async(self) -> None: + """Create and register all eligible converters.""" + converter_registry = ConverterRegistry.get_registry_singleton() + target_registry = TargetRegistry.get_registry_singleton() + adversarial_chat_available = self._ADVERSARIAL_CHAT_TARGET in target_registry.instances + + for metadata in converter_registry.get_all_registered_class_metadata(): + creation_kwargs = self._get_creation_kwargs( + metadata=metadata, + adversarial_chat_available=adversarial_chat_available, + ) + if creation_kwargs is None: + continue + + instance_name = class_name_to_snake_case(metadata.class_name, suffix="Converter") + try: + converter = converter_registry.create_instance(metadata.registry_name, **creation_kwargs) + converter_registry.instances.register(converter, name=instance_name) + logger.info("Registered converter: %s", instance_name) + except (KeyError, TypeError, ValueError) as ex: + logger.warning("Skipping converter '%s': %s", instance_name, ex) + + def _get_creation_kwargs( + self, + *, + metadata: ConverterMetadata, + adversarial_chat_available: bool, + ) -> dict[str, object] | None: + """ + Get constructor arguments for an eligible converter. + + Returns: + dict[str, object] | None: Constructor arguments, or None when the + converter needs configuration that this initializer cannot supply. + """ + required_parameters = [parameter for parameter in metadata.parameters if parameter.required] + if not required_parameters: + return {} + + if not adversarial_chat_available or not all( + parameter.is_reference_to(ComponentType.TARGET) for parameter in required_parameters + ): + return None + + return {parameter.name: self._ADVERSARIAL_CHAT_TARGET for parameter in required_parameters} diff --git a/tests/unit/setup/test_configuration_loader.py b/tests/unit/setup/test_configuration_loader.py index 87aa79709d..04b0b3f39d 100644 --- a/tests/unit/setup/test_configuration_loader.py +++ b/tests/unit/setup/test_configuration_loader.py @@ -272,7 +272,7 @@ def test_example_uses_on_demand_dataset_fetching(self) -> None: initializer_names = [initializer.name for initializer in config._initializer_configs] example_text = example_path.read_text(encoding="utf-8") - assert initializer_names == ["target", "scorer", "technique"] + assert initializer_names == ["target", "converter", "scorer", "technique"] assert "# - name: load_default_datasets" in example_text assert "several minutes" in example_text assert "provider credentials" in example_text diff --git a/tests/unit/setup/test_converter_initializer.py b/tests/unit/setup/test_converter_initializer.py new file mode 100644 index 0000000000..a561eac54c --- /dev/null +++ b/tests/unit/setup/test_converter_initializer.py @@ -0,0 +1,133 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Tests for the converter initializer.""" + +import logging +from collections.abc import Iterator +from unittest.mock import patch + +import pytest + +from pyrit.converter import Base64Converter, ROT13Converter, VariationConverter +from pyrit.models.parameter import ComponentType +from pyrit.registry import ConverterRegistry, InitializerRegistry, TargetRegistry +from pyrit.registry.components import ConverterMetadata +from pyrit.setup.initializers import ConverterInitializer +from tests.unit.mocks import MockPromptTarget + + +@pytest.fixture(autouse=True) +def reset_registries() -> Iterator[None]: + """Reset the component registries around each test.""" + ConverterRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + yield + ConverterRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + + +def _get_metadata(*, registry: ConverterRegistry, name: str) -> ConverterMetadata: + metadata = registry.get_registered_class_metadata(name) + assert metadata is not None + return metadata + + +async def test_initialize_registers_converter_with_default_arguments() -> None: + registry = ConverterRegistry.get_registry_singleton() + metadata = _get_metadata(registry=registry, name="Base64Converter") + + with patch.object(registry, "get_all_registered_class_metadata", return_value=[metadata]): + await ConverterInitializer().initialize_async() + + assert isinstance(registry.instances.get("base64"), Base64Converter) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_initialize_registers_target_only_converter_with_adversarial_chat() -> None: + converter_registry = ConverterRegistry.get_registry_singleton() + target_registry = TargetRegistry.get_registry_singleton() + adversarial_chat = MockPromptTarget() + target_registry.instances.register(adversarial_chat, name="adversarial_chat") + metadata = _get_metadata(registry=converter_registry, name="VariationConverter") + + with patch.object(converter_registry, "get_all_registered_class_metadata", return_value=[metadata]): + await ConverterInitializer().initialize_async() + + converter = converter_registry.instances.get("variation") + assert isinstance(converter, VariationConverter) + assert converter._converter_target is adversarial_chat + + +async def test_initialize_skips_target_only_converter_without_adversarial_chat() -> None: + registry = ConverterRegistry.get_registry_singleton() + metadata = _get_metadata(registry=registry, name="VariationConverter") + + with patch.object(registry, "get_all_registered_class_metadata", return_value=[metadata]): + await ConverterInitializer().initialize_async() + + assert "variation" not in registry.instances + + +@pytest.mark.usefixtures("patch_central_database") +async def test_initialize_skips_converter_with_additional_required_argument() -> None: + converter_registry = ConverterRegistry.get_registry_singleton() + TargetRegistry.get_registry_singleton().instances.register(MockPromptTarget(), name="adversarial_chat") + metadata = _get_metadata(registry=converter_registry, name="TenseConverter") + + with patch.object(converter_registry, "get_all_registered_class_metadata", return_value=[metadata]): + await ConverterInitializer().initialize_async() + + assert "tense" not in converter_registry.instances + + +@pytest.mark.usefixtures("patch_central_database") +async def test_initialize_attempts_every_eligible_catalog_converter() -> None: + converter_registry = ConverterRegistry.get_registry_singleton() + TargetRegistry.get_registry_singleton().instances.register(MockPromptTarget(), name="adversarial_chat") + metadata_items = converter_registry.get_all_registered_class_metadata() + expected_classes = { + metadata.registry_name + for metadata in metadata_items + if not (required := [parameter for parameter in metadata.parameters if parameter.required]) + or all(parameter.is_reference_to(ComponentType.TARGET) for parameter in required) + } + + with patch.object(converter_registry, "create_instance", return_value=Base64Converter()) as create_instance: + await ConverterInitializer().initialize_async() + + calls_by_class = {call.args[0]: call.kwargs for call in create_instance.call_args_list} + assert set(calls_by_class) == expected_classes + assert calls_by_class["Base64Converter"] == {} + assert calls_by_class["VariationConverter"] == {"converter_target": "adversarial_chat"} + assert "TenseConverter" not in calls_by_class + + +async def test_initialize_continues_after_expected_construction_error( + caplog: pytest.LogCaptureFixture, +) -> None: + registry = ConverterRegistry.get_registry_singleton() + metadata_items = [ + _get_metadata(registry=registry, name="Base64Converter"), + _get_metadata(registry=registry, name="ROT13Converter"), + ] + + with ( + patch.object(registry, "get_all_registered_class_metadata", return_value=metadata_items), + patch.object( + registry, + "create_instance", + side_effect=[ValueError("missing configuration"), ROT13Converter()], + ), + caplog.at_level(logging.WARNING, logger="pyrit.setup.initializers.converters"), + ): + await ConverterInitializer().initialize_async() + + assert "base64" not in registry.instances + assert isinstance(registry.instances.get("rot13"), ROT13Converter) + assert "Skipping converter 'base64': missing configuration" in caplog.text + + +def test_initializer_is_discovered() -> None: + registry = InitializerRegistry() + assert "converter" in registry.get_class_names() From 9027f38f6d12a22fc0e877875f41fd9be92f965c Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Wed, 2 Sep 2026 15:34:39 -0700 Subject: [PATCH 2/3] FEAT: Add curated converter presets Define an intentional catalog of converter presets with explicit target dependencies and parameter values. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f6b8dd2-e89a-43b2-9781-68d6acb6d76c --- .pyrit_conf_example | 2 +- MANIFEST.in | 1 + doc/getting_started/pyrit_conf.md | 11 +- .../local/examples/benign_cake_question.jpg | Bin 0 -> 3096 bytes pyrit/setup/initializers/converters.py | 205 ++++++++++++++---- .../unit/setup/test_converter_initializer.py | 159 ++++++++++---- 6 files changed, 285 insertions(+), 93 deletions(-) create mode 100644 pyrit/datasets/seed_datasets/local/examples/benign_cake_question.jpg diff --git a/.pyrit_conf_example b/.pyrit_conf_example index e148fda961..189eb8bdcd 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -23,7 +23,7 @@ memory_db_type: sqlite # # Available initializers: # - target: Registers available prompt targets into the TargetRegistry -# - converter: Registers converters that use defaults or the adversarial_chat target +# - converter: Registers the curated core converter presets # - scorer: Registers pre-configured scorers into the ScorerRegistry # - technique: Registers attack techniques into the AttackTechniqueRegistry # - load_default_datasets: Optionally preloads all registered datasets into memory diff --git a/MANIFEST.in b/MANIFEST.in index 9ceb63cd55..6995774068 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ recursive-include pyrit *.prompt recursive-include pyrit *.yaml recursive-include pyrit *.pdf recursive-include pyrit *.png +recursive-include pyrit *.jpg recursive-include pyrit *.wav recursive-include pyrit *.mp4 recursive-include pyrit *.md diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index 925fe1686c..41706d8ba9 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -103,11 +103,18 @@ Most users should enable the following initializers. These are what the `.pyrit_ | Initializer | What It Registers | When You Need It | | --- | --- | --- | | `target` | Prompt targets (OpenAI, Azure, AML, etc.) into the `TargetRegistry` | Recommended for `pyrit_scan` and registry-based workflows | -| `converter` | Converters that use constructor defaults or only the registered `adversarial_chat` target | Recommended when attaching registered converters to techniques | +| `converter` | Curated core converter presets | Recommended when attaching registered converters to techniques | | `scorer` | Scorers (refusal, content safety, harm-category, Likert, etc.) into the `ScorerRegistry` | Recommended for automated scoring and `pyrit_scan` evaluations | | `technique` | Attack techniques into the `AttackTechniqueRegistry` | Recommended for scenarios that select registered techniques | -The `converter` initializer removes the `Converter` suffix and uses snake case for instance names. For example, it registers `Base64Converter` as `base64`. +The `converter` initializer registers a curated set of text, LLM, audio, and image converter presets. The +LLM-backed presets use the registered `adversarial_chat` target and are skipped when that target is not available. +The Azure Speech preset uses the standard Azure Speech environment variables and is skipped when they are not set. +Use `pyrit_scan list-converters` to list the presets that were registered. + +Parameterized presets include past and future tense, professional and sarcastic tone, Spanish translation, +whitespace replacement with underscores, the `jailbreak_1.yaml` text jailbreak template, the packaged blank +canvas for text-to-image conversion, and the packaged benign cake image for the transparency attack. ```{note} **Execution order follows listing order.** Initializers execute in the order they appear in the config. Ensure dependencies are satisfied — for example, list `target` before `scorer` since scorers need targets to be registered first. diff --git a/pyrit/datasets/seed_datasets/local/examples/benign_cake_question.jpg b/pyrit/datasets/seed_datasets/local/examples/benign_cake_question.jpg new file mode 100644 index 0000000000000000000000000000000000000000..11652dcd273b9d8384c4b5704e39d4b132250ebe GIT binary patch literal 3096 zcmdUuXH-*L7RS%!l7uE5k&XrwLD5K;E=?W}5Cu^b5J8Y$Botqy7j;1(DDcP@YZ^7KFo)i`82cd+W))nJ^S4I|E+Tl>oaQ{2>xVl zX$~L=0@v6VU`+r|vj}fb0IaQnG5~-Ja6*y*!LA^78bDHi*5;5BfcM^Wum>@Ktw4~S zHtgJuYwX;evv)t|<{c2KYGYR6fkf^l zikl0G;zl8nT)bS|JiEcJ_;`8vb{}>>^2byT1Pmj1P)O8X%YP)SMj*rmvOpFL2?Gux z2o{1^?|~fKJ7Uk?F2Y_w955#WiDKK}WjBxn+4f~cW> z{sW7x|B_3H&Bd|12B=*wh$Ebxun;HWAO6rPh}81a`x6dr9A9x8RYPLE02w6!iU9= z;C6H5r((u9`8hMpSqwYh4~+>&xLo=m~l<_nxJF-&$j+>djg;B^(AbHasHI0xKF(mcX*ASQCPB`PUK zQ#GKpq;#sTt`>j5{d}(#8ewnOpAo@Pm)7bkMKTmDYnZU*k1#7MIll1a{@l8UE+NRi zJ>{B`{u@GQu)+3YcafVu9`wnS@1sYby-vsYO#MDWFfmc>B=3Y;LQU233)u}54>d;I7(b1nxGkyX3X+B_WZaIy63)>%vv-6#Bm5eZ6d5uL}ta#9IBy4iY zLr`1)lz_`Y)GA-{k26@^o{|S7^-J z;9xg);$ZjcTiq@dS+7CW^VT`Dxlbpx4Ks18^gLQR)BCAU@0HJ`0|V2|6+=Unf;_RR zGn0MFn4JM|2kjD0Ye<#sO6;@henua8>-#3L;-f}Y_H}LlcoEX&A!$bESy<-wIaHye z3jGx=R8BZ_v)EsEA^YN~gg1$v!Y6ddD*XB5JI-I>7rTcG?$T4Fr@8bP9P@EG-5=oR zQCXKUQJ5%0H2&KUzPOms`46v5@&!&;GxbYb8TdfiuHfsTgLRGmpDdQcs*rl_o}C{qPPMxZZ^imRN%ilMK znX}bbBd(qXjm&}xXOosuL_NJFQynwx9O-gp7{jI6h7U11QuuhpH!4Pl1rqB;`*A|! zz9;zRDoqrdUq9&hB}|cjRw9WTte{DuqRv{@CVNFCOw8rsgbM{YJo&6crtu! zqn>zTr$>V1@!0l6J2`OQW?g(nf}ns&pT;+PXlh5DDKmLOv@C}vK{~GKe4W}O-;d78 zEqMLehSE7aoh=m$KTdEgIfp{ivm2d~%abucM3s|X7_=fi#PB+GuokTy;8+-imi&o1Fw%hzpvrlXEAMOy2RFt&3C z%F>C?(xx~!6(}uA?olC~V)0nAhnGCGXe5(H?z&6VdY>w5edTP@O*IA z-MGtX#?UJdj;L(fFIm)ZKlD`v>3vTXy3=ALW^gklc&HTkZ5cD0CmNC|1aB=(igNJUDmwBN(}S6I4#JylS?D< z>6SDz(d`-Yr7a62cxvyA@f4*5jhfprf=UyQt;099N8<5y-H5RyS8??*PNtkGy}LI6 z%R4{oiD?RG7&B?zB615SL04-vB%S5Ga_7XDTtBxf&(U~eo^`_e ZD0{yU)bHw$#y@SqzqaE4V+FAW{{i{*5a9p- literal 0 HcmV?d00001 diff --git a/pyrit/setup/initializers/converters.py b/pyrit/setup/initializers/converters.py index 38c3c820fe..93034295cd 100644 --- a/pyrit/setup/initializers/converters.py +++ b/pyrit/setup/initializers/converters.py @@ -1,72 +1,187 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Initializer for registering converters that need no user-supplied configuration.""" +"""Initializer for registering the core converter presets.""" +import asyncio import logging +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar -from pyrit.models.identifiers.class_name_utils import class_name_to_snake_case -from pyrit.models.parameter import ComponentType -from pyrit.registry import ConverterRegistry, TargetRegistry -from pyrit.registry.components import ConverterMetadata +from pyrit.common.path import DATASETS_PATH +from pyrit.registry import ConverterRegistry from pyrit.setup.pyrit_initializer import PyRITInitializer +if TYPE_CHECKING: + from pyrit.converter import Converter + logger = logging.getLogger(__name__) +def _get_text_jailbreak_args() -> dict[str, Any]: + """ + Build the constructor arguments for the text jailbreak preset. + + Returns: + dict[str, Any]: Constructor arguments containing the jailbreak template. + """ + from pyrit.datasets import TextJailBreak + + return {"jailbreak_template": TextJailBreak(template_file_name="jailbreak_1.yaml")} + + +@dataclass(frozen=True) +class ConverterConfig: + """Configuration for a converter preset.""" + + registry_name: str + converter_type: str + constructor_args: dict[str, Any] = field(default_factory=dict) + constructor_args_factory: Callable[[], dict[str, Any]] | None = None + + def get_constructor_args(self) -> dict[str, Any]: + """ + Get a new constructor argument dictionary for this preset. + + Returns: + dict[str, Any]: A copy of the configured arguments with deferred arguments included. + """ + args = dict(self.constructor_args) + if self.constructor_args_factory: + args.update(self.constructor_args_factory()) + return args + + class ConverterInitializer(PyRITInitializer): """ - Register converters that can be created from defaults or the adversarial chat target. + Register the curated core converter presets into the ConverterRegistry. - A converter is eligible when it has no required constructor parameters. A converter - whose required parameters are all target references is also eligible when the - ``adversarial_chat`` target is registered. + Each preset explicitly declares its constructor arguments and target references. + Custom initializers can use the same pattern to register additional or parameterized + converter presets. """ - _ADVERSARIAL_CHAT_TARGET = "adversarial_chat" + CONFIGS: ClassVar[tuple[ConverterConfig, ...]] = ( + ConverterConfig(registry_name="base64", converter_type="Base64Converter"), + ConverterConfig(registry_name="binary", converter_type="BinaryConverter"), + ConverterConfig(registry_name="char_swap", converter_type="CharSwapConverter"), + ConverterConfig(registry_name="ecoji", converter_type="EcojiConverter"), + ConverterConfig(registry_name="insert_punctuation", converter_type="InsertPunctuationConverter"), + ConverterConfig(registry_name="leetspeak", converter_type="LeetspeakConverter"), + ConverterConfig(registry_name="rot13", converter_type="ROT13Converter"), + ConverterConfig( + registry_name="search_replace", + converter_type="SearchReplaceConverter", + constructor_args={"pattern": r"\s+", "replace": "_"}, + ), + ConverterConfig(registry_name="string_join", converter_type="StringJoinConverter"), + ConverterConfig( + registry_name="text_jailbreak", + converter_type="TextJailbreakConverter", + constructor_args_factory=_get_text_jailbreak_args, + ), + ConverterConfig(registry_name="zalgo", converter_type="ZalgoConverter"), + ConverterConfig( + registry_name="malicious_question_generator", + converter_type="MaliciousQuestionGeneratorConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="math_prompt", + converter_type="MathPromptConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="noise", + converter_type="NoiseConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="tense_future", + converter_type="TenseConverter", + constructor_args={"converter_target": "adversarial_chat", "tense": "future"}, + ), + ConverterConfig( + registry_name="tense_past", + converter_type="TenseConverter", + constructor_args={"converter_target": "adversarial_chat", "tense": "past"}, + ), + ConverterConfig( + registry_name="tone_professional", + converter_type="ToneConverter", + constructor_args={"converter_target": "adversarial_chat", "tone": "professional"}, + ), + ConverterConfig( + registry_name="tone_sarcastic", + converter_type="ToneConverter", + constructor_args={"converter_target": "adversarial_chat", "tone": "sarcastic"}, + ), + ConverterConfig( + registry_name="translation_spanish", + converter_type="TranslationConverter", + constructor_args={"converter_target": "adversarial_chat", "language": "Spanish"}, + ), + ConverterConfig( + registry_name="variation", + converter_type="VariationConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="add_image_text", + converter_type="AddImageTextConverter", + constructor_args={ + "img_to_add": str(DATASETS_PATH / "seed_datasets" / "local" / "examples" / "blank_canvas.png") + }, + ), + ConverterConfig( + registry_name="add_text_image", + converter_type="AddTextImageConverter", + constructor_args={"text_to_add": "PyRIT"}, + ), + ConverterConfig( + registry_name="azure_speech_audio_to_text", + converter_type="AzureSpeechAudioToTextConverter", + ), + ConverterConfig(registry_name="image_color_saturation", converter_type="ImageColorSaturationConverter"), + ConverterConfig(registry_name="image_compression", converter_type="ImageCompressionConverter"), + ConverterConfig(registry_name="image_rotation", converter_type="ImageRotationConverter"), + ConverterConfig(registry_name="qr_code", converter_type="QRCodeConverter"), + ConverterConfig( + registry_name="transparency_attack", + converter_type="TransparencyAttackConverter", + constructor_args={ + "benign_image_path": DATASETS_PATH / "seed_datasets" / "local" / "examples" / "benign_cake_question.jpg" + }, + ), + ) async def initialize_async(self) -> None: - """Create and register all eligible converters.""" + """Create and register the core converter presets.""" converter_registry = ConverterRegistry.get_registry_singleton() - target_registry = TargetRegistry.get_registry_singleton() - adversarial_chat_available = self._ADVERSARIAL_CHAT_TARGET in target_registry.instances - - for metadata in converter_registry.get_all_registered_class_metadata(): - creation_kwargs = self._get_creation_kwargs( - metadata=metadata, - adversarial_chat_available=adversarial_chat_available, - ) - if creation_kwargs is None: - continue - - instance_name = class_name_to_snake_case(metadata.class_name, suffix="Converter") + + for config in self.CONFIGS: try: - converter = converter_registry.create_instance(metadata.registry_name, **creation_kwargs) - converter_registry.instances.register(converter, name=instance_name) - logger.info("Registered converter: %s", instance_name) - except (KeyError, TypeError, ValueError) as ex: - logger.warning("Skipping converter '%s': %s", instance_name, ex) + converter = await asyncio.to_thread( + self._create_converter, + converter_registry=converter_registry, + config=config, + ) + converter_registry.instances.register(converter, name=config.registry_name) + logger.info("Registered converter: %s", config.registry_name) + except (FileNotFoundError, KeyError, TypeError, ValueError) as ex: + logger.warning("Skipping converter '%s': %s", config.registry_name, ex) - def _get_creation_kwargs( + def _create_converter( self, *, - metadata: ConverterMetadata, - adversarial_chat_available: bool, - ) -> dict[str, object] | None: + converter_registry: ConverterRegistry, + config: ConverterConfig, + ) -> "Converter": """ - Get constructor arguments for an eligible converter. + Create one configured converter outside the event-loop thread. Returns: - dict[str, object] | None: Constructor arguments, or None when the - converter needs configuration that this initializer cannot supply. + Converter: The configured converter instance. """ - required_parameters = [parameter for parameter in metadata.parameters if parameter.required] - if not required_parameters: - return {} - - if not adversarial_chat_available or not all( - parameter.is_reference_to(ComponentType.TARGET) for parameter in required_parameters - ): - return None - - return {parameter.name: self._ADVERSARIAL_CHAT_TARGET for parameter in required_parameters} + return converter_registry.create_instance(config.converter_type, **config.get_constructor_args()) diff --git a/tests/unit/setup/test_converter_initializer.py b/tests/unit/setup/test_converter_initializer.py index a561eac54c..b5ce7f3b49 100644 --- a/tests/unit/setup/test_converter_initializer.py +++ b/tests/unit/setup/test_converter_initializer.py @@ -5,15 +5,14 @@ import logging from collections.abc import Iterator -from unittest.mock import patch +from unittest.mock import call, patch import pytest -from pyrit.converter import Base64Converter, ROT13Converter, VariationConverter -from pyrit.models.parameter import ComponentType +from pyrit.converter import Base64Converter, LeetspeakConverter, ROT13Converter, VariationConverter from pyrit.registry import ConverterRegistry, InitializerRegistry, TargetRegistry -from pyrit.registry.components import ConverterMetadata from pyrit.setup.initializers import ConverterInitializer +from pyrit.setup.initializers.converters import ConverterConfig from tests.unit.mocks import MockPromptTarget @@ -27,31 +26,34 @@ def reset_registries() -> Iterator[None]: TargetRegistry.reset_registry_singleton() -def _get_metadata(*, registry: ConverterRegistry, name: str) -> ConverterMetadata: - metadata = registry.get_registered_class_metadata(name) - assert metadata is not None - return metadata +def _get_configs(*names: str) -> tuple[ConverterConfig, ...]: + configs_by_name = {config.registry_name: config for config in ConverterInitializer.CONFIGS} + return tuple(configs_by_name[name] for name in names) -async def test_initialize_registers_converter_with_default_arguments() -> None: +async def test_initialize_registers_core_converter_presets() -> None: registry = ConverterRegistry.get_registry_singleton() - metadata = _get_metadata(registry=registry, name="Base64Converter") - with patch.object(registry, "get_all_registered_class_metadata", return_value=[metadata]): + with patch.object( + ConverterInitializer, + "CONFIGS", + _get_configs("base64", "leetspeak", "rot13"), + ): await ConverterInitializer().initialize_async() assert isinstance(registry.instances.get("base64"), Base64Converter) + assert isinstance(registry.instances.get("leetspeak"), LeetspeakConverter) + assert isinstance(registry.instances.get("rot13"), ROT13Converter) @pytest.mark.usefixtures("patch_central_database") -async def test_initialize_registers_target_only_converter_with_adversarial_chat() -> None: +async def test_initialize_registers_variation_with_declared_target() -> None: converter_registry = ConverterRegistry.get_registry_singleton() target_registry = TargetRegistry.get_registry_singleton() adversarial_chat = MockPromptTarget() target_registry.instances.register(adversarial_chat, name="adversarial_chat") - metadata = _get_metadata(registry=converter_registry, name="VariationConverter") - with patch.object(converter_registry, "get_all_registered_class_metadata", return_value=[metadata]): + with patch.object(ConverterInitializer, "CONFIGS", _get_configs("variation")): await ConverterInitializer().initialize_async() converter = converter_registry.instances.get("variation") @@ -59,71 +61,138 @@ async def test_initialize_registers_target_only_converter_with_adversarial_chat( assert converter._converter_target is adversarial_chat -async def test_initialize_skips_target_only_converter_without_adversarial_chat() -> None: +async def test_initialize_skips_converter_without_declared_target() -> None: registry = ConverterRegistry.get_registry_singleton() - metadata = _get_metadata(registry=registry, name="VariationConverter") - with patch.object(registry, "get_all_registered_class_metadata", return_value=[metadata]): + with patch.object(ConverterInitializer, "CONFIGS", _get_configs("variation")): await ConverterInitializer().initialize_async() assert "variation" not in registry.instances -@pytest.mark.usefixtures("patch_central_database") -async def test_initialize_skips_converter_with_additional_required_argument() -> None: +async def test_initialize_uses_explicit_converter_configs() -> None: converter_registry = ConverterRegistry.get_registry_singleton() - TargetRegistry.get_registry_singleton().instances.register(MockPromptTarget(), name="adversarial_chat") - metadata = _get_metadata(registry=converter_registry, name="TenseConverter") - with patch.object(converter_registry, "get_all_registered_class_metadata", return_value=[metadata]): + with patch.object(converter_registry, "create_instance", return_value=Base64Converter()) as create_instance: await ConverterInitializer().initialize_async() - assert "tense" not in converter_registry.instances + assert converter_registry.instances.get_names() == sorted( + { + "add_image_text", + "add_text_image", + "azure_speech_audio_to_text", + "base64", + "binary", + "char_swap", + "ecoji", + "image_color_saturation", + "image_compression", + "image_rotation", + "insert_punctuation", + "leetspeak", + "malicious_question_generator", + "math_prompt", + "noise", + "qr_code", + "rot13", + "search_replace", + "string_join", + "tense_future", + "tense_past", + "text_jailbreak", + "tone_professional", + "tone_sarcastic", + "translation_spanish", + "transparency_attack", + "variation", + "zalgo", + } + ) + assert create_instance.call_count == len(ConverterInitializer.CONFIGS) + assert call("Base64Converter") in create_instance.call_args_list + assert ( + call( + "SearchReplaceConverter", + pattern=r"\s+", + replace="_", + ) + in create_instance.call_args_list + ) + assert ( + call( + "TenseConverter", + converter_target="adversarial_chat", + tense="future", + ) + in create_instance.call_args_list + ) + assert ( + call( + "ToneConverter", + converter_target="adversarial_chat", + tone="sarcastic", + ) + in create_instance.call_args_list + ) + assert ( + call( + "TranslationConverter", + converter_target="adversarial_chat", + language="Spanish", + ) + in create_instance.call_args_list + ) + assert ( + call( + "VariationConverter", + converter_target="adversarial_chat", + ) + in create_instance.call_args_list + ) @pytest.mark.usefixtures("patch_central_database") -async def test_initialize_attempts_every_eligible_catalog_converter() -> None: +async def test_explicit_converter_configs_can_be_constructed() -> None: converter_registry = ConverterRegistry.get_registry_singleton() - TargetRegistry.get_registry_singleton().instances.register(MockPromptTarget(), name="adversarial_chat") - metadata_items = converter_registry.get_all_registered_class_metadata() - expected_classes = { - metadata.registry_name - for metadata in metadata_items - if not (required := [parameter for parameter in metadata.parameters if parameter.required]) - or all(parameter.is_reference_to(ComponentType.TARGET) for parameter in required) - } + target_registry = TargetRegistry.get_registry_singleton() + target_registry.instances.register(MockPromptTarget(), name="adversarial_chat") - with patch.object(converter_registry, "create_instance", return_value=Base64Converter()) as create_instance: - await ConverterInitializer().initialize_async() + await ConverterInitializer().initialize_async() + + expected_names = {config.registry_name for config in ConverterInitializer.CONFIGS} + expected_names.remove("azure_speech_audio_to_text") + assert expected_names <= set(converter_registry.instances.get_names()) + + +def test_configs_have_unique_names() -> None: + names = [config.registry_name for config in ConverterInitializer.CONFIGS] - calls_by_class = {call.args[0]: call.kwargs for call in create_instance.call_args_list} - assert set(calls_by_class) == expected_classes - assert calls_by_class["Base64Converter"] == {} - assert calls_by_class["VariationConverter"] == {"converter_target": "adversarial_chat"} - assert "TenseConverter" not in calls_by_class + assert len(names) == len(set(names)) async def test_initialize_continues_after_expected_construction_error( caplog: pytest.LogCaptureFixture, ) -> None: registry = ConverterRegistry.get_registry_singleton() - metadata_items = [ - _get_metadata(registry=registry, name="Base64Converter"), - _get_metadata(registry=registry, name="ROT13Converter"), - ] + configs = _get_configs("base64", "leetspeak", "rot13") with ( - patch.object(registry, "get_all_registered_class_metadata", return_value=metadata_items), + patch.object(ConverterInitializer, "CONFIGS", configs), patch.object( registry, "create_instance", - side_effect=[ValueError("missing configuration"), ROT13Converter()], + side_effect=[ + ValueError("missing configuration"), + LeetspeakConverter(), + ROT13Converter(), + ], ), caplog.at_level(logging.WARNING, logger="pyrit.setup.initializers.converters"), ): await ConverterInitializer().initialize_async() assert "base64" not in registry.instances + assert isinstance(registry.instances.get("leetspeak"), LeetspeakConverter) assert isinstance(registry.instances.get("rot13"), ROT13Converter) assert "Skipping converter 'base64': missing configuration" in caplog.text From f289db0a0a66dca506f9cd2adcdad5d9cc8538ea Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Wed, 2 Sep 2026 16:28:31 -0700 Subject: [PATCH 3/3] TEST: Stabilize initializer dialog test Wait for the Fluent UI dialog and its form controls when the frontend test suite runs under load. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6f6b8dd2-e89a-43b2-9781-68d6acb6d76c --- .../Configuration/Configuration.test.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Configuration/Configuration.test.tsx b/frontend/src/components/Configuration/Configuration.test.tsx index 1c2ab32b4e..2f4986df92 100644 --- a/frontend/src/components/Configuration/Configuration.test.tsx +++ b/frontend/src/components/Configuration/Configuration.test.tsx @@ -26,6 +26,9 @@ jest.mock('@/services/api', () => ({ const mockedConfigurationApi = jest.mocked(configurationApi) const mockedInitializersApi = jest.mocked(initializersApi) +// Fluent UI dialogs can render slowly in JSDOM under full test load. +jest.setTimeout(60_000) + function renderPage(): void { render( @@ -204,8 +207,17 @@ describe('Configuration', () => { { selector: 'label' }, )).toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'Add initializer' })) - const dialog = screen.getByRole('dialog', { name: 'Add custom initializer' }) - await user.type(within(dialog).getByRole('textbox', { name: /Initializer name/ }), 'new_custom') + const dialog = await screen.findByRole( + 'dialog', + { name: 'Add custom initializer' }, + { timeout: 15_000 }, + ) + const nameInput = await within(dialog).findByRole( + 'textbox', + { name: /Initializer name/ }, + { timeout: 15_000 }, + ) + await user.type(nameInput, 'new_custom') fireEvent.change(within(dialog).getByRole('textbox', { name: 'Python source' }), { target: { value: 'class NewCustom: pass' }, })