diff --git a/.pyrit_conf_example b/.pyrit_conf_example index 978c8a293f..189eb8bdcd 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -23,6 +23,7 @@ memory_db_type: sqlite # # Available initializers: # - target: Registers available prompt targets into the TargetRegistry +# - converter: Registers the curated core converter presets # - scorer: Registers pre-configured scorers into the ScorerRegistry # - technique: Registers attack techniques into the AttackTechniqueRegistry # - load_default_datasets: Optionally preloads all registered datasets into memory @@ -49,6 +50,7 @@ initializers: tags: - default - scorer + - name: converter - name: scorer - name: technique # Optional full preload/cache warming for offline or shared environments. diff --git a/MANIFEST.in b/MANIFEST.in index 9ceb63cd55..6995774068 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ recursive-include pyrit *.prompt recursive-include pyrit *.yaml recursive-include pyrit *.pdf recursive-include pyrit *.png +recursive-include pyrit *.jpg recursive-include pyrit *.wav recursive-include pyrit *.mp4 recursive-include pyrit *.md diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index 54c530537a..20f922c9bd 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -103,9 +103,19 @@ Most users should enable the following initializers. These are what the `.pyrit_ | Initializer | What It Registers | When You Need It | | --- | --- | --- | | `target` | Prompt targets (OpenAI, Azure, AML, etc.) into the `TargetRegistry` | Recommended for `pyrit_scan` and registry-based workflows | +| `converter` | Curated core converter presets | Recommended when attaching registered converters to techniques | | `scorer` | Scorers (refusal, content safety, harm-category, Likert, etc.) into the `ScorerRegistry` | Recommended for automated scoring and `pyrit_scan` evaluations | | `technique` | Attack techniques into the `AttackTechniqueRegistry` | Recommended for scenarios that select registered techniques | +The `converter` initializer registers a curated set of text, LLM, audio, and image converter presets. The +LLM-backed presets use the registered `adversarial_chat` target and are skipped when that target is not available. +The Azure Speech preset uses the standard Azure Speech environment variables and is skipped when they are not set. +Use `pyrit_scan list-converters` to list the presets that were registered. + +Parameterized presets include past and future tense, professional and sarcastic tone, Spanish translation, +whitespace replacement with underscores, the `jailbreak_1.yaml` text jailbreak template, the packaged blank +canvas for text-to-image conversion, and the packaged benign cake image for the transparency attack. + ```{note} **Execution order follows listing order.** Initializers execute in the order they appear in the config. Ensure dependencies are satisfied — for example, list `target` before `scorer` since scorers need targets to be registered first. ``` @@ -123,6 +133,7 @@ initializers: tags: - default - scorer + - name: converter - name: scorer - name: technique ``` @@ -404,6 +415,7 @@ initializers: tags: - default - scorer + - name: converter - name: scorer - name: technique # Optional full preload/cache warming; scenarios fetch requested datasets on demand. diff --git a/frontend/src/components/Configuration/Configuration.test.tsx b/frontend/src/components/Configuration/Configuration.test.tsx index 1c2ab32b4e..2f4986df92 100644 --- a/frontend/src/components/Configuration/Configuration.test.tsx +++ b/frontend/src/components/Configuration/Configuration.test.tsx @@ -26,6 +26,9 @@ jest.mock('@/services/api', () => ({ const mockedConfigurationApi = jest.mocked(configurationApi) const mockedInitializersApi = jest.mocked(initializersApi) +// Fluent UI dialogs can render slowly in JSDOM under full test load. +jest.setTimeout(60_000) + function renderPage(): void { render( @@ -204,8 +207,17 @@ describe('Configuration', () => { { selector: 'label' }, )).toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'Add initializer' })) - const dialog = screen.getByRole('dialog', { name: 'Add custom initializer' }) - await user.type(within(dialog).getByRole('textbox', { name: /Initializer name/ }), 'new_custom') + const dialog = await screen.findByRole( + 'dialog', + { name: 'Add custom initializer' }, + { timeout: 15_000 }, + ) + const nameInput = await within(dialog).findByRole( + 'textbox', + { name: /Initializer name/ }, + { timeout: 15_000 }, + ) + await user.type(nameInput, 'new_custom') fireEvent.change(within(dialog).getByRole('textbox', { name: 'Python source' }), { target: { value: 'class NewCustom: pass' }, }) diff --git a/pyrit/datasets/seed_datasets/local/examples/benign_cake_question.jpg b/pyrit/datasets/seed_datasets/local/examples/benign_cake_question.jpg new file mode 100644 index 0000000000..11652dcd27 Binary files /dev/null and b/pyrit/datasets/seed_datasets/local/examples/benign_cake_question.jpg differ diff --git a/pyrit/setup/initializers/__init__.py b/pyrit/setup/initializers/__init__.py index ceb5c65bd8..e8ab3be972 100644 --- a/pyrit/setup/initializers/__init__.py +++ b/pyrit/setup/initializers/__init__.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from pyrit.models.parameter import Parameter + from pyrit.setup.initializers.converters import ConverterInitializer from pyrit.setup.initializers.load_default_datasets import LoadDefaultDatasets from pyrit.setup.initializers.preload_scenario_metadata import PreloadScenarioMetadata from pyrit.setup.initializers.refresh_datasets import RefreshDatasets @@ -21,6 +22,7 @@ _LAZY_EXPORTS: dict[str, str | tuple[str, str | None]] = { "Parameter": "pyrit.models.parameter", "PyRITInitializer": "pyrit.setup.pyrit_initializer", + "ConverterInitializer": "pyrit.setup.initializers.converters", "TechniqueInitializer": "pyrit.setup.initializers.techniques", "ScorerInitializer": "pyrit.setup.initializers.scorers", "TargetInitializer": "pyrit.setup.initializers.targets", diff --git a/pyrit/setup/initializers/converters.py b/pyrit/setup/initializers/converters.py new file mode 100644 index 0000000000..93034295cd --- /dev/null +++ b/pyrit/setup/initializers/converters.py @@ -0,0 +1,187 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Initializer for registering the core converter presets.""" + +import asyncio +import logging +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar + +from pyrit.common.path import DATASETS_PATH +from pyrit.registry import ConverterRegistry +from pyrit.setup.pyrit_initializer import PyRITInitializer + +if TYPE_CHECKING: + from pyrit.converter import Converter + +logger = logging.getLogger(__name__) + + +def _get_text_jailbreak_args() -> dict[str, Any]: + """ + Build the constructor arguments for the text jailbreak preset. + + Returns: + dict[str, Any]: Constructor arguments containing the jailbreak template. + """ + from pyrit.datasets import TextJailBreak + + return {"jailbreak_template": TextJailBreak(template_file_name="jailbreak_1.yaml")} + + +@dataclass(frozen=True) +class ConverterConfig: + """Configuration for a converter preset.""" + + registry_name: str + converter_type: str + constructor_args: dict[str, Any] = field(default_factory=dict) + constructor_args_factory: Callable[[], dict[str, Any]] | None = None + + def get_constructor_args(self) -> dict[str, Any]: + """ + Get a new constructor argument dictionary for this preset. + + Returns: + dict[str, Any]: A copy of the configured arguments with deferred arguments included. + """ + args = dict(self.constructor_args) + if self.constructor_args_factory: + args.update(self.constructor_args_factory()) + return args + + +class ConverterInitializer(PyRITInitializer): + """ + Register the curated core converter presets into the ConverterRegistry. + + Each preset explicitly declares its constructor arguments and target references. + Custom initializers can use the same pattern to register additional or parameterized + converter presets. + """ + + CONFIGS: ClassVar[tuple[ConverterConfig, ...]] = ( + ConverterConfig(registry_name="base64", converter_type="Base64Converter"), + ConverterConfig(registry_name="binary", converter_type="BinaryConverter"), + ConverterConfig(registry_name="char_swap", converter_type="CharSwapConverter"), + ConverterConfig(registry_name="ecoji", converter_type="EcojiConverter"), + ConverterConfig(registry_name="insert_punctuation", converter_type="InsertPunctuationConverter"), + ConverterConfig(registry_name="leetspeak", converter_type="LeetspeakConverter"), + ConverterConfig(registry_name="rot13", converter_type="ROT13Converter"), + ConverterConfig( + registry_name="search_replace", + converter_type="SearchReplaceConverter", + constructor_args={"pattern": r"\s+", "replace": "_"}, + ), + ConverterConfig(registry_name="string_join", converter_type="StringJoinConverter"), + ConverterConfig( + registry_name="text_jailbreak", + converter_type="TextJailbreakConverter", + constructor_args_factory=_get_text_jailbreak_args, + ), + ConverterConfig(registry_name="zalgo", converter_type="ZalgoConverter"), + ConverterConfig( + registry_name="malicious_question_generator", + converter_type="MaliciousQuestionGeneratorConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="math_prompt", + converter_type="MathPromptConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="noise", + converter_type="NoiseConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="tense_future", + converter_type="TenseConverter", + constructor_args={"converter_target": "adversarial_chat", "tense": "future"}, + ), + ConverterConfig( + registry_name="tense_past", + converter_type="TenseConverter", + constructor_args={"converter_target": "adversarial_chat", "tense": "past"}, + ), + ConverterConfig( + registry_name="tone_professional", + converter_type="ToneConverter", + constructor_args={"converter_target": "adversarial_chat", "tone": "professional"}, + ), + ConverterConfig( + registry_name="tone_sarcastic", + converter_type="ToneConverter", + constructor_args={"converter_target": "adversarial_chat", "tone": "sarcastic"}, + ), + ConverterConfig( + registry_name="translation_spanish", + converter_type="TranslationConverter", + constructor_args={"converter_target": "adversarial_chat", "language": "Spanish"}, + ), + ConverterConfig( + registry_name="variation", + converter_type="VariationConverter", + constructor_args={"converter_target": "adversarial_chat"}, + ), + ConverterConfig( + registry_name="add_image_text", + converter_type="AddImageTextConverter", + constructor_args={ + "img_to_add": str(DATASETS_PATH / "seed_datasets" / "local" / "examples" / "blank_canvas.png") + }, + ), + ConverterConfig( + registry_name="add_text_image", + converter_type="AddTextImageConverter", + constructor_args={"text_to_add": "PyRIT"}, + ), + ConverterConfig( + registry_name="azure_speech_audio_to_text", + converter_type="AzureSpeechAudioToTextConverter", + ), + ConverterConfig(registry_name="image_color_saturation", converter_type="ImageColorSaturationConverter"), + ConverterConfig(registry_name="image_compression", converter_type="ImageCompressionConverter"), + ConverterConfig(registry_name="image_rotation", converter_type="ImageRotationConverter"), + ConverterConfig(registry_name="qr_code", converter_type="QRCodeConverter"), + ConverterConfig( + registry_name="transparency_attack", + converter_type="TransparencyAttackConverter", + constructor_args={ + "benign_image_path": DATASETS_PATH / "seed_datasets" / "local" / "examples" / "benign_cake_question.jpg" + }, + ), + ) + + async def initialize_async(self) -> None: + """Create and register the core converter presets.""" + converter_registry = ConverterRegistry.get_registry_singleton() + + for config in self.CONFIGS: + try: + converter = await asyncio.to_thread( + self._create_converter, + converter_registry=converter_registry, + config=config, + ) + converter_registry.instances.register(converter, name=config.registry_name) + logger.info("Registered converter: %s", config.registry_name) + except (FileNotFoundError, KeyError, TypeError, ValueError) as ex: + logger.warning("Skipping converter '%s': %s", config.registry_name, ex) + + def _create_converter( + self, + *, + converter_registry: ConverterRegistry, + config: ConverterConfig, + ) -> "Converter": + """ + Create one configured converter outside the event-loop thread. + + Returns: + Converter: The configured converter instance. + """ + return converter_registry.create_instance(config.converter_type, **config.get_constructor_args()) diff --git a/tests/unit/setup/test_configuration_loader.py b/tests/unit/setup/test_configuration_loader.py index 87aa79709d..04b0b3f39d 100644 --- a/tests/unit/setup/test_configuration_loader.py +++ b/tests/unit/setup/test_configuration_loader.py @@ -272,7 +272,7 @@ def test_example_uses_on_demand_dataset_fetching(self) -> None: initializer_names = [initializer.name for initializer in config._initializer_configs] example_text = example_path.read_text(encoding="utf-8") - assert initializer_names == ["target", "scorer", "technique"] + assert initializer_names == ["target", "converter", "scorer", "technique"] assert "# - name: load_default_datasets" in example_text assert "several minutes" in example_text assert "provider credentials" in example_text diff --git a/tests/unit/setup/test_converter_initializer.py b/tests/unit/setup/test_converter_initializer.py new file mode 100644 index 0000000000..b5ce7f3b49 --- /dev/null +++ b/tests/unit/setup/test_converter_initializer.py @@ -0,0 +1,202 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Tests for the converter initializer.""" + +import logging +from collections.abc import Iterator +from unittest.mock import call, patch + +import pytest + +from pyrit.converter import Base64Converter, LeetspeakConverter, ROT13Converter, VariationConverter +from pyrit.registry import ConverterRegistry, InitializerRegistry, TargetRegistry +from pyrit.setup.initializers import ConverterInitializer +from pyrit.setup.initializers.converters import ConverterConfig +from tests.unit.mocks import MockPromptTarget + + +@pytest.fixture(autouse=True) +def reset_registries() -> Iterator[None]: + """Reset the component registries around each test.""" + ConverterRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + yield + ConverterRegistry.reset_registry_singleton() + TargetRegistry.reset_registry_singleton() + + +def _get_configs(*names: str) -> tuple[ConverterConfig, ...]: + configs_by_name = {config.registry_name: config for config in ConverterInitializer.CONFIGS} + return tuple(configs_by_name[name] for name in names) + + +async def test_initialize_registers_core_converter_presets() -> None: + registry = ConverterRegistry.get_registry_singleton() + + with patch.object( + ConverterInitializer, + "CONFIGS", + _get_configs("base64", "leetspeak", "rot13"), + ): + await ConverterInitializer().initialize_async() + + assert isinstance(registry.instances.get("base64"), Base64Converter) + assert isinstance(registry.instances.get("leetspeak"), LeetspeakConverter) + assert isinstance(registry.instances.get("rot13"), ROT13Converter) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_initialize_registers_variation_with_declared_target() -> None: + converter_registry = ConverterRegistry.get_registry_singleton() + target_registry = TargetRegistry.get_registry_singleton() + adversarial_chat = MockPromptTarget() + target_registry.instances.register(adversarial_chat, name="adversarial_chat") + + with patch.object(ConverterInitializer, "CONFIGS", _get_configs("variation")): + await ConverterInitializer().initialize_async() + + converter = converter_registry.instances.get("variation") + assert isinstance(converter, VariationConverter) + assert converter._converter_target is adversarial_chat + + +async def test_initialize_skips_converter_without_declared_target() -> None: + registry = ConverterRegistry.get_registry_singleton() + + with patch.object(ConverterInitializer, "CONFIGS", _get_configs("variation")): + await ConverterInitializer().initialize_async() + + assert "variation" not in registry.instances + + +async def test_initialize_uses_explicit_converter_configs() -> None: + converter_registry = ConverterRegistry.get_registry_singleton() + + with patch.object(converter_registry, "create_instance", return_value=Base64Converter()) as create_instance: + await ConverterInitializer().initialize_async() + + assert converter_registry.instances.get_names() == sorted( + { + "add_image_text", + "add_text_image", + "azure_speech_audio_to_text", + "base64", + "binary", + "char_swap", + "ecoji", + "image_color_saturation", + "image_compression", + "image_rotation", + "insert_punctuation", + "leetspeak", + "malicious_question_generator", + "math_prompt", + "noise", + "qr_code", + "rot13", + "search_replace", + "string_join", + "tense_future", + "tense_past", + "text_jailbreak", + "tone_professional", + "tone_sarcastic", + "translation_spanish", + "transparency_attack", + "variation", + "zalgo", + } + ) + assert create_instance.call_count == len(ConverterInitializer.CONFIGS) + assert call("Base64Converter") in create_instance.call_args_list + assert ( + call( + "SearchReplaceConverter", + pattern=r"\s+", + replace="_", + ) + in create_instance.call_args_list + ) + assert ( + call( + "TenseConverter", + converter_target="adversarial_chat", + tense="future", + ) + in create_instance.call_args_list + ) + assert ( + call( + "ToneConverter", + converter_target="adversarial_chat", + tone="sarcastic", + ) + in create_instance.call_args_list + ) + assert ( + call( + "TranslationConverter", + converter_target="adversarial_chat", + language="Spanish", + ) + in create_instance.call_args_list + ) + assert ( + call( + "VariationConverter", + converter_target="adversarial_chat", + ) + in create_instance.call_args_list + ) + + +@pytest.mark.usefixtures("patch_central_database") +async def test_explicit_converter_configs_can_be_constructed() -> None: + converter_registry = ConverterRegistry.get_registry_singleton() + target_registry = TargetRegistry.get_registry_singleton() + target_registry.instances.register(MockPromptTarget(), name="adversarial_chat") + + await ConverterInitializer().initialize_async() + + expected_names = {config.registry_name for config in ConverterInitializer.CONFIGS} + expected_names.remove("azure_speech_audio_to_text") + assert expected_names <= set(converter_registry.instances.get_names()) + + +def test_configs_have_unique_names() -> None: + names = [config.registry_name for config in ConverterInitializer.CONFIGS] + + assert len(names) == len(set(names)) + + +async def test_initialize_continues_after_expected_construction_error( + caplog: pytest.LogCaptureFixture, +) -> None: + registry = ConverterRegistry.get_registry_singleton() + configs = _get_configs("base64", "leetspeak", "rot13") + + with ( + patch.object(ConverterInitializer, "CONFIGS", configs), + patch.object( + registry, + "create_instance", + side_effect=[ + ValueError("missing configuration"), + LeetspeakConverter(), + ROT13Converter(), + ], + ), + caplog.at_level(logging.WARNING, logger="pyrit.setup.initializers.converters"), + ): + await ConverterInitializer().initialize_async() + + assert "base64" not in registry.instances + assert isinstance(registry.instances.get("leetspeak"), LeetspeakConverter) + assert isinstance(registry.instances.get("rot13"), ROT13Converter) + assert "Skipping converter 'base64': missing configuration" in caplog.text + + +def test_initializer_is_discovered() -> None: + registry = InitializerRegistry() + assert "converter" in registry.get_class_names()