diff --git a/doc/code/datasets/2_seed_programming.ipynb b/doc/code/datasets/2_seed_programming.ipynb index 87c85e10af..575f480636 100644 --- a/doc/code/datasets/2_seed_programming.ipynb +++ b/doc/code/datasets/2_seed_programming.ipynb @@ -488,7 +488,7 @@ " role=\"system\",\n", " ),\n", " SeedSimulatedConversation(\n", - " adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH / \"naive_crescendo.yaml\",\n", + " adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(EXECUTOR_RED_TEAM_PATH / \"naive_crescendo.yaml\"),\n", " sequence=1,\n", " num_turns=4,\n", " next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / \"direct_next_message.yaml\",\n", diff --git a/doc/code/datasets/2_seed_programming.py b/doc/code/datasets/2_seed_programming.py index 134a561592..81a2599c2c 100644 --- a/doc/code/datasets/2_seed_programming.py +++ b/doc/code/datasets/2_seed_programming.py @@ -102,7 +102,7 @@ role="system", ), SeedSimulatedConversation( - adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH / "naive_crescendo.yaml", + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(EXECUTOR_RED_TEAM_PATH / "naive_crescendo.yaml"), sequence=1, num_turns=4, next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "direct_next_message.yaml", diff --git a/doc/code/datasets/5_simulated_conversation.ipynb b/doc/code/datasets/5_simulated_conversation.ipynb index b3fffd5d1e..57c5cc438f 100644 --- a/doc/code/datasets/5_simulated_conversation.ipynb +++ b/doc/code/datasets/5_simulated_conversation.ipynb @@ -24,7 +24,7 @@ "\n", "## Generating a Simulated Conversation\n", "\n", - "The function takes an objective, an adversarial chat model, a scorer, and a system prompt path.\n", + "The function takes an objective, an adversarial chat model, a scorer, and a system `SeedPrompt`.\n", "It runs a `RedTeamingAttack` internally with the adversarial LLM playing both attacker and target\n", "roles." ] @@ -60,11 +60,9 @@ } ], "source": [ - "from pathlib import Path\n", - "\n", "from pyrit.common.path import EXECUTOR_SEED_PROMPT_PATH\n", "from pyrit.executor.attack import generate_simulated_conversation_async\n", - "from pyrit.models import SeedGroup\n", + "from pyrit.models import SeedGroup, SeedPrompt\n", "from pyrit.output import output_attack_async\n", "from pyrit.prompt_target import OpenAIChatTarget\n", "from pyrit.score import SelfAskRefusalScorer\n", @@ -83,7 +81,9 @@ " adversarial_chat=adversarial_chat,\n", " objective_scorer=objective_scorer,\n", " num_turns=3,\n", - " adversarial_chat_system_prompt_path=Path(EXECUTOR_SEED_PROMPT_PATH) / \"red_teaming\" / \"naive_crescendo.yaml\",\n", + " adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(\n", + " EXECUTOR_SEED_PROMPT_PATH / \"red_teaming\" / \"naive_crescendo.yaml\"\n", + " ),\n", ")\n", "\n", "print(f\"Generated {len(simulated_conversation_prompts)} messages\")" @@ -517,7 +517,8 @@ "| `adversarial_chat` | `PromptTarget` | The LLM that generates attack prompts (also plays the simulated target). Must declare `supports_multi_turn=True` and `supports_editable_history=True`. |\n", "| `objective_scorer` | `TrueFalseScorer` | Evaluates whether the final turn achieved the objective |\n", "| `num_turns` | `int` | Number of conversation turns to generate (default: 3) |\n", - "| `adversarial_chat_system_prompt_path` | `str \\| Path` | System prompt for the adversarial chat role |\n", + "| `adversarial_chat_system_prompt` | `SeedPrompt` | System prompt for the adversarial chat role |\n", + "| `adversarial_chat_system_prompt_path` | `str \\| Path \\| None` | Compatibility adapter for legacy path-based callers |\n", "| `simulated_target_system_prompt_path` | `str \\| Path \\| None` | Optional system prompt for the simulated target role |\n", "| `next_message_system_prompt_path` | `str \\| Path \\| None` | Optional path to generate a final user message that elicits objective fulfillment |\n", "| `attack_converter_config` | `AttackConverterConfig \\| None` | Optional converter configuration for the attack |\n", diff --git a/doc/code/datasets/5_simulated_conversation.py b/doc/code/datasets/5_simulated_conversation.py index c4ed3e09ba..9185d07063 100644 --- a/doc/code/datasets/5_simulated_conversation.py +++ b/doc/code/datasets/5_simulated_conversation.py @@ -28,16 +28,14 @@ # # ## Generating a Simulated Conversation # -# The function takes an objective, an adversarial chat model, a scorer, and a system prompt path. +# The function takes an objective, an adversarial chat model, a scorer, and a system `SeedPrompt`. # It runs a `RedTeamingAttack` internally with the adversarial LLM playing both attacker and target # roles. # %% -from pathlib import Path - from pyrit.common.path import EXECUTOR_SEED_PROMPT_PATH from pyrit.executor.attack import generate_simulated_conversation_async -from pyrit.models import SeedGroup +from pyrit.models import SeedGroup, SeedPrompt from pyrit.output import output_attack_async from pyrit.prompt_target import OpenAIChatTarget from pyrit.score import SelfAskRefusalScorer @@ -56,7 +54,9 @@ adversarial_chat=adversarial_chat, objective_scorer=objective_scorer, num_turns=3, - adversarial_chat_system_prompt_path=Path(EXECUTOR_SEED_PROMPT_PATH) / "red_teaming" / "naive_crescendo.yaml", + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( + EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "naive_crescendo.yaml" + ), ) print(f"Generated {len(simulated_conversation_prompts)} messages") @@ -126,7 +126,8 @@ # | `adversarial_chat` | `PromptTarget` | The LLM that generates attack prompts (also plays the simulated target). Must declare `supports_multi_turn=True` and `supports_editable_history=True`. | # | `objective_scorer` | `TrueFalseScorer` | Evaluates whether the final turn achieved the objective | # | `num_turns` | `int` | Number of conversation turns to generate (default: 3) | -# | `adversarial_chat_system_prompt_path` | `str \| Path` | System prompt for the adversarial chat role | +# | `adversarial_chat_system_prompt` | `SeedPrompt` | System prompt for the adversarial chat role | +# | `adversarial_chat_system_prompt_path` | `str \| Path \| None` | Compatibility adapter for legacy path-based callers | # | `simulated_target_system_prompt_path` | `str \| Path \| None` | Optional system prompt for the simulated target role | # | `next_message_system_prompt_path` | `str \| Path \| None` | Optional path to generate a final user message that elicits objective fulfillment | # | `attack_converter_config` | `AttackConverterConfig \| None` | Optional converter configuration for the attack | diff --git a/doc/code/scenarios/0_attack_techniques.ipynb b/doc/code/scenarios/0_attack_techniques.ipynb index 92e0512f93..131aaeed41 100644 --- a/doc/code/scenarios/0_attack_techniques.ipynb +++ b/doc/code/scenarios/0_attack_techniques.ipynb @@ -38,7 +38,13 @@ "\n", "The objective is *not* part of the technique — it stays separate and is supplied by the dataset at\n", "run time. You rarely build a technique by hand; instead you register a **factory** and let scenarios\n", - "construct techniques on demand with the scenario's own objective target and scorer." + "construct techniques on demand with the scenario's own objective target and scorer.\n", + "\n", + "`adversarial_chat_system_prompt` accepts a `SeedPrompt`, so created techniques carry portable prompt\n", + "content rather than a runtime file dependency. Callers that own YAML load it with\n", + "`SeedPrompt.from_yaml_file(...)` at setup time. The legacy `adversarial_chat_system_prompt_path` name\n", + "remains a compatibility adapter, and if neither parameter is supplied, the existing\n", + "`red_teaming/{technique_name}.yaml` convention remains the default." ] }, { diff --git a/doc/code/scenarios/0_attack_techniques.py b/doc/code/scenarios/0_attack_techniques.py index 7e96150f83..da8c3cc093 100644 --- a/doc/code/scenarios/0_attack_techniques.py +++ b/doc/code/scenarios/0_attack_techniques.py @@ -43,6 +43,12 @@ # The objective is *not* part of the technique — it stays separate and is supplied by the dataset at # run time. You rarely build a technique by hand; instead you register a **factory** and let scenarios # construct techniques on demand with the scenario's own objective target and scorer. +# +# `adversarial_chat_system_prompt` accepts a `SeedPrompt`, so created techniques carry portable prompt +# content rather than a runtime file dependency. Callers that own YAML load it with +# `SeedPrompt.from_yaml_file(...)` at setup time. The legacy `adversarial_chat_system_prompt_path` name +# remains a compatibility adapter, and if neither parameter is supplied, the existing +# `red_teaming/{technique_name}.yaml` convention remains the default. # %% [markdown] # ## Where techniques come from: initializers diff --git a/pyrit/executor/attack/core/attack_parameters.py b/pyrit/executor/attack/core/attack_parameters.py index e03108448a..395b0db548 100644 --- a/pyrit/executor/attack/core/attack_parameters.py +++ b/pyrit/executor/attack/core/attack_parameters.py @@ -112,6 +112,7 @@ async def from_seed_group_async( """ # Import here to avoid circular imports from pyrit.executor.attack.multi_turn.simulated_conversation import ( + _resolve_adversarial_chat_system_prompt_async, generate_simulated_conversation_async, ) @@ -159,6 +160,11 @@ async def from_seed_group_async( if objective_scorer is None: raise ValueError("objective_scorer is required when seed_group has a simulated conversation config") + adversarial_chat_system_prompt = await _resolve_adversarial_chat_system_prompt_async( + adversarial_chat_system_prompt_path=simulated_conversation_config.adversarial_chat_system_prompt_path, + adversarial_chat_system_prompt=simulated_conversation_config.adversarial_chat_system_prompt, + ) + # Generate the simulated conversation - returns list[SeedPrompt] simulated_prompts = await generate_simulated_conversation_async( objective=seed_group.objective.value, @@ -166,7 +172,7 @@ async def from_seed_group_async( objective_scorer=objective_scorer, num_turns=simulated_conversation_config.num_turns, starting_sequence=simulated_conversation_config.sequence, - adversarial_chat_system_prompt_path=simulated_conversation_config.adversarial_chat_system_prompt_path, + adversarial_chat_system_prompt=adversarial_chat_system_prompt, simulated_target_system_prompt_path=simulated_conversation_config.simulated_target_system_prompt_path, next_message_system_prompt_path=simulated_conversation_config.next_message_system_prompt_path, ) diff --git a/pyrit/executor/attack/multi_turn/simulated_conversation.py b/pyrit/executor/attack/multi_turn/simulated_conversation.py index 872f9571fc..aaa38f4f53 100644 --- a/pyrit/executor/attack/multi_turn/simulated_conversation.py +++ b/pyrit/executor/attack/multi_turn/simulated_conversation.py @@ -10,6 +10,7 @@ from __future__ import annotations +import asyncio import logging from typing import TYPE_CHECKING @@ -43,7 +44,8 @@ async def generate_simulated_conversation_async( objective_scorer: TrueFalseScorer, num_turns: int = 3, starting_sequence: int = 0, - adversarial_chat_system_prompt_path: str | Path, + adversarial_chat_system_prompt_path: str | Path | None = None, + adversarial_chat_system_prompt: SeedPrompt | None = None, simulated_target_system_prompt_path: str | Path | None = None, next_message_system_prompt_path: str | Path | None = None, attack_converter_config: AttackConverterConfig | None = None, @@ -70,7 +72,10 @@ async def generate_simulated_conversation_async( num_turns: Number of conversation turns to generate. Defaults to 3. starting_sequence: The starting sequence number for the generated SeedPrompts. Each message gets an incrementing sequence number. Defaults to 0. - adversarial_chat_system_prompt_path: Path to the system prompt for the adversarial chat. + adversarial_chat_system_prompt_path: Compatibility-only path adapter for the adversarial chat + system prompt. New callers should load the YAML at their composition boundary and pass + ``adversarial_chat_system_prompt``. + adversarial_chat_system_prompt: Canonical inline system prompt for the adversarial chat. simulated_target_system_prompt_path: Path to the system prompt for the simulated target. If None, no system prompt is used for the simulated target. next_message_system_prompt_path: Optional path to a system prompt for generating @@ -89,7 +94,7 @@ async def generate_simulated_conversation_async( generated to elicit the objective fulfillment. Raises: - ValueError: If num_turns is not a positive integer. + ValueError: If num_turns is not positive or the adversarial prompt source is ambiguous or missing. """ # Use the same LLM for both adversarial chat and simulated target # They get different system prompts to play different roles @@ -105,10 +110,9 @@ async def generate_simulated_conversation_async( simulated_target_system_prompt_path=simulated_target_system_prompt_path, ) - # Create adversarial config for the simulation. Load the optional path into a SeedPrompt so the - # resolved prompt is stored directly on the configuration. - adversarial_system_prompt = ( - SeedPrompt.from_yaml_file(adversarial_chat_system_prompt_path) if adversarial_chat_system_prompt_path else None + adversarial_system_prompt = await _resolve_adversarial_chat_system_prompt_async( + adversarial_chat_system_prompt_path=adversarial_chat_system_prompt_path, + adversarial_chat_system_prompt=adversarial_chat_system_prompt, ) adversarial_config = AttackAdversarialConfig( target=adversarial_chat, @@ -176,6 +180,35 @@ async def generate_simulated_conversation_async( return seed_prompts +async def _resolve_adversarial_chat_system_prompt_async( + *, + adversarial_chat_system_prompt_path: str | Path | None, + adversarial_chat_system_prompt: SeedPrompt | None, +) -> SeedPrompt: + """ + Adapt a legacy path-backed prompt to the canonical inline execution input. + + Args: + adversarial_chat_system_prompt_path: Legacy YAML prompt path. + adversarial_chat_system_prompt: Canonical inline prompt. + + Returns: + The resolved adversarial chat system prompt. + + Raises: + ValueError: If both or neither prompt sources are provided. + """ + has_prompt_path = adversarial_chat_system_prompt_path is not None + has_inline_prompt = adversarial_chat_system_prompt is not None + if has_prompt_path == has_inline_prompt: + raise ValueError("Set exactly one of adversarial_chat_system_prompt_path or adversarial_chat_system_prompt.") + if adversarial_chat_system_prompt is not None: + return adversarial_chat_system_prompt + + assert adversarial_chat_system_prompt_path is not None + return await asyncio.to_thread(SeedPrompt.from_yaml_file, adversarial_chat_system_prompt_path) + + async def _generate_next_message_async( *, objective: str, diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 12a1dea54d..1cdb8d9c39 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -1497,8 +1497,10 @@ def get_seed(self) -> Seed: num_turns=config.get("num_turns", 3), sequence=config.get("sequence", 0), adversarial_chat_system_prompt_path=config.get("adversarial_chat_system_prompt_path"), + adversarial_chat_system_prompt=config.get("adversarial_chat_system_prompt"), simulated_target_system_prompt_path=config.get("simulated_target_system_prompt_path"), next_message_system_prompt_path=config.get("next_message_system_prompt_path"), + pyrit_version=config.get("pyrit_version"), ) return SeedPrompt( id=self.id, diff --git a/pyrit/models/seeds/seed_simulated_conversation.py b/pyrit/models/seeds/seed_simulated_conversation.py index f5c29525dc..f9d974cde5 100644 --- a/pyrit/models/seeds/seed_simulated_conversation.py +++ b/pyrit/models/seeds/seed_simulated_conversation.py @@ -46,7 +46,7 @@ class SeedSimulatedConversation(Seed): """ Configuration for generating a simulated conversation dynamically. - This class holds the paths and parameters needed to generate prepended conversation + This class holds the prompts, paths, and parameters needed to generate prepended conversation content by running an adversarial chat against a simulated (compliant) target. This is a pure configuration class. The actual generation is performed by @@ -58,7 +58,8 @@ class SeedSimulatedConversation(Seed): Attributes: num_turns: Number of conversation turns to generate. - adversarial_chat_system_prompt_path: Path to the adversarial chat system prompt YAML. + adversarial_chat_system_prompt_path: Legacy path to the adversarial chat system prompt YAML. + adversarial_chat_system_prompt: Canonical inline adversarial chat system prompt. simulated_target_system_prompt_path: Path to the simulated target system prompt YAML. Defaults to the compliant prompt if not specified. next_message_system_prompt_path: Optional path to the system prompt for generating @@ -85,7 +86,9 @@ class SeedSimulatedConversation(Seed): num_turns: int = 3 sequence: int = 0 - adversarial_chat_system_prompt_path: Path + # Retained for direct legacy construction and persisted path-backed records. + adversarial_chat_system_prompt_path: Path | None = None + adversarial_chat_system_prompt: SeedPrompt | None = None simulated_target_system_prompt_path: Path = SimulatedTargetSystemPromptPaths.COMPLIANT.value next_message_system_prompt_path: Path | None = None pyrit_version: str | None = None @@ -116,6 +119,12 @@ def _default_simulated_target_path(cls, value: Any) -> Any: @model_validator(mode="after") def _validate_and_compute_value(self) -> SeedSimulatedConversation: + has_prompt_path = self.adversarial_chat_system_prompt_path is not None + has_inline_prompt = self.adversarial_chat_system_prompt is not None + if has_prompt_path == has_inline_prompt: + raise ValueError( + "Set exactly one of adversarial_chat_system_prompt_path or adversarial_chat_system_prompt." + ) if self.num_turns <= 0: raise ValueError("num_turns must be a positive integer") if self.sequence < 0: @@ -133,18 +142,63 @@ def _compute_value(self) -> str: str: Deterministic JSON representation of this configuration. """ - config = { + config: dict[str, Any] = { "num_turns": self.num_turns, "sequence": self.sequence, - "adversarial_chat_system_prompt_path": str(self.adversarial_chat_system_prompt_path), "simulated_target_system_prompt_path": str(self.simulated_target_system_prompt_path), "next_message_system_prompt_path": ( str(self.next_message_system_prompt_path) if self.next_message_system_prompt_path else None ), "pyrit_version": self.pyrit_version, } + if self.adversarial_chat_system_prompt is not None: + config["adversarial_chat_system_prompt"] = self._serialize_adversarial_chat_system_prompt() + else: + config["adversarial_chat_system_prompt_path"] = str(self.adversarial_chat_system_prompt_path) return json.dumps(config, sort_keys=True, separators=(",", ":")) + def _serialize_adversarial_chat_system_prompt(self) -> dict[str, Any]: + """ + Serialize the inline prompt without generated identity or timestamp fields. + + Returns: + A deterministic JSON-compatible prompt representation. + """ + assert self.adversarial_chat_system_prompt is not None + prompt_data = self.adversarial_chat_system_prompt.model_dump( + mode="python", + exclude={"id", "date_added", "value_sha256", "prompt_group_id"}, + ) + self._reject_unordered_collections(prompt_data) + return self.adversarial_chat_system_prompt.model_dump( + mode="json", + exclude={"id", "date_added", "value_sha256", "prompt_group_id"}, + ) + + @classmethod + def _reject_unordered_collections(cls, value: Any) -> None: + """ + Reject values whose JSON list order can vary across processes. + + Args: + value: Prompt data to inspect recursively. + + Raises: + ValueError: If the prompt contains a set or frozenset. + """ + if isinstance(value, (set, frozenset)): + raise ValueError( + "Inline adversarial chat system prompts must use ordered JSON-compatible values; " + "set and frozenset values are not supported." + ) + if isinstance(value, dict): + for nested_key, nested_value in value.items(): + cls._reject_unordered_collections(nested_key) + cls._reject_unordered_collections(nested_value) + elif isinstance(value, (list, tuple)): + for nested_value in value: + cls._reject_unordered_collections(nested_value) + def get_identifier(self) -> dict[str, Any]: """ Get an identifier dict capturing this configuration for comparison/storage. @@ -153,17 +207,21 @@ def get_identifier(self) -> dict[str, Any]: Dictionary with configuration details. """ - return { + identifier: dict[str, Any] = { "__type__": "SeedSimulatedConversation", "num_turns": self.num_turns, "sequence": self.sequence, - "adversarial_chat_system_prompt_path": str(self.adversarial_chat_system_prompt_path), "simulated_target_system_prompt_path": str(self.simulated_target_system_prompt_path), "next_message_system_prompt_path": ( str(self.next_message_system_prompt_path) if self.next_message_system_prompt_path else None ), "pyrit_version": self.pyrit_version, } + if self.adversarial_chat_system_prompt is not None: + identifier["adversarial_chat_system_prompt"] = self._serialize_adversarial_chat_system_prompt() + else: + identifier["adversarial_chat_system_prompt_path"] = str(self.adversarial_chat_system_prompt_path) + return identifier def compute_hash(self) -> str: """ @@ -242,6 +300,15 @@ def __repr__(self) -> str: """ has_next_msg = self.next_message_system_prompt_path is not None + if self.adversarial_chat_system_prompt is not None: + adversarial_source = self.adversarial_chat_system_prompt.name or "" + return ( + f"" + ) + + assert self.adversarial_chat_system_prompt_path is not None return ( f" SeedPrompt: + """ + Resolve one simulated-conversation adversarial prompt source to a ``SeedPrompt``. + + The preferred source is an inline prompt. The + ``adversarial_chat_system_prompt_path`` parameter remains as a compatibility + adapter. When neither is explicit, ``default_system_prompt_path`` supplies + the factory's conventional name-based YAML fallback. + + Args: + adversarial_chat_system_prompt: Canonical inline prompt. + adversarial_chat_system_prompt_path: Legacy YAML prompt path alias. + default_system_prompt_path: YAML fallback used when no explicit source is provided. + + Returns: + The canonical inline prompt. + + Raises: + ValueError: If both sources are provided or no source/default is available. + TypeError: If the preferred source is not a SeedPrompt. + """ + if adversarial_chat_system_prompt_path is not None and adversarial_chat_system_prompt is not None: + raise ValueError("Set only one of adversarial_chat_system_prompt_path or adversarial_chat_system_prompt.") + + if adversarial_chat_system_prompt is not None: + if not isinstance(adversarial_chat_system_prompt, SeedPrompt): + raise TypeError( + "adversarial_chat_system_prompt must be a SeedPrompt; " + "load YAML with SeedPrompt.from_yaml_file(...) or use " + "adversarial_chat_system_prompt_path for legacy path compatibility." + ) + return adversarial_chat_system_prompt + + prompt_path = ( + adversarial_chat_system_prompt_path + if adversarial_chat_system_prompt_path is not None + else default_system_prompt_path + ) + if prompt_path is None: + raise ValueError( + "Set one of adversarial_chat_system_prompt or adversarial_chat_system_prompt_path, " + "or provide default_system_prompt_path." + ) + + return SeedPrompt.from_yaml_file(prompt_path) diff --git a/pyrit/scenario/scenarios/airt/psychosocial.py b/pyrit/scenario/scenarios/airt/psychosocial.py index af6b28e193..2705267cc2 100644 --- a/pyrit/scenario/scenarios/airt/psychosocial.py +++ b/pyrit/scenario/scenarios/airt/psychosocial.py @@ -3,6 +3,7 @@ from __future__ import annotations +import asyncio import logging import pathlib from dataclasses import dataclass @@ -569,6 +570,10 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list scorer = self._scorers_by_harm[harm.name] scoring_config = AttackScoringConfig(objective_scorer=scorer) + adversarial_system_prompt = await asyncio.to_thread( + SeedPrompt.from_yaml_file, + harm.escalation_prompt_path, + ) if context.include_baseline: baselines.append( @@ -584,14 +589,14 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list base_factory = AttackTechniqueFactory.with_simulated_conversation( name=f"psychosocial_{harm.name}", - adversarial_chat_system_prompt_path=harm.escalation_prompt_path, + adversarial_chat_system_prompt=adversarial_system_prompt, num_turns=max_turns, ) for technique in techniques: if technique is PsychosocialTechnique.Crescendo: attack_technique = self._build_crescendo_technique( - harm=harm, + adversarial_system_prompt=adversarial_system_prompt, objective_target=context.objective_target, adversarial_chat=adversarial_chat, scoring_config=scoring_config, @@ -626,7 +631,7 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list @staticmethod def _build_crescendo_technique( *, - harm: _SubHarm, + adversarial_system_prompt: SeedPrompt, objective_target: PromptTarget, adversarial_chat: PromptTarget, scoring_config: AttackScoringConfig, @@ -640,7 +645,7 @@ def _build_crescendo_technique( harm-specific framing as the simulated base. Args: - harm: The sub-harm being attacked. + adversarial_system_prompt: The sub-harm-specific escalation prompt. objective_target: The target under test. adversarial_chat: The adversarial chat driving Crescendo. scoring_config: The sub-harm's scoring config. @@ -653,7 +658,7 @@ def _build_crescendo_technique( objective_target=objective_target, attack_adversarial_config=AttackAdversarialConfig( target=adversarial_chat, - system_prompt=SeedPrompt.from_yaml_file(harm.escalation_prompt_path), + system_prompt=adversarial_system_prompt, ), attack_scoring_config=scoring_config, attack_converter_config=AttackConverterConfig(), diff --git a/pyrit/scenario/scenarios/airt/scam.py b/pyrit/scenario/scenarios/airt/scam.py index b7a64dd7be..f42fd0b22a 100644 --- a/pyrit/scenario/scenarios/airt/scam.py +++ b/pyrit/scenario/scenarios/airt/scam.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import asyncio import logging from pathlib import Path from typing import TYPE_CHECKING, Any @@ -156,7 +157,9 @@ def __init__( scenario_result_id=scenario_result_id, ) - def _get_atomic_attack_from_technique(self, *, technique: str, seed_groups: list[AttackSeedGroup]) -> AtomicAttack: + async def _get_atomic_attack_from_technique_async( + self, *, technique: str, seed_groups: list[AttackSeedGroup] + ) -> AtomicAttack: """ Translate the techniques into actual AtomicAttacks. @@ -179,8 +182,9 @@ def _get_atomic_attack_from_technique(self, *, technique: str, seed_groups: list if technique == "persuasive_rta": # Set system prompt to generic persuasion persona - self._adversarial_config.system_prompt = SeedPrompt.from_yaml_file( - Path(PERSUASION_DECEPTION_PATH, "persuasion_persona_generic.yaml").resolve() + self._adversarial_config.system_prompt = await asyncio.to_thread( + SeedPrompt.from_yaml_file, + Path(PERSUASION_DECEPTION_PATH, "persuasion_persona_generic.yaml").resolve(), ) attack_strategy = RedTeamingAttack( @@ -195,9 +199,10 @@ def _get_atomic_attack_from_technique(self, *, technique: str, seed_groups: list # objective is delivered to the target. role_play_technique = AttackTechniqueFactory.with_simulated_conversation( name="role_play_persuasion_written", - adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH - / "role_play" - / "role_play_persuasion_written.yaml", + adversarial_chat_system_prompt=await asyncio.to_thread( + SeedPrompt.from_yaml_file, + EXECUTOR_RED_TEAM_PATH / "role_play" / "role_play_persuasion_written.yaml", + ), next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", num_turns=2, ).create( @@ -219,9 +224,10 @@ def _get_atomic_attack_from_technique(self, *, technique: str, seed_groups: list # further, then a fixed "yes." is delivered to the target as the final user turn. context_compliance_technique = AttackTechniqueFactory.with_simulated_conversation( name="context_compliance", - adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH - / "context_compliance" - / "context_compliance.yaml", + adversarial_chat_system_prompt=await asyncio.to_thread( + SeedPrompt.from_yaml_file, + EXECUTOR_RED_TEAM_PATH / "context_compliance" / "context_compliance.yaml", + ), simulated_target_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "context_compliance_target.yaml", final_user_message="yes.", num_turns=1, @@ -272,7 +278,12 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list ) ) atomic_attacks.extend( - self._get_atomic_attack_from_technique(technique=technique, seed_groups=seed_groups) - for technique in techniques + [ + await self._get_atomic_attack_from_technique_async( + technique=technique, + seed_groups=seed_groups, + ) + for technique in techniques + ] ) return atomic_attacks diff --git a/pyrit/setup/initializers/techniques/core.py b/pyrit/setup/initializers/techniques/core.py index d4dbc9667c..33f6417581 100644 --- a/pyrit/setup/initializers/techniques/core.py +++ b/pyrit/setup/initializers/techniques/core.py @@ -112,7 +112,7 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: AttackTechniqueFactory.with_simulated_conversation( name="role_play_movie_script", description="Frames the objective as a movie beat by casting the adversarial chat as a screenwriter.", - adversarial_chat_system_prompt_path=( + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "role_play" / "role_play_movie_script.yaml" ), next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", @@ -122,7 +122,7 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: AttackTechniqueFactory.with_simulated_conversation( name="role_play_video_game", description="Frames the objective as an in-game mechanic by casting the adversarial chat as a designer.", - adversarial_chat_system_prompt_path=( + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "role_play" / "role_play_video_game.yaml" ), next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", @@ -132,7 +132,7 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: AttackTechniqueFactory.with_simulated_conversation( name="role_play_trivia_game", description="Frames the objective as a quiz answer by casting the adversarial chat as a trivia host.", - adversarial_chat_system_prompt_path=( + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "role_play" / "role_play_trivia_game.yaml" ), next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", @@ -142,7 +142,7 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: AttackTechniqueFactory.with_simulated_conversation( name="role_play_persuasion", description="Frames a persuasion-resistance test so the target drafts a persuasive spoken script.", - adversarial_chat_system_prompt_path=( + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "role_play" / "role_play_persuasion.yaml" ), next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", @@ -152,7 +152,7 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: AttackTechniqueFactory.with_simulated_conversation( name="role_play_persuasion_written", description="Frames a persuasion-resistance test so the target drafts written persuasive material.", - adversarial_chat_system_prompt_path=( + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "role_play" / "role_play_persuasion_written.yaml" ), next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "role_play_next_message.yaml", @@ -200,9 +200,9 @@ def get_technique_factories() -> list[AttackTechniqueFactory]: AttackTechniqueFactory.with_simulated_conversation( name="context_compliance", description="Injects a fabricated prior exchange so the target continues as if it already agreed.", - adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH - / "context_compliance" - / "context_compliance.yaml", + adversarial_chat_system_prompt=SeedPrompt.from_yaml_file( + EXECUTOR_RED_TEAM_PATH / "context_compliance" / "context_compliance.yaml" + ), simulated_target_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "context_compliance_target.yaml", final_user_message="yes.", num_turns=1, diff --git a/pyrit/setup/initializers/techniques/technique_initializer.py b/pyrit/setup/initializers/techniques/technique_initializer.py index 80370d6cca..9c2b3cdce5 100644 --- a/pyrit/setup/initializers/techniques/technique_initializer.py +++ b/pyrit/setup/initializers/techniques/technique_initializer.py @@ -17,6 +17,7 @@ not overwritten. """ +import asyncio import logging from enum import Enum @@ -113,7 +114,7 @@ async def initialize_async(self) -> None: if TechniqueInitializerTags.ALL.value in tags: tags = [TechniqueInitializerTags.CORE.value, TechniqueInitializerTags.EXTRA.value] - factories = build_technique_factories(groups=tags) + factories = await asyncio.to_thread(build_technique_factories, groups=tags) registry = AttackTechniqueRegistry.get_registry_singleton() registry.register_from_factories(factories) diff --git a/tests/unit/executor/attack/component/test_simulated_conversation.py b/tests/unit/executor/attack/component/test_simulated_conversation.py index e14909ca40..771160b0b9 100644 --- a/tests/unit/executor/attack/component/test_simulated_conversation.py +++ b/tests/unit/executor/attack/component/test_simulated_conversation.py @@ -12,6 +12,7 @@ from pyrit.executor.attack import AttackConverterConfig, RTASystemPromptPaths from pyrit.executor.attack.multi_turn.simulated_conversation import ( _generate_next_message_async, + _resolve_adversarial_chat_system_prompt_async, generate_simulated_conversation_async, ) from pyrit.models import ( @@ -150,6 +151,97 @@ async def test_raises_error_for_negative_turns( num_turns=-1, ) + async def test_inline_adversarial_prompt_is_forwarded_without_file_loading( + self, + mock_adversarial_chat: MagicMock, + mock_objective_scorer: MagicMock, + sample_conversation: list[Message], + ): + prompt = SeedPrompt( + value="Use {{ objective }}", + parameters=["objective"], + response_json_schema={ + "type": "object", + "properties": {"next_message": {"type": "string"}}, + }, + ) + with ( + patch("pyrit.executor.attack.multi_turn.simulated_conversation.RedTeamingAttack") as mock_attack_class, + patch("pyrit.executor.attack.multi_turn.simulated_conversation.CentralMemory") as mock_memory_class, + ): + mock_attack = MagicMock() + mock_attack.execute_async = AsyncMock( + return_value=AttackResult( + atomic_attack_identifier=ComponentIdentifier( + class_name="RedTeamingAttack", + class_module="pyrit.executor.attack", + ), + conversation_id=str(uuid.uuid4()), + objective="Test objective", + outcome=AttackOutcome.SUCCESS, + executed_turns=3, + ) + ) + mock_attack_class.return_value = mock_attack + mock_memory_class.get_memory_instance.return_value.get_conversation_messages.return_value = iter( + sample_conversation + ) + + await generate_simulated_conversation_async( + objective="Test objective", + adversarial_chat=mock_adversarial_chat, + objective_scorer=mock_objective_scorer, + adversarial_chat_system_prompt=prompt, + ) + + adversarial_config = mock_attack_class.call_args.kwargs["attack_adversarial_config"] + assert adversarial_config.system_prompt is prompt + + async def test_returns_inline_prompt(self): + prompt = SeedPrompt(value="inline") + + resolved = await _resolve_adversarial_chat_system_prompt_async( + adversarial_chat_system_prompt_path=None, + adversarial_chat_system_prompt=prompt, + ) + + assert resolved is prompt + + async def test_loads_legacy_path_off_event_loop(self, tmp_path): + prompt_path = tmp_path / "prompt.yaml" + resolved_prompt = SeedPrompt(value="resolved") + + with patch( + "pyrit.executor.attack.multi_turn.simulated_conversation.asyncio.to_thread", + new_callable=AsyncMock, + return_value=resolved_prompt, + ) as mock_to_thread: + resolved = await _resolve_adversarial_chat_system_prompt_async( + adversarial_chat_system_prompt_path=prompt_path, + adversarial_chat_system_prompt=None, + ) + + assert resolved is resolved_prompt + mock_to_thread.assert_awaited_once_with(SeedPrompt.from_yaml_file, prompt_path) + + @pytest.mark.parametrize( + ("path", "prompt"), + [ + (None, None), + ("prompt.yaml", SeedPrompt(value="inline")), + ], + ) + async def test_rejects_ambiguous_or_missing_adversarial_prompt_source( + self, + path: str | None, + prompt: SeedPrompt | None, + ) -> None: + with pytest.raises(ValueError, match="exactly one"): + await _resolve_adversarial_chat_system_prompt_async( + adversarial_chat_system_prompt_path=path, + adversarial_chat_system_prompt=prompt, + ) + async def test_uses_adversarial_chat_as_simulated_target( self, mock_adversarial_chat: MagicMock, diff --git a/tests/unit/executor/attack/core/test_attack_parameters.py b/tests/unit/executor/attack/core/test_attack_parameters.py index c7bd56811d..5a5424445c 100644 --- a/tests/unit/executor/attack/core/test_attack_parameters.py +++ b/tests/unit/executor/attack/core/test_attack_parameters.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import dataclasses +from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -129,7 +130,7 @@ def simulated_conversation_config(self) -> SeedSimulatedConversation: """Create a SeedSimulatedConversation config.""" return SeedSimulatedConversation( num_turns=3, - adversarial_chat_system_prompt_path="/path/to/adversarial.yaml", + adversarial_chat_system_prompt=SeedPrompt(value="Adversarial system prompt"), simulated_target_system_prompt_path="/path/to/target.yaml", ) @@ -208,19 +209,28 @@ async def test_raises_when_multi_sequence_prompts_overlap_with_simulated_conv( AttackSeedGroup(seeds=[seed_objective, prompt, simulated_conversation_config]) @patch("pyrit.executor.attack.multi_turn.simulated_conversation.generate_simulated_conversation_async") - async def test_generates_simulated_conversation( + async def test_resolves_legacy_path_before_generating_simulated_conversation( self, mock_generate: AsyncMock, - seed_group_with_simulated_conv: AttackSeedGroup, + seed_objective: SeedObjective, mock_adversarial_chat: MagicMock, mock_objective_scorer: MagicMock, mock_simulated_result: MagicMock, + tmp_path: Path, ) -> None: - """Test that simulated conversation is generated when config is present.""" + """Test that a legacy path is resolved before the generator is called.""" + prompt_path = tmp_path / "adversarial.yaml" + prompt_path.write_text("value: Resolved legacy prompt\ndata_type: text\n", encoding="utf-8") + config = SeedSimulatedConversation( + num_turns=3, + adversarial_chat_system_prompt_path=prompt_path, + simulated_target_system_prompt_path="/path/to/target.yaml", + ) + seed_group = AttackSeedGroup(seeds=[seed_objective, config]) mock_generate.return_value = mock_simulated_result await AttackParameters.from_seed_group_async( - seed_group=seed_group_with_simulated_conv, + seed_group=seed_group, adversarial_chat=mock_adversarial_chat, objective_scorer=mock_objective_scorer, ) @@ -231,6 +241,36 @@ async def test_generates_simulated_conversation( assert call_kwargs["adversarial_chat"] == mock_adversarial_chat assert call_kwargs["objective_scorer"] == mock_objective_scorer assert call_kwargs["num_turns"] == 3 + assert call_kwargs["adversarial_chat_system_prompt"].value == "Resolved legacy prompt" + assert "adversarial_chat_system_prompt_path" not in call_kwargs + + @patch("pyrit.executor.attack.multi_turn.simulated_conversation.generate_simulated_conversation_async") + async def test_forwards_inline_adversarial_prompt( + self, + mock_generate: AsyncMock, + seed_objective: SeedObjective, + mock_adversarial_chat: MagicMock, + mock_objective_scorer: MagicMock, + mock_simulated_result: list, + ) -> None: + prompt = SeedPrompt(value="Use {{ objective }}", parameters=["objective"]) + seed_group = AttackSeedGroup( + seeds=[ + seed_objective, + SeedSimulatedConversation(adversarial_chat_system_prompt=prompt), + ] + ) + mock_generate.return_value = mock_simulated_result + + await AttackParameters.from_seed_group_async( + seed_group=seed_group, + adversarial_chat=mock_adversarial_chat, + objective_scorer=mock_objective_scorer, + ) + + call_kwargs = mock_generate.call_args.kwargs + assert call_kwargs["adversarial_chat_system_prompt"] is prompt + assert "adversarial_chat_system_prompt_path" not in call_kwargs @patch("pyrit.executor.attack.multi_turn.simulated_conversation.generate_simulated_conversation_async") async def test_uses_generated_prepended_messages( diff --git a/tests/unit/memory/memory_interface/test_interface_seed_prompts.py b/tests/unit/memory/memory_interface/test_interface_seed_prompts.py index 37d256612d..c6d80d3654 100644 --- a/tests/unit/memory/memory_interface/test_interface_seed_prompts.py +++ b/tests/unit/memory/memory_interface/test_interface_seed_prompts.py @@ -11,7 +11,7 @@ from sqlalchemy.exc import SQLAlchemyError from pyrit.memory import MemoryInterface -from pyrit.models import MessagePiece, SeedDataset, SeedGroup, SeedObjective, SeedPrompt +from pyrit.models import MessagePiece, SeedDataset, SeedGroup, SeedObjective, SeedPrompt, SeedSimulatedConversation def assert_original_value_in_list(original_value: str, message_pieces: Sequence[MessagePiece]): @@ -130,6 +130,59 @@ async def test_get_seeds_with_dataset_name_filter(sqlite_instance: MemoryInterfa assert result[0].dataset_name == "dataset1" +async def test_legacy_path_backed_simulated_conversation_persistence_round_trip( + sqlite_instance: MemoryInterface, +) -> None: + seed = SeedSimulatedConversation( + adversarial_chat_system_prompt_path="/legacy/adversarial.yaml", + dataset_name="legacy_simulated", + ) + + await sqlite_instance.add_seeds_to_memory_async(seeds=[seed], added_by="test") + + recovered = sqlite_instance.get_seeds(seed_type="simulated_conversation") + assert len(recovered) == 1 + assert isinstance(recovered[0], SeedSimulatedConversation) + assert recovered[0].adversarial_chat_system_prompt_path == seed.adversarial_chat_system_prompt_path + assert recovered[0].adversarial_chat_system_prompt is None + assert recovered[0].value == seed.value + + +async def test_inline_prompt_simulated_conversation_persistence_round_trip( + sqlite_instance: MemoryInterface, +) -> None: + response_schema = { + "type": "object", + "properties": {"next_message": {"type": "string"}}, + } + seed = SeedSimulatedConversation( + adversarial_chat_system_prompt=SeedPrompt( + value="Use {{ objective }}", + data_type="text", + parameters=["objective"], + response_json_schema=response_schema, + metadata={"source_kind": "inline"}, + is_jinja_template=True, + ), + dataset_name="inline_simulated", + ) + + await sqlite_instance.add_seeds_to_memory_async(seeds=[seed], added_by="test") + + recovered = sqlite_instance.get_seeds(seed_type="simulated_conversation") + assert len(recovered) == 1 + assert isinstance(recovered[0], SeedSimulatedConversation) + assert recovered[0].adversarial_chat_system_prompt_path is None + recovered_prompt = recovered[0].adversarial_chat_system_prompt + assert recovered_prompt is not None + assert recovered_prompt.value == "Use {{ objective }}" + assert recovered_prompt.parameters == ["objective"] + assert recovered_prompt.response_json_schema == response_schema + assert recovered_prompt.metadata == {"source_kind": "inline"} + assert recovered_prompt.is_jinja_template is True + assert recovered[0].value == seed.value + + async def test_get_seeds_with_dataset_name_pattern_startswith(sqlite_instance: MemoryInterface): seed_prompts = [ SeedPrompt(value="prompt1", dataset_name="harm_category_1", data_type="text"), diff --git a/tests/unit/models/test_seed_simulated_conversation.py b/tests/unit/models/test_seed_simulated_conversation.py index c8239a9caa..3df081144c 100644 --- a/tests/unit/models/test_seed_simulated_conversation.py +++ b/tests/unit/models/test_seed_simulated_conversation.py @@ -9,6 +9,7 @@ import pytest from pyrit.models.seeds import ( + SeedPrompt, SeedSimulatedConversation, SimulatedTargetSystemPromptPaths, ) @@ -50,6 +51,25 @@ def test_init_with_minimal_parameters(self, tmp_path): # Default simulated_target_system_prompt_path is the compliant prompt assert conv.simulated_target_system_prompt_path == SimulatedTargetSystemPromptPaths.COMPLIANT.value + def test_init_with_inline_adversarial_prompt(self): + prompt = SeedPrompt(value="Use {{ objective }}", parameters=["objective"], data_type="text") + + conv = SeedSimulatedConversation(adversarial_chat_system_prompt=prompt) + + assert conv.adversarial_chat_system_prompt is prompt + assert conv.adversarial_chat_system_prompt_path is None + + def test_init_rejects_both_adversarial_prompt_sources(self, tmp_path): + with pytest.raises(ValueError, match="exactly one"): + SeedSimulatedConversation( + adversarial_chat_system_prompt_path=tmp_path / "prompt.yaml", + adversarial_chat_system_prompt=SeedPrompt(value="inline"), + ) + + def test_init_rejects_missing_adversarial_prompt_source(self): + with pytest.raises(ValueError, match="exactly one"): + SeedSimulatedConversation() + def test_init_default_num_turns(self, tmp_path): """Test that default num_turns is 3.""" adv_path = tmp_path / "adversarial.yaml" @@ -125,6 +145,57 @@ def test_init_value_is_deterministic(self, tmp_path): assert conv1.value == conv2.value + def test_inline_prompt_value_preserves_template_contract_and_is_deterministic(self): + prompt_kwargs = { + "value": "Use {{ objective }}", + "data_type": "text", + "parameters": ["objective"], + "response_json_schema": { + "type": "object", + "properties": {"next_message": {"type": "string"}}, + }, + "metadata": {"source_kind": "inline"}, + "is_jinja_template": True, + } + + conv1 = SeedSimulatedConversation(adversarial_chat_system_prompt=SeedPrompt(**prompt_kwargs)) + conv2 = SeedSimulatedConversation(adversarial_chat_system_prompt=SeedPrompt(**prompt_kwargs)) + + assert conv1.value == conv2.value + assert conv1.compute_hash() == conv2.compute_hash() + serialized_prompt = json.loads(conv1.value)["adversarial_chat_system_prompt"] + assert serialized_prompt["parameters"] == ["objective"] + assert serialized_prompt["response_json_schema"] == prompt_kwargs["response_json_schema"] + assert serialized_prompt["metadata"] == {"source_kind": "inline"} + assert serialized_prompt["is_jinja_template"] is True + assert "id" not in serialized_prompt + assert "date_added" not in serialized_prompt + + def test_inline_prompt_value_preserves_explicit_null_fields(self): + conv = SeedSimulatedConversation( + adversarial_chat_system_prompt=SeedPrompt( + value="inline", + parameters=None, + metadata=None, + ) + ) + + reconstructed = SeedSimulatedConversation(**json.loads(conv.value)) + prompt = reconstructed.adversarial_chat_system_prompt + assert prompt is not None + assert prompt.parameters is None + assert prompt.metadata is None + assert reconstructed.value == conv.value + + def test_inline_prompt_rejects_unordered_metadata(self): + with pytest.raises(ValueError, match="ordered JSON-compatible values"): + SeedSimulatedConversation( + adversarial_chat_system_prompt=SeedPrompt( + value="inline", + metadata={"tags": {"alpha", "beta"}}, + ) + ) + def test_init_default_sequence_is_zero(self, tmp_path): """Test that default sequence is 0.""" adv_path = tmp_path / "adversarial.yaml" @@ -217,11 +288,11 @@ def test_from_dict_default_num_turns(self, tmp_path): assert conv.num_turns == 3 - def test_from_dict_missing_adversarial_path_raises_error(self): - """Test that construction raises when adversarial path is missing (required field).""" + def test_from_dict_missing_adversarial_source_raises_error(self): + """Test that construction raises when both adversarial prompt sources are missing.""" data = {"num_turns": 3} - with pytest.raises(ValueError, match="adversarial_chat_system_prompt_path"): + with pytest.raises(ValueError, match="exactly one"): SeedSimulatedConversation.model_validate(data) diff --git a/tests/unit/scenario/core/test_attack_technique_factory.py b/tests/unit/scenario/core/test_attack_technique_factory.py index c5324d8d39..216bdabff2 100644 --- a/tests/unit/scenario/core/test_attack_technique_factory.py +++ b/tests/unit/scenario/core/test_attack_technique_factory.py @@ -4,6 +4,7 @@ """Tests for the AttackTechniqueFactory class.""" import typing +from pathlib import Path from unittest.mock import MagicMock, patch import pytest @@ -11,7 +12,13 @@ from pyrit.converter import Base64Converter, QRCodeConverter, ROT13Converter, TranslationConverter from pyrit.executor.attack.core.attack_config import AttackConverterConfig, AttackScoringConfig from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack -from pyrit.models import AttackTechniqueSeedGroup, ComponentIdentifier, Identifiable, SeedPrompt +from pyrit.models import ( + AttackTechniqueSeedGroup, + ComponentIdentifier, + Identifiable, + SeedPrompt, + SeedSimulatedConversation, +) from pyrit.prompt_normalizer import ConverterConfiguration from pyrit.prompt_target import PromptTarget from pyrit.scenario.core.attack_technique import AttackTechnique @@ -92,6 +99,88 @@ def test_with_simulated_conversation_forwards_description(self): ) assert factory.description == "Staged as a journalist interview." + assert factory.seed_technique is not None + config = factory.seed_technique.simulated_conversation_config + assert isinstance(config, SeedSimulatedConversation) + assert config.adversarial_chat_system_prompt is not None + assert config.adversarial_chat_system_prompt_path is None + + def test_with_simulated_conversation_rejects_path_on_preferred_prompt_source(self): + with pytest.raises(TypeError, match="must be a SeedPrompt"): + AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt=Path("prompt.yaml"), # type: ignore[arg-type] + ) + + def test_with_simulated_conversation_accepts_legacy_path_alias_without_warning(self, tmp_path, recwarn): + prompt_path = tmp_path / "prompt.yaml" + prompt_path.write_text("value: Legacy prompt\ndata_type: text\n", encoding="utf-8") + + factory = AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt_path=str(prompt_path), + ) + + assert factory.seed_technique is not None + config = factory.seed_technique.simulated_conversation_config + assert isinstance(config, SeedSimulatedConversation) + assert config.adversarial_chat_system_prompt_path is None + assert config.adversarial_chat_system_prompt is not None + assert config.adversarial_chat_system_prompt.value == "Legacy prompt" + assert not [warning for warning in recwarn if issubclass(warning.category, DeprecationWarning)] + + def test_with_simulated_conversation_rejects_string_on_preferred_prompt_source(self): + with pytest.raises(TypeError, match="must be a SeedPrompt"): + AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt="prompt.yaml", # type: ignore[arg-type] + ) + + def test_with_simulated_conversation_accepts_inline_prompt(self): + prompt = SeedPrompt( + value="Use {{ objective }}", + data_type="text", + parameters=["objective"], + metadata={"source_kind": "inline"}, + ) + + factory = AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt=prompt, + ) + + assert factory.seed_technique is not None + config = factory.seed_technique.simulated_conversation_config + assert isinstance(config, SeedSimulatedConversation) + assert config.adversarial_chat_system_prompt is prompt + assert config.adversarial_chat_system_prompt_path is None + + def test_with_simulated_conversation_rejects_both_prompt_sources(self, tmp_path): + with pytest.raises(ValueError, match="only one"): + AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt_path=tmp_path / "prompt.yaml", + adversarial_chat_system_prompt=SeedPrompt(value="inline"), + ) + + def test_with_simulated_conversation_identifier_is_deterministic_for_inline_prompt(self): + prompt_kwargs = { + "value": "Use {{ objective }}", + "data_type": "text", + "parameters": ["objective"], + "metadata": {"source_kind": "inline"}, + } + + first = AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt=SeedPrompt(**prompt_kwargs), + ) + second = AttackTechniqueFactory.with_simulated_conversation( + name="test", + adversarial_chat_system_prompt=SeedPrompt(**prompt_kwargs), + ) + + assert first.get_identifier().hash == second.get_identifier().hash def test_description_does_not_affect_identifier(self): """Description is decorative metadata and must not change the behavioral identity hash.""" diff --git a/tests/unit/scenario/core/test_simulated_conversation_prompt.py b/tests/unit/scenario/core/test_simulated_conversation_prompt.py new file mode 100644 index 0000000000..61fdbd46e0 --- /dev/null +++ b/tests/unit/scenario/core/test_simulated_conversation_prompt.py @@ -0,0 +1,70 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Tests for simulated-conversation adversarial prompt-source normalization.""" + +from pathlib import Path + +import pytest + +from pyrit.models import SeedPrompt +from pyrit.scenario.core import resolve_simulated_conversation_adversarial_prompt + + +def test_resolve_simulated_conversation_adversarial_prompt_returns_inline_prompt() -> None: + prompt = SeedPrompt(value="Use {{ objective }}", parameters=["objective"], data_type="text") + + resolved = resolve_simulated_conversation_adversarial_prompt( + adversarial_chat_system_prompt=prompt, + ) + + assert resolved is prompt + + +def test_resolve_simulated_conversation_adversarial_prompt_rejects_path_on_preferred_source() -> None: + with pytest.raises(TypeError, match="must be a SeedPrompt"): + resolve_simulated_conversation_adversarial_prompt( + adversarial_chat_system_prompt=Path("prompt.yaml"), # type: ignore[arg-type] + ) + + +def test_resolve_simulated_conversation_adversarial_prompt_accepts_legacy_path_alias(tmp_path: Path) -> None: + prompt_path = tmp_path / "prompt.yaml" + prompt_path.write_text("value: Legacy prompt\ndata_type: text\n", encoding="utf-8") + + resolved = resolve_simulated_conversation_adversarial_prompt( + adversarial_chat_system_prompt_path=str(prompt_path), + ) + + assert resolved.value == "Legacy prompt" + + +def test_resolve_simulated_conversation_adversarial_prompt_rejects_string_on_preferred_source() -> None: + with pytest.raises(TypeError, match="must be a SeedPrompt"): + resolve_simulated_conversation_adversarial_prompt( + adversarial_chat_system_prompt="prompt.yaml", # type: ignore[arg-type] + ) + + +def test_resolve_simulated_conversation_adversarial_prompt_uses_default_path(tmp_path: Path) -> None: + default_path = tmp_path / "default.yaml" + default_path.write_text("value: Default prompt\ndata_type: text\n", encoding="utf-8") + + resolved = resolve_simulated_conversation_adversarial_prompt( + default_system_prompt_path=default_path, + ) + + assert resolved.value == "Default prompt" + + +def test_resolve_simulated_conversation_adversarial_prompt_rejects_both_sources(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="only one"): + resolve_simulated_conversation_adversarial_prompt( + adversarial_chat_system_prompt_path=tmp_path / "prompt.yaml", + adversarial_chat_system_prompt=SeedPrompt(value="inline"), + ) + + +def test_resolve_simulated_conversation_adversarial_prompt_rejects_missing_source() -> None: + with pytest.raises(ValueError, match="Set one of"): + resolve_simulated_conversation_adversarial_prompt() diff --git a/tests/unit/setup/test_technique_initializer.py b/tests/unit/setup/test_technique_initializer.py index 66f791a12f..f16b61a386 100644 --- a/tests/unit/setup/test_technique_initializer.py +++ b/tests/unit/setup/test_technique_initializer.py @@ -4,7 +4,7 @@ """Tests for TechniqueInitializer and the technique group catalogs.""" from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -260,11 +260,15 @@ def test_seed_technique_num_turns_matches_canonical_default(self): assert sim is not None assert sim.num_turns == 3 - def test_seed_technique_yaml_path_resolves_to_existing_file(self): + def test_seed_technique_contains_resolved_inline_prompt(self): for f in self._persona_factories(): sim = f.seed_technique.simulated_conversation_config assert sim is not None - assert sim.adversarial_chat_system_prompt_path.exists() + assert sim.adversarial_chat_system_prompt_path is None + prompt = sim.adversarial_chat_system_prompt + assert prompt is not None + assert prompt.name == f.name + assert prompt.parameters == ["objective", "max_turns"] class TestPersonaCrescendoYamls: @@ -343,12 +347,15 @@ def test_final_user_message_is_fixed_affirmation(self): assert yes_prompt.role == "user" assert yes_prompt.sequence == 2 - def test_adversarial_yaml_resolves_to_existing_file(self): + def test_adversarial_prompt_is_resolved_inline(self): factory = self._context_compliance_factory() sim = factory.seed_technique.simulated_conversation_config assert sim is not None - assert sim.adversarial_chat_system_prompt_path.name == "context_compliance.yaml" - assert sim.adversarial_chat_system_prompt_path.exists() + assert sim.adversarial_chat_system_prompt_path is None + prompt = sim.adversarial_chat_system_prompt + assert prompt is not None + assert prompt.name == "context_compliance" + assert prompt.parameters == ["objective", "max_turns"] def test_tagged_core_single_turn_light(self): factory = self._context_compliance_factory() @@ -417,11 +424,15 @@ def test_seed_technique_num_turns_matches_role_play_default(self): assert sim is not None assert sim.num_turns == 2 - def test_seed_technique_yaml_path_resolves_to_existing_file(self): + def test_seed_technique_contains_resolved_inline_prompt(self): for f in self._role_play_factories(): sim = f.seed_technique.simulated_conversation_config assert sim is not None - assert sim.adversarial_chat_system_prompt_path.exists() + assert sim.adversarial_chat_system_prompt_path is None + prompt = sim.adversarial_chat_system_prompt + assert prompt is not None + assert prompt.name == f.name + assert prompt.parameters == ["objective", "max_turns"] def test_all_use_role_play_next_message_prompt(self): for f in self._role_play_factories(): @@ -473,6 +484,18 @@ def test_yaml_has_no_em_or_en_dashes(self, technique_name): class TestTechniqueInitializerRegistration: """Tests that initialize_async wires factories into the registry per the tags param.""" + async def test_builds_factories_off_event_loop(self): + init = TechniqueInitializer() + + with patch( + "pyrit.setup.initializers.techniques.technique_initializer.asyncio.to_thread", + new_callable=AsyncMock, + return_value=[], + ) as mock_to_thread: + await init.initialize_async() + + mock_to_thread.assert_awaited_once_with(build_technique_factories, groups=["core"]) + async def test_default_registers_only_core(self, mock_adversarial_target): init = TechniqueInitializer() await init.initialize_async()