Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/code/datasets/2_seed_programming.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@
" role=\"system\",\n",
" ),\n",
" SeedSimulatedConversation(\n",
" adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH / \"naive_crescendo.yaml\",\n",
" adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(EXECUTOR_RED_TEAM_PATH / \"naive_crescendo.yaml\"),\n",
" sequence=1,\n",
" num_turns=4,\n",
" next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / \"direct_next_message.yaml\",\n",
Expand Down
2 changes: 1 addition & 1 deletion doc/code/datasets/2_seed_programming.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@
role="system",
),
SeedSimulatedConversation(
adversarial_chat_system_prompt_path=EXECUTOR_RED_TEAM_PATH / "naive_crescendo.yaml",
adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(EXECUTOR_RED_TEAM_PATH / "naive_crescendo.yaml"),
sequence=1,
num_turns=4,
next_message_system_prompt_path=EXECUTOR_SIMULATED_TARGET_PATH / "direct_next_message.yaml",
Expand Down
13 changes: 7 additions & 6 deletions doc/code/datasets/5_simulated_conversation.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"\n",
"## Generating a Simulated Conversation\n",
"\n",
"The function takes an objective, an adversarial chat model, a scorer, and a system prompt path.\n",
"The function takes an objective, an adversarial chat model, a scorer, and a system `SeedPrompt`.\n",
"It runs a `RedTeamingAttack` internally with the adversarial LLM playing both attacker and target\n",
"roles."
]
Expand Down Expand Up @@ -60,11 +60,9 @@
}
],
"source": [
"from pathlib import Path\n",
"\n",
"from pyrit.common.path import EXECUTOR_SEED_PROMPT_PATH\n",
"from pyrit.executor.attack import generate_simulated_conversation_async\n",
"from pyrit.models import SeedGroup\n",
"from pyrit.models import SeedGroup, SeedPrompt\n",
"from pyrit.output import output_attack_async\n",
"from pyrit.prompt_target import OpenAIChatTarget\n",
"from pyrit.score import SelfAskRefusalScorer\n",
Expand All @@ -83,7 +81,9 @@
" adversarial_chat=adversarial_chat,\n",
" objective_scorer=objective_scorer,\n",
" num_turns=3,\n",
" adversarial_chat_system_prompt_path=Path(EXECUTOR_SEED_PROMPT_PATH) / \"red_teaming\" / \"naive_crescendo.yaml\",\n",
" adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(\n",
" EXECUTOR_SEED_PROMPT_PATH / \"red_teaming\" / \"naive_crescendo.yaml\"\n",
" ),\n",
")\n",
"\n",
"print(f\"Generated {len(simulated_conversation_prompts)} messages\")"
Expand Down Expand Up @@ -517,7 +517,8 @@
"| `adversarial_chat` | `PromptTarget` | The LLM that generates attack prompts (also plays the simulated target). Must declare `supports_multi_turn=True` and `supports_editable_history=True`. |\n",
"| `objective_scorer` | `TrueFalseScorer` | Evaluates whether the final turn achieved the objective |\n",
"| `num_turns` | `int` | Number of conversation turns to generate (default: 3) |\n",
"| `adversarial_chat_system_prompt_path` | `str \\| Path` | System prompt for the adversarial chat role |\n",
"| `adversarial_chat_system_prompt` | `SeedPrompt` | System prompt for the adversarial chat role |\n",
"| `adversarial_chat_system_prompt_path` | `str \\| Path \\| None` | Compatibility adapter for legacy path-based callers |\n",
"| `simulated_target_system_prompt_path` | `str \\| Path \\| None` | Optional system prompt for the simulated target role |\n",
"| `next_message_system_prompt_path` | `str \\| Path \\| None` | Optional path to generate a final user message that elicits objective fulfillment |\n",
"| `attack_converter_config` | `AttackConverterConfig \\| None` | Optional converter configuration for the attack |\n",
Expand Down
13 changes: 7 additions & 6 deletions doc/code/datasets/5_simulated_conversation.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,14 @@
#
# ## Generating a Simulated Conversation
#
# The function takes an objective, an adversarial chat model, a scorer, and a system prompt path.
# The function takes an objective, an adversarial chat model, a scorer, and a system `SeedPrompt`.
# It runs a `RedTeamingAttack` internally with the adversarial LLM playing both attacker and target
# roles.

# %%
from pathlib import Path

from pyrit.common.path import EXECUTOR_SEED_PROMPT_PATH
from pyrit.executor.attack import generate_simulated_conversation_async
from pyrit.models import SeedGroup
from pyrit.models import SeedGroup, SeedPrompt
from pyrit.output import output_attack_async
from pyrit.prompt_target import OpenAIChatTarget
from pyrit.score import SelfAskRefusalScorer
Expand All @@ -56,7 +54,9 @@
adversarial_chat=adversarial_chat,
objective_scorer=objective_scorer,
num_turns=3,
adversarial_chat_system_prompt_path=Path(EXECUTOR_SEED_PROMPT_PATH) / "red_teaming" / "naive_crescendo.yaml",
adversarial_chat_system_prompt=SeedPrompt.from_yaml_file(
EXECUTOR_SEED_PROMPT_PATH / "red_teaming" / "naive_crescendo.yaml"
),
)

print(f"Generated {len(simulated_conversation_prompts)} messages")
Expand Down Expand Up @@ -126,7 +126,8 @@
# | `adversarial_chat` | `PromptTarget` | The LLM that generates attack prompts (also plays the simulated target). Must declare `supports_multi_turn=True` and `supports_editable_history=True`. |
# | `objective_scorer` | `TrueFalseScorer` | Evaluates whether the final turn achieved the objective |
# | `num_turns` | `int` | Number of conversation turns to generate (default: 3) |
# | `adversarial_chat_system_prompt_path` | `str \| Path` | System prompt for the adversarial chat role |
# | `adversarial_chat_system_prompt` | `SeedPrompt` | System prompt for the adversarial chat role |
# | `adversarial_chat_system_prompt_path` | `str \| Path \| None` | Compatibility adapter for legacy path-based callers |
# | `simulated_target_system_prompt_path` | `str \| Path \| None` | Optional system prompt for the simulated target role |
# | `next_message_system_prompt_path` | `str \| Path \| None` | Optional path to generate a final user message that elicits objective fulfillment |
# | `attack_converter_config` | `AttackConverterConfig \| None` | Optional converter configuration for the attack |
Expand Down
8 changes: 7 additions & 1 deletion doc/code/scenarios/0_attack_techniques.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@
"\n",
"The objective is *not* part of the technique — it stays separate and is supplied by the dataset at\n",
"run time. You rarely build a technique by hand; instead you register a **factory** and let scenarios\n",
"construct techniques on demand with the scenario's own objective target and scorer."
"construct techniques on demand with the scenario's own objective target and scorer.\n",
"\n",
"`adversarial_chat_system_prompt` accepts a `SeedPrompt`, so created techniques carry portable prompt\n",
"content rather than a runtime file dependency. Callers that own YAML load it with\n",
"`SeedPrompt.from_yaml_file(...)` at setup time. The legacy `adversarial_chat_system_prompt_path` name\n",
"remains a compatibility adapter, and if neither parameter is supplied, the existing\n",
"`red_teaming/{technique_name}.yaml` convention remains the default."
]
},
{
Expand Down
6 changes: 6 additions & 0 deletions doc/code/scenarios/0_attack_techniques.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@
# The objective is *not* part of the technique — it stays separate and is supplied by the dataset at
# run time. You rarely build a technique by hand; instead you register a **factory** and let scenarios
# construct techniques on demand with the scenario's own objective target and scorer.
#
# `adversarial_chat_system_prompt` accepts a `SeedPrompt`, so created techniques carry portable prompt
# content rather than a runtime file dependency. Callers that own YAML load it with
# `SeedPrompt.from_yaml_file(...)` at setup time. The legacy `adversarial_chat_system_prompt_path` name
# remains a compatibility adapter, and if neither parameter is supplied, the existing
# `red_teaming/{technique_name}.yaml` convention remains the default.

# %% [markdown]
# ## Where techniques come from: initializers
Expand Down
8 changes: 7 additions & 1 deletion pyrit/executor/attack/core/attack_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ async def from_seed_group_async(
"""
# Import here to avoid circular imports
from pyrit.executor.attack.multi_turn.simulated_conversation import (
_resolve_adversarial_chat_system_prompt_async,
generate_simulated_conversation_async,
)

Expand Down Expand Up @@ -159,14 +160,19 @@ async def from_seed_group_async(
if objective_scorer is None:
raise ValueError("objective_scorer is required when seed_group has a simulated conversation config")

adversarial_chat_system_prompt = await _resolve_adversarial_chat_system_prompt_async(
adversarial_chat_system_prompt_path=simulated_conversation_config.adversarial_chat_system_prompt_path,
adversarial_chat_system_prompt=simulated_conversation_config.adversarial_chat_system_prompt,
)

# Generate the simulated conversation - returns list[SeedPrompt]
simulated_prompts = await generate_simulated_conversation_async(
objective=seed_group.objective.value,
adversarial_chat=adversarial_chat,
objective_scorer=objective_scorer,
num_turns=simulated_conversation_config.num_turns,
starting_sequence=simulated_conversation_config.sequence,
adversarial_chat_system_prompt_path=simulated_conversation_config.adversarial_chat_system_prompt_path,
adversarial_chat_system_prompt=adversarial_chat_system_prompt,
simulated_target_system_prompt_path=simulated_conversation_config.simulated_target_system_prompt_path,
next_message_system_prompt_path=simulated_conversation_config.next_message_system_prompt_path,
)
Expand Down
47 changes: 40 additions & 7 deletions pyrit/executor/attack/multi_turn/simulated_conversation.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from __future__ import annotations

import asyncio
import logging
from typing import TYPE_CHECKING

Expand Down Expand Up @@ -43,7 +44,8 @@ async def generate_simulated_conversation_async(
objective_scorer: TrueFalseScorer,
num_turns: int = 3,
starting_sequence: int = 0,
adversarial_chat_system_prompt_path: str | Path,
adversarial_chat_system_prompt_path: str | Path | None = None,
adversarial_chat_system_prompt: SeedPrompt | None = None,
simulated_target_system_prompt_path: str | Path | None = None,
next_message_system_prompt_path: str | Path | None = None,
attack_converter_config: AttackConverterConfig | None = None,
Expand All @@ -70,7 +72,10 @@ async def generate_simulated_conversation_async(
num_turns: Number of conversation turns to generate. Defaults to 3.
starting_sequence: The starting sequence number for the generated SeedPrompts.
Each message gets an incrementing sequence number. Defaults to 0.
adversarial_chat_system_prompt_path: Path to the system prompt for the adversarial chat.
adversarial_chat_system_prompt_path: Compatibility-only path adapter for the adversarial chat
system prompt. New callers should load the YAML at their composition boundary and pass
``adversarial_chat_system_prompt``.
adversarial_chat_system_prompt: Canonical inline system prompt for the adversarial chat.
simulated_target_system_prompt_path: Path to the system prompt for the simulated target.
If None, no system prompt is used for the simulated target.
next_message_system_prompt_path: Optional path to a system prompt for generating
Expand All @@ -89,7 +94,7 @@ async def generate_simulated_conversation_async(
generated to elicit the objective fulfillment.

Raises:
ValueError: If num_turns is not a positive integer.
ValueError: If num_turns is not positive or the adversarial prompt source is ambiguous or missing.
"""
# Use the same LLM for both adversarial chat and simulated target
# They get different system prompts to play different roles
Expand All @@ -105,10 +110,9 @@ async def generate_simulated_conversation_async(
simulated_target_system_prompt_path=simulated_target_system_prompt_path,
)

# Create adversarial config for the simulation. Load the optional path into a SeedPrompt so the
# resolved prompt is stored directly on the configuration.
adversarial_system_prompt = (
SeedPrompt.from_yaml_file(adversarial_chat_system_prompt_path) if adversarial_chat_system_prompt_path else None
adversarial_system_prompt = await _resolve_adversarial_chat_system_prompt_async(
adversarial_chat_system_prompt_path=adversarial_chat_system_prompt_path,
adversarial_chat_system_prompt=adversarial_chat_system_prompt,
)
adversarial_config = AttackAdversarialConfig(
target=adversarial_chat,
Expand Down Expand Up @@ -176,6 +180,35 @@ async def generate_simulated_conversation_async(
return seed_prompts


async def _resolve_adversarial_chat_system_prompt_async(
*,
adversarial_chat_system_prompt_path: str | Path | None,
adversarial_chat_system_prompt: SeedPrompt | None,
) -> SeedPrompt:
"""
Adapt a legacy path-backed prompt to the canonical inline execution input.

Args:
adversarial_chat_system_prompt_path: Legacy YAML prompt path.
adversarial_chat_system_prompt: Canonical inline prompt.

Returns:
The resolved adversarial chat system prompt.

Raises:
ValueError: If both or neither prompt sources are provided.
"""
has_prompt_path = adversarial_chat_system_prompt_path is not None
has_inline_prompt = adversarial_chat_system_prompt is not None
if has_prompt_path == has_inline_prompt:
raise ValueError("Set exactly one of adversarial_chat_system_prompt_path or adversarial_chat_system_prompt.")
if adversarial_chat_system_prompt is not None:
return adversarial_chat_system_prompt

assert adversarial_chat_system_prompt_path is not None
return await asyncio.to_thread(SeedPrompt.from_yaml_file, adversarial_chat_system_prompt_path)


async def _generate_next_message_async(
*,
objective: str,
Expand Down
2 changes: 2 additions & 0 deletions pyrit/memory/memory_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -1497,8 +1497,10 @@ def get_seed(self) -> Seed:
num_turns=config.get("num_turns", 3),
sequence=config.get("sequence", 0),
adversarial_chat_system_prompt_path=config.get("adversarial_chat_system_prompt_path"),
adversarial_chat_system_prompt=config.get("adversarial_chat_system_prompt"),
simulated_target_system_prompt_path=config.get("simulated_target_system_prompt_path"),
next_message_system_prompt_path=config.get("next_message_system_prompt_path"),
pyrit_version=config.get("pyrit_version"),
)
return SeedPrompt(
id=self.id,
Expand Down
Loading
Loading