From 2060cdece107fb203cd9560a1575f51a01ab4b33 Mon Sep 17 00:00:00 2001 From: YusefSyed <211442445+YusefSyed@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:31:27 -0400 Subject: [PATCH] FEAT: Add Dart, Perl, and Raku package hallucination techniques --- doc/scanner/garak.ipynb | 6 +-- doc/scanner/garak.py | 6 +-- .../scenarios/garak/package_hallucination.py | 8 ++- .../garak/test_package_hallucination.py | 49 ++++++++++++++----- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 9bdd8b08fb..d2b9f18067 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -308,12 +308,12 @@ "pyrit_scan garak.package_hallucination --target openai_chat\n", "\n", "# Select another supported language.\n", - "pyrit_scan garak.package_hallucination --target openai_chat --techniques python\n", + "pyrit_scan garak.package_hallucination --target openai_chat --techniques dart\n", "```\n", "\n", - "**Available techniques** (4 languages): Python, JavaScript, Ruby, Rust.\n", + "**Available techniques** (7 languages): Python, JavaScript, Ruby, Rust, Dart, Perl, Raku.\n", "\n", - "**Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all four languages.\n", + "**Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all seven languages.\n", "\n", "> **Note:** Rust and its crates.io registry are the default because this registry is much smaller.\n", "> If you select another language, PyRIT downloads its registry on demand. The raw package names\n", diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 93b87caa4a..35a439e562 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -237,12 +237,12 @@ # pyrit_scan garak.package_hallucination --target openai_chat # # # Select another supported language. -# pyrit_scan garak.package_hallucination --target openai_chat --techniques python +# pyrit_scan garak.package_hallucination --target openai_chat --techniques dart # ``` # -# **Available techniques** (4 languages): Python, JavaScript, Ruby, Rust. +# **Available techniques** (7 languages): Python, JavaScript, Ruby, Rust, Dart, Perl, Raku. # -# **Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all four languages. +# **Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all seven languages. # # > **Note:** Rust and its crates.io registry are the default because this registry is much smaller. # > If you select another language, PyRIT downloads its registry on demand. The raw package names diff --git a/pyrit/scenario/scenarios/garak/package_hallucination.py b/pyrit/scenario/scenarios/garak/package_hallucination.py index 32c7f4bf72..858d005f20 100644 --- a/pyrit/scenario/scenarios/garak/package_hallucination.py +++ b/pyrit/scenario/scenarios/garak/package_hallucination.py @@ -71,6 +71,9 @@ class _LanguageSpec: language_name="Ruby", dataset_name="garak_rubygems_packages", ecosystem=PackageEcosystem.RUBY ), "rust": _LanguageSpec(language_name="Rust", dataset_name="garak_crates_packages", ecosystem=PackageEcosystem.RUST), + "dart": _LanguageSpec(language_name="Dart", dataset_name="garak_dart_packages", ecosystem=PackageEcosystem.DART), + "perl": _LanguageSpec(language_name="Perl", dataset_name="garak_perl_packages", ecosystem=PackageEcosystem.PERL), + "raku": _LanguageSpec(language_name="Raku", dataset_name="garak_raku_packages", ecosystem=PackageEcosystem.RAKU), } @@ -106,6 +109,9 @@ class PackageHallucinationTechnique(ScenarioTechnique): JavaScript = ("javascript", set()) Ruby = ("ruby", set()) Rust = ("rust", {"default"}) + Dart = ("dart", set()) + Perl = ("perl", set()) + Raku = ("raku", set()) @classmethod def get_aggregate_tags(cls) -> set[str]: @@ -135,7 +141,7 @@ class PackageHallucination(Scenario): Reference: [@derczynski2024garak] """ - VERSION: int = 2 + VERSION: int = 3 # The plain code request is not an adversarial baseline to compare against, so no baseline. BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden diff --git a/tests/unit/scenario/garak/test_package_hallucination.py b/tests/unit/scenario/garak/test_package_hallucination.py index 0b40253877..cff59f0e8e 100644 --- a/tests/unit/scenario/garak/test_package_hallucination.py +++ b/tests/unit/scenario/garak/test_package_hallucination.py @@ -42,6 +42,9 @@ def fake_registry_memory(): "garak_npm_packages": ["react", "left-pad"], "garak_rubygems_packages": ["rails", "rspec"], "garak_crates_packages": ["serde", "rand"], + "garak_dart_packages": ["http", "provider"], + "garak_perl_packages": ["JSON::MaybeXS", "Path::Tiny"], + "garak_raku_packages": ["JSON::Fast", "Cro::HTTP"], "garak_package_hallucination_stubs": [ "Generate some code to ", "How do I in ?", @@ -64,7 +67,7 @@ class TestPackageHallucinationInitialization: def test_no_arg_instantiation(self): scenario = PackageHallucination() assert scenario.name == "PackageHallucination" - assert scenario.VERSION == 2 + assert scenario.VERSION == 3 def test_default_objective_scorer_is_package_hallucination_scorer(self): scenario = PackageHallucination() @@ -82,6 +85,9 @@ def test_required_datasets(self): "garak_npm_packages", "garak_rubygems_packages", "garak_crates_packages", + "garak_dart_packages", + "garak_perl_packages", + "garak_raku_packages", ] def test_default_dataset_config_declares_rust_registry_and_corpus(self): @@ -103,11 +109,11 @@ def test_default_technique_is_default(self): class TestPackageHallucinationTechnique: def test_concrete_strategy_values(self): values = {s.value for s in PackageHallucinationTechnique} - assert values == {"all", "default", "python", "javascript", "ruby", "rust"} + assert values == {"all", "default", "python", "javascript", "ruby", "rust", "dart", "perl", "raku"} - def test_all_expands_to_four_languages(self): + def test_all_expands_to_seven_languages(self): expanded = {s.value for s in PackageHallucinationTechnique.expand({PackageHallucinationTechnique.ALL})} - assert expanded == {"python", "javascript", "ruby", "rust"} + assert expanded == {"python", "javascript", "ruby", "rust", "dart", "perl", "raku"} def test_default_expands_to_rust(self): expanded = {s.value for s in PackageHallucinationTechnique.expand({PackageHallucinationTechnique.DEFAULT})} @@ -170,6 +176,9 @@ async def test_include_baseline_true_raises(self, mock_objective_target, fake_re (PackageHallucinationTechnique.JavaScript, PackageEcosystem.JAVASCRIPT), (PackageHallucinationTechnique.Ruby, PackageEcosystem.RUBY), (PackageHallucinationTechnique.Rust, PackageEcosystem.RUST), + (PackageHallucinationTechnique.Dart, PackageEcosystem.DART), + (PackageHallucinationTechnique.Perl, PackageEcosystem.PERL), + (PackageHallucinationTechnique.Raku, PackageEcosystem.RAKU), ], ) async def test_per_language_scorer_ecosystem( @@ -183,22 +192,38 @@ async def test_per_language_scorer_ecosystem( assert isinstance(scorer, PackageHallucinationScorer) assert scorer._ecosystem is ecosystem - async def test_non_default_registry_is_fetched_lazily(self, mock_objective_target, fake_registry_memory): - fake_registry_memory.packages_by_dataset.pop("garak_pypi_packages") + @pytest.mark.parametrize( + ("technique", "dataset_name", "packages", "ecosystem"), + [ + (PackageHallucinationTechnique.Python, "garak_pypi_packages", ["requests"], PackageEcosystem.PYTHON), + ( + PackageHallucinationTechnique.JavaScript, + "garak_npm_packages", + ["react"], + PackageEcosystem.JAVASCRIPT, + ), + (PackageHallucinationTechnique.Ruby, "garak_rubygems_packages", ["rails"], PackageEcosystem.RUBY), + (PackageHallucinationTechnique.Dart, "garak_dart_packages", ["http"], PackageEcosystem.DART), + (PackageHallucinationTechnique.Perl, "garak_perl_packages", ["Path::Tiny"], PackageEcosystem.PERL), + (PackageHallucinationTechnique.Raku, "garak_raku_packages", ["JSON::Fast"], PackageEcosystem.RAKU), + ], + ) + async def test_non_default_registry_is_fetched_lazily( + self, mock_objective_target, fake_registry_memory, technique, dataset_name, packages, ecosystem + ): + fake_registry_memory.packages_by_dataset.pop(dataset_name) async def _fetch_dataset_async(*, dataset_name: str) -> None: - fake_registry_memory.packages_by_dataset[dataset_name] = ["requests", "flask"] + fake_registry_memory.packages_by_dataset[dataset_name] = packages fetch_mock = AsyncMock(side_effect=_fetch_dataset_async) with patch.object(DatasetConfiguration, "_fetch_dataset_async", new=fetch_mock): scenario = PackageHallucination() - await self._initialize( - scenario, mock_objective_target, [PackageHallucinationTechnique.Python], fake_registry_memory - ) + await self._initialize(scenario, mock_objective_target, [technique], fake_registry_memory) - fetch_mock.assert_awaited_once_with(dataset_name="garak_pypi_packages") + fetch_mock.assert_awaited_once_with(dataset_name=dataset_name) scorer = scenario._atomic_attacks[0].attack_technique.attack._objective_scorer - assert scorer._ecosystem is PackageEcosystem.PYTHON + assert scorer._ecosystem is ecosystem async def test_seed_groups_pair_objective_and_prompt(self, mock_objective_target, fake_registry_memory): scenario = PackageHallucination()