FEAT: Add Dart, Perl, and Raku package hallucination techniques

doc/scanner/garak.ipynb

@@ -308,12 +308,12 @@
     "pyrit_scan garak.package_hallucination --target openai_chat\n",
     "\n",
     "# Select another supported language.\n",
-    "pyrit_scan garak.package_hallucination --target openai_chat --techniques python\n",
+    "pyrit_scan garak.package_hallucination --target openai_chat --techniques dart\n",
     "```\n",
     "\n",
-    "**Available techniques** (4 languages): Python, JavaScript, Ruby, Rust.\n",
+    "**Available techniques** (7 languages): Python, JavaScript, Ruby, Rust, Dart, Perl, Raku.\n",
     "\n",
-    "**Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all four languages.\n",
+    "**Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all seven languages.\n",
     "\n",
     "> **Note:** Rust and its crates.io registry are the default because this registry is much smaller.\n",
     "> If you select another language, PyRIT downloads its registry on demand. The raw package names\n",

doc/scanner/garak.py

@@ -237,12 +237,12 @@
 # pyrit_scan garak.package_hallucination --target openai_chat
 #
 # # Select another supported language.
-# pyrit_scan garak.package_hallucination --target openai_chat --techniques python
+# pyrit_scan garak.package_hallucination --target openai_chat --techniques dart
 # ```
 #
-# **Available techniques** (4 languages): Python, JavaScript, Ruby, Rust.
+# **Available techniques** (7 languages): Python, JavaScript, Ruby, Rust, Dart, Perl, Raku.
 #
-# **Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all four languages.
+# **Aggregate techniques:** `DEFAULT` runs Rust. `ALL` runs all seven languages.
 #
 # > **Note:** Rust and its crates.io registry are the default because this registry is much smaller.
 # > If you select another language, PyRIT downloads its registry on demand. The raw package names

pyrit/scenario/scenarios/garak/package_hallucination.py

@@ -71,6 +71,9 @@ class _LanguageSpec:
         language_name="Ruby", dataset_name="garak_rubygems_packages", ecosystem=PackageEcosystem.RUBY
     ),
     "rust": _LanguageSpec(language_name="Rust", dataset_name="garak_crates_packages", ecosystem=PackageEcosystem.RUST),
+    "dart": _LanguageSpec(language_name="Dart", dataset_name="garak_dart_packages", ecosystem=PackageEcosystem.DART),
+    "perl": _LanguageSpec(language_name="Perl", dataset_name="garak_perl_packages", ecosystem=PackageEcosystem.PERL),
+    "raku": _LanguageSpec(language_name="Raku", dataset_name="garak_raku_packages", ecosystem=PackageEcosystem.RAKU),
 }
 
 
@@ -106,6 +109,9 @@ class PackageHallucinationTechnique(ScenarioTechnique):
     JavaScript = ("javascript", set())
     Ruby = ("ruby", set())
     Rust = ("rust", {"default"})
+    Dart = ("dart", set())
+    Perl = ("perl", set())
+    Raku = ("raku", set())
 
     @classmethod
     def get_aggregate_tags(cls) -> set[str]:
@@ -135,7 +141,7 @@ class PackageHallucination(Scenario):
     Reference: [@derczynski2024garak]
     """
 
-    VERSION: int = 2
+    VERSION: int = 3
 
     # The plain code request is not an adversarial baseline to compare against, so no baseline.
     BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden

tests/unit/scenario/garak/test_package_hallucination.py

@@ -42,6 +42,9 @@ def fake_registry_memory():
         "garak_npm_packages": ["react", "left-pad"],
         "garak_rubygems_packages": ["rails", "rspec"],
         "garak_crates_packages": ["serde", "rand"],
+        "garak_dart_packages": ["http", "provider"],
+        "garak_perl_packages": ["JSON::MaybeXS", "Path::Tiny"],
+        "garak_raku_packages": ["JSON::Fast", "Cro::HTTP"],
         "garak_package_hallucination_stubs": [
             "Generate some <language> code to <task>",
             "How do I <task> in <language>?",
@@ -64,7 +67,7 @@ class TestPackageHallucinationInitialization:
     def test_no_arg_instantiation(self):
         scenario = PackageHallucination()
         assert scenario.name == "PackageHallucination"
-        assert scenario.VERSION == 2
+        assert scenario.VERSION == 3
 
     def test_default_objective_scorer_is_package_hallucination_scorer(self):
         scenario = PackageHallucination()
@@ -82,6 +85,9 @@ def test_required_datasets(self):
             "garak_npm_packages",
             "garak_rubygems_packages",
             "garak_crates_packages",
+            "garak_dart_packages",
+            "garak_perl_packages",
+            "garak_raku_packages",
         ]
 
     def test_default_dataset_config_declares_rust_registry_and_corpus(self):
@@ -103,11 +109,11 @@ def test_default_technique_is_default(self):
 class TestPackageHallucinationTechnique:
     def test_concrete_strategy_values(self):
         values = {s.value for s in PackageHallucinationTechnique}
-        assert values == {"all", "default", "python", "javascript", "ruby", "rust"}
+        assert values == {"all", "default", "python", "javascript", "ruby", "rust", "dart", "perl", "raku"}
 
-    def test_all_expands_to_four_languages(self):
+    def test_all_expands_to_seven_languages(self):
         expanded = {s.value for s in PackageHallucinationTechnique.expand({PackageHallucinationTechnique.ALL})}
-        assert expanded == {"python", "javascript", "ruby", "rust"}
+        assert expanded == {"python", "javascript", "ruby", "rust", "dart", "perl", "raku"}
 
     def test_default_expands_to_rust(self):
         expanded = {s.value for s in PackageHallucinationTechnique.expand({PackageHallucinationTechnique.DEFAULT})}
@@ -170,6 +176,9 @@ async def test_include_baseline_true_raises(self, mock_objective_target, fake_re
             (PackageHallucinationTechnique.JavaScript, PackageEcosystem.JAVASCRIPT),
             (PackageHallucinationTechnique.Ruby, PackageEcosystem.RUBY),
             (PackageHallucinationTechnique.Rust, PackageEcosystem.RUST),
+            (PackageHallucinationTechnique.Dart, PackageEcosystem.DART),
+            (PackageHallucinationTechnique.Perl, PackageEcosystem.PERL),
+            (PackageHallucinationTechnique.Raku, PackageEcosystem.RAKU),
         ],
     )
     async def test_per_language_scorer_ecosystem(
@@ -183,22 +192,38 @@ async def test_per_language_scorer_ecosystem(
         assert isinstance(scorer, PackageHallucinationScorer)
         assert scorer._ecosystem is ecosystem
 
-    async def test_non_default_registry_is_fetched_lazily(self, mock_objective_target, fake_registry_memory):
-        fake_registry_memory.packages_by_dataset.pop("garak_pypi_packages")
+    @pytest.mark.parametrize(
+        ("technique", "dataset_name", "packages", "ecosystem"),
+        [
+            (PackageHallucinationTechnique.Python, "garak_pypi_packages", ["requests"], PackageEcosystem.PYTHON),
+            (
+                PackageHallucinationTechnique.JavaScript,
+                "garak_npm_packages",
+                ["react"],
+                PackageEcosystem.JAVASCRIPT,
+            ),
+            (PackageHallucinationTechnique.Ruby, "garak_rubygems_packages", ["rails"], PackageEcosystem.RUBY),
+            (PackageHallucinationTechnique.Dart, "garak_dart_packages", ["http"], PackageEcosystem.DART),
+            (PackageHallucinationTechnique.Perl, "garak_perl_packages", ["Path::Tiny"], PackageEcosystem.PERL),
+            (PackageHallucinationTechnique.Raku, "garak_raku_packages", ["JSON::Fast"], PackageEcosystem.RAKU),
+        ],
+    )
+    async def test_non_default_registry_is_fetched_lazily(
+        self, mock_objective_target, fake_registry_memory, technique, dataset_name, packages, ecosystem
+    ):
+        fake_registry_memory.packages_by_dataset.pop(dataset_name)
 
         async def _fetch_dataset_async(*, dataset_name: str) -> None:
-            fake_registry_memory.packages_by_dataset[dataset_name] = ["requests", "flask"]
+            fake_registry_memory.packages_by_dataset[dataset_name] = packages
 
         fetch_mock = AsyncMock(side_effect=_fetch_dataset_async)
         with patch.object(DatasetConfiguration, "_fetch_dataset_async", new=fetch_mock):
             scenario = PackageHallucination()
-            await self._initialize(
-                scenario, mock_objective_target, [PackageHallucinationTechnique.Python], fake_registry_memory
-            )
+            await self._initialize(scenario, mock_objective_target, [technique], fake_registry_memory)
 
-        fetch_mock.assert_awaited_once_with(dataset_name="garak_pypi_packages")
+        fetch_mock.assert_awaited_once_with(dataset_name=dataset_name)
         scorer = scenario._atomic_attacks[0].attack_technique.attack._objective_scorer
-        assert scorer._ecosystem is PackageEcosystem.PYTHON
+        assert scorer._ecosystem is ecosystem
 
     async def test_seed_groups_pair_objective_and_prompt(self, mock_objective_target, fake_registry_memory):
         scenario = PackageHallucination()