Install annotators in modelgauge using Poetry

modelgauge/multiple_choice_questions.py

@@ -2,6 +2,7 @@
 This module supports the common pattern of structuring Prompts
 as multiple choice questions.
 """
+
 import random
 from collections import defaultdict
 from dataclasses import dataclass

modelgauge/pipeline.py

@@ -50,6 +50,7 @@ def __init__(self, ):
 Lock (GIL) means that generally only one bit of python is running at once.
 
  """
+
 import datetime
 import queue
 import sys

pyproject.toml

@@ -53,6 +53,7 @@ modelgauge_standard_tests = {version = "*", optional = true}
 modelgauge_openai = {version = "*", optional = true}
 modelgauge_huggingface = {version = "*", optional = true}
 modelgauge_perspective_api = {version = "*", optional = true}
+modelgauge_teacher_models = {git = "https://github.com/mlcommons/modeltune.git", rev = "feat/ensemble-annotators-installer", subdirectory = "annotations/auto_annotation", extras = ["main"], "optional" = true}
 tomli = "^2.0.1"
 click = "^8.1.7"
 typing-extensions = "^4.10.0"
@@ -71,14 +72,15 @@ modelgauge_openai = {path = "plugins/openai", develop = true, optional=true}
 modelgauge_huggingface = {path = "plugins/huggingface", develop = true, optional=true}
 modelgauge_perspective_api = {path = "plugins/perspective_api", develop = true, optional=true}
 pytest-datafiles = "^3.0.0"
-pytest = "^7.4.3"
+pytest = "^8.3.1"
 mypy = "^1.7.1"
-black = "^23.11.0"
 pytest-mock = "^3.12.0"
 pytest-timeout = "^2.3.1"
 flaky = "^3.8.1"
 nbmake = "^1.5.3"
 tox = "^4.14.2"
+black = {extras = ["jupyter"], version = "^24.8.0"}
+
 
 [tool.pytest.ini_options]
 # Ignore the main source that might have things named "test"
@@ -101,6 +103,7 @@ standard_tests = ["modelgauge_standard_tests"]
 openai = ["modelgauge_openai"]
 huggingface = ["modelgauge_huggingface"]
 perspective_api = ["modelgauge_perspective_api"]
+modeltune = ["modelgauge_teacher_models"]
 all_plugins = ["modelgauge_demo_plugin", "modelgauge_openai", "modelgauge_huggingface", "modelgauge_standard_tests", "modelgauge_perspective_api"]
 
 

tests/test_api_server.py

@@ -10,9 +10,9 @@ def setup_method(self):
         self.secret_key = "whatever"
         with patch(
             "os.getenv",
-            lambda *args: self.secret_key
-            if args[0] == "SECRET_KEY"
-            else real_getenv(*args),
+            lambda *args: (
+                self.secret_key if args[0] == "SECRET_KEY" else real_getenv(*args)
+            ),
         ):
             with patch(
                 "modelgauge.config.load_secrets_from_config",

tests/test_notebook.ipynb

@@ -18,7 +18,9 @@
     "secrets = fake_all_secrets()\n",
     "demo_test_instance_factory = TESTS.make_instance(\"demo_01\", secrets=secrets)\n",
     "\n",
-    "assert demo_test_instance_factory.get_dependencies() == demo_test_import.get_dependencies()"
+    "assert (\n",
+    "    demo_test_instance_factory.get_dependencies() == demo_test_import.get_dependencies()\n",
+    ")"
    ]
   },
   {
@@ -39,11 +41,27 @@
     "run_sut([\"--sut\", \"demo_yes_no\", \"--prompt\", \"My test prompt\"], standalone_mode=False)\n",
     "with tempfile.TemporaryDirectory() as tmpdirname:\n",
     "    tmp_output = os.path.join(tmpdirname, \"record.json\")\n",
-    "    run_test([\"--test\", \"demo_01\", \"--sut\", \"demo_yes_no\", \"--data-dir\", tmpdirname, \"--output-file\", tmp_output], standalone_mode=False)\n",
+    "    run_test(\n",
+    "        [\n",
+    "            \"--test\",\n",
+    "            \"demo_01\",\n",
+    "            \"--sut\",\n",
+    "            \"demo_yes_no\",\n",
+    "            \"--data-dir\",\n",
+    "            tmpdirname,\n",
+    "            \"--output-file\",\n",
+    "            tmp_output,\n",
+    "        ],\n",
+    "        standalone_mode=False,\n",
+    "    )\n",
     "\n",
     "    # Try using runner directly\n",
     "    sut = SUTS.make_instance(\"demo_yes_no\", secrets=secrets)\n",
-    "    record = run_prompt_response_test(demo_test_import, sut, tmpdirname, )"
+    "    record = run_prompt_response_test(\n",
+    "        demo_test_import,\n",
+    "        sut,\n",
+    "        tmpdirname,\n",
+    "    )"
    ]
   }
  ],