Sketch out regression suite

experimental/regression_suite/ireers/__init__.py

@@ -4,6 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+from .compile import (
+    compile_iree,
+)
+
+from .execute import (
+    benchmark_module,
+    execute_module,
+)
+
 from .fetch import (
-    fetch_source,
+    fetch_source_fixture,
 )

experimental/regression_suite/ireers/compile.py

@@ -7,3 +7,5 @@
 import pytest
 
 
+def compile_iree(source, flags):
+    return "Compiled"

experimental/regression_suite/ireers/execute.py

@@ -0,0 +1,13 @@
+# Copyright 2023 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+
+def execute_module(vmfb, driver, inputs, expected_outputs):
+    ...
+
+
+def benchmark_module(vmfb, driver, inputs):
+    ...

experimental/regression_suite/ireers/fetch.py

@@ -7,7 +7,7 @@
 import pytest
 
 
-def fetch_source(url: str):
+def fetch_source_fixture(url: str):
     @pytest.fixture
     def fetcher(tmp_path_factory, worker_id):
         return f"Hi: {tmp_path_factory}, {worker_id}"

experimental/regression_suite/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
 markers = [
-    "plat_rdna_vulkan: mark tests as running on AMD RDNA Vulkan device",
+    "plat_rdna3_vulkan: mark tests as running on AMD RDNA3 Vulkan device",
     "presubmit: mark test as running on presubmit",
     "unstable_linalg: mark test as depending on unstable, serialized linalg IR",
 ]

experimental/regression_suite/tests/pregenerated/test_llama2.py

@@ -7,11 +7,40 @@
 import pytest
 from ireers import *
 
-llama2_7b_f16qi4_source = fetch_source("foobar")
+llama2_7b_f16qi4_source = fetch_source_fixture("foobar")
+
+
+@pytest.fixture
+def llama2_7b_f16qi4_rdna3_vmfb(llama2_7b_f16qi4_source):
+    return compile_iree(
+        llama2_7b_f16qi4_source,
+        flags=[
+            "--iree-input-type=none",
+            "--iree-hal-target-backends=vulkan",
+            "--iree-llvmcpu-target-cpu-features=host",
+            "--iree-stream-resource-index-bits=64",
+            "--iree-vm-target-index-bits=64",
+            "--iree-vulkan-target-triple=rdna3-unknown-linux",
+            "--iree-opt-const-expr-hoisting=false",
+            "--iree-stream-resource-max-allocation-size=3221225472",
+        ],
+    )
+
 
-@pytest.mark.plat_rdna_vulkan
 @pytest.mark.presubmit
 @pytest.mark.unstable_linalg
-def test_step_rdna_vulkan(llama2_7b_f16qi4_source):
-    print("GOOD")
-    print(llama2_7b_f16qi4_source)
+@pytest.mark.plat_rdna3_vulkan
+def test_step_rdna3_vulkan(llama2_7b_f16qi4_rdna3_vmfb):
+    golden_inputs = "TODO"
+    expected_outputs = "TODO"
+    execute_module(
+        llama2_7b_f16qi4_rdna3_vmfb,
+        driver="vulkan",
+        inputs=golden_inputs,
+        expected_outputs=expected_outputs,
+    )
+    benchmark_module(
+        llama2_7b_f16qi4_rdna3_vmfb,
+        driver="vulkan",
+        inputs=golden_inputs,
+    )