From 06ad2594507bd1bc28db2c554538dd1b3af14143 Mon Sep 17 00:00:00 2001
From: Stella Laurenzo <stellaraccident@gmail.com>
Date: Wed, 23 Aug 2023 19:42:51 -0700
Subject: [PATCH] Sketch out regression suite

---
 .../regression_suite/ireers/__init__.py       | 11 +++++-
 .../regression_suite/ireers/compile.py        |  2 +
 .../regression_suite/ireers/execute.py        | 13 +++++++
 experimental/regression_suite/ireers/fetch.py |  2 +-
 experimental/regression_suite/pyproject.toml  |  2 +-
 .../tests/pregenerated/test_llama2.py         | 39 ++++++++++++++++---
 6 files changed, 61 insertions(+), 8 deletions(-)
 create mode 100644 experimental/regression_suite/ireers/execute.py

diff --git a/experimental/regression_suite/ireers/__init__.py b/experimental/regression_suite/ireers/__init__.py
index 9f28fc6331f0..9c9633c4f040 100644
--- a/experimental/regression_suite/ireers/__init__.py
+++ b/experimental/regression_suite/ireers/__init__.py
@@ -4,6 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+from .compile import (
+    compile_iree,
+)
+
+from .execute import (
+    benchmark_module,
+    execute_module,
+)
+
 from .fetch import (
-    fetch_source,
+    fetch_source_fixture,
 )
diff --git a/experimental/regression_suite/ireers/compile.py b/experimental/regression_suite/ireers/compile.py
index fae5826e0c2c..f5f474b47bfc 100644
--- a/experimental/regression_suite/ireers/compile.py
+++ b/experimental/regression_suite/ireers/compile.py
@@ -7,3 +7,5 @@
 import pytest
 
 
+def compile_iree(source, flags):
+    return "Compiled"
diff --git a/experimental/regression_suite/ireers/execute.py b/experimental/regression_suite/ireers/execute.py
new file mode 100644
index 000000000000..73db40f05aa5
--- /dev/null
+++ b/experimental/regression_suite/ireers/execute.py
@@ -0,0 +1,13 @@
+# Copyright 2023 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+
+def execute_module(vmfb, driver, inputs, expected_outputs):
+    ...
+
+
+def benchmark_module(vmfb, driver, inputs):
+    ...
diff --git a/experimental/regression_suite/ireers/fetch.py b/experimental/regression_suite/ireers/fetch.py
index 2e4dc5428fa3..22d4e33923f4 100644
--- a/experimental/regression_suite/ireers/fetch.py
+++ b/experimental/regression_suite/ireers/fetch.py
@@ -7,7 +7,7 @@
 import pytest
 
 
-def fetch_source(url: str):
+def fetch_source_fixture(url: str):
     @pytest.fixture
     def fetcher(tmp_path_factory, worker_id):
         return f"Hi: {tmp_path_factory}, {worker_id}"
diff --git a/experimental/regression_suite/pyproject.toml b/experimental/regression_suite/pyproject.toml
index fda7366e411e..4d2ac2f6531a 100644
--- a/experimental/regression_suite/pyproject.toml
+++ b/experimental/regression_suite/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
 markers = [
-    "plat_rdna_vulkan: mark tests as running on AMD RDNA Vulkan device",
+    "plat_rdna3_vulkan: mark tests as running on AMD RDNA3 Vulkan device",
     "presubmit: mark test as running on presubmit",
     "unstable_linalg: mark test as depending on unstable, serialized linalg IR",
 ]
diff --git a/experimental/regression_suite/tests/pregenerated/test_llama2.py b/experimental/regression_suite/tests/pregenerated/test_llama2.py
index 5a4f3aa7e31d..b5094a86131f 100644
--- a/experimental/regression_suite/tests/pregenerated/test_llama2.py
+++ b/experimental/regression_suite/tests/pregenerated/test_llama2.py
@@ -7,11 +7,40 @@
 import pytest
 from ireers import *
 
-llama2_7b_f16qi4_source = fetch_source("foobar")
+llama2_7b_f16qi4_source = fetch_source_fixture("foobar")
+
+
+@pytest.fixture
+def llama2_7b_f16qi4_rdna3_vmfb(llama2_7b_f16qi4_source):
+    return compile_iree(
+        llama2_7b_f16qi4_source,
+        flags=[
+            "--iree-input-type=none",
+            "--iree-hal-target-backends=vulkan",
+            "--iree-llvmcpu-target-cpu-features=host",
+            "--iree-stream-resource-index-bits=64",
+            "--iree-vm-target-index-bits=64",
+            "--iree-vulkan-target-triple=rdna3-unknown-linux",
+            "--iree-opt-const-expr-hoisting=false",
+            "--iree-stream-resource-max-allocation-size=3221225472",
+        ],
+    )
+
 
-@pytest.mark.plat_rdna_vulkan
 @pytest.mark.presubmit
 @pytest.mark.unstable_linalg
-def test_step_rdna_vulkan(llama2_7b_f16qi4_source):
-    print("GOOD")
-    print(llama2_7b_f16qi4_source)
+@pytest.mark.plat_rdna3_vulkan
+def test_step_rdna3_vulkan(llama2_7b_f16qi4_rdna3_vmfb):
+    golden_inputs = "TODO"
+    expected_outputs = "TODO"
+    execute_module(
+        llama2_7b_f16qi4_rdna3_vmfb,
+        driver="vulkan",
+        inputs=golden_inputs,
+        expected_outputs=expected_outputs,
+    )
+    benchmark_module(
+        llama2_7b_f16qi4_rdna3_vmfb,
+        driver="vulkan",
+        inputs=golden_inputs,
+    )