From 8e54ed53ba3b4ad7dce2695fd19af0e0efb58e66 Mon Sep 17 00:00:00 2001
From: Scott Todd <scott.todd0@gmail.com>
Date: Wed, 16 Oct 2024 17:50:05 -0700
Subject: [PATCH] Run ONNX model tests as part of pkgci_test_onnx. (#18795)

Progress on https://github.com/iree-org/iree-test-suites/issues/6.

Current tests included and their statuses:
```
PASSED onnx_models/tests/vision/classification_models_test.py::test_alexnet
PASSED onnx_models/tests/vision/classification_models_test.py::test_caffenet
PASSED onnx_models/tests/vision/classification_models_test.py::test_densenet_121
PASSED onnx_models/tests/vision/classification_models_test.py::test_googlenet
PASSED onnx_models/tests/vision/classification_models_test.py::test_inception_v2
PASSED onnx_models/tests/vision/classification_models_test.py::test_mnist
PASSED onnx_models/tests/vision/classification_models_test.py::test_resnet50_v1
PASSED onnx_models/tests/vision/classification_models_test.py::test_resnet50_v2
PASSED onnx_models/tests/vision/classification_models_test.py::test_shufflenet
PASSED onnx_models/tests/vision/classification_models_test.py::test_shufflenet_v2
PASSED onnx_models/tests/vision/classification_models_test.py::test_squeezenet
PASSED onnx_models/tests/vision/classification_models_test.py::test_vgg19
XFAIL onnx_models/tests/vision/classification_models_test.py::test_efficientnet_lite4
XFAIL onnx_models/tests/vision/classification_models_test.py::test_inception_v1
XFAIL onnx_models/tests/vision/classification_models_test.py::test_mobilenet
XFAIL onnx_models/tests/vision/classification_models_test.py::test_rcnn_ilsvrc13
XFAIL onnx_models/tests/vision/classification_models_test.py::test_zfnet_512
```

* CPU only for now. We haven't yet parameterized those tests to allow
for other backends or flags.
* Starting with `--override-ini=xfail_strict=false` so newly _passing_
tests won't fail the job. Newly _failing_ tests will fail the job. We
can add an external config file to customize which tests are expected to
fail like the onnx op tests if we want to track which are
passing/failing in this repository instead of in the test suite repo.

Sample logs:
https://github.com/iree-org/iree/actions/runs/11371239238/job/31633406729?pr=18795

ci-exactly: build_packages, test_onnx
---
 .github/workflows/pkgci_test_onnx.yml | 57 ++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pkgci_test_onnx.yml b/.github/workflows/pkgci_test_onnx.yml
index 1d6b890a5ceb..65d606c9958a 100644
--- a/.github/workflows/pkgci_test_onnx.yml
+++ b/.github/workflows/pkgci_test_onnx.yml
@@ -19,7 +19,7 @@ on:
 
 jobs:
   test_onnx_ops:
-    name: "test_onnx :: ${{ matrix.name }}"
+    name: "test_onnx_ops :: ${{ matrix.name }}"
     runs-on: ${{ matrix.runs-on }}
     strategy:
       fail-fast: false
@@ -90,7 +90,7 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           repository: iree-org/iree-test-suites
-          ref: 9e921d0ea271a85f772eee22965585461c9b14c2
+          ref: 3a0ea13cbdc954365d653a48cc99cc63a9ff09b3
           path: iree-test-suites
       - name: Install ONNX ops test suite requirements
         run: |
@@ -120,3 +120,56 @@ jobs:
         with:
           name: ${{ matrix.config-file }}
           path: ${{ env.CONFIG_FILE_PATH }}
+
+  test_onnx_models:
+    name: "test_onnx_models :: ${{ matrix.name }}"
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # CPU
+          - name: cpu_llvm_task
+            runs-on: ubuntu-20.04
+
+          # TODO(scotttodd): test other backends (parameterize the test suite)
+    env:
+      VENV_DIR: ${{ github.workspace }}/venv
+    steps:
+      - name: Checking out IREE repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          submodules: false
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.1.0
+        with:
+          # Must match the subset of versions built in pkgci_build_packages.
+          python-version: "3.11"
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: linux_x86_64_release_packages
+          path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
+      - name: Setup venv
+        run: |
+          ./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
+            --artifact-path=${PACKAGE_DOWNLOAD_DIR} \
+            --fetch-gh-workflow=${{ inputs.artifact_run_id }}
+
+      - name: Checkout test suites repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          repository: iree-org/iree-test-suites
+          ref: 3a0ea13cbdc954365d653a48cc99cc63a9ff09b3
+          path: iree-test-suites
+      - name: Install ONNX models test suite requirements
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python -m pip install -r iree-test-suites/onnx_models/requirements.txt
+      - name: Run ONNX models test suite
+        run: |
+          source ${VENV_DIR}/bin/activate
+          pytest iree-test-suites/onnx_models/ \
+              -rA \
+              --log-cli-level=info \
+              --override-ini=xfail_strict=false \
+              --timeout=120 \
+              --durations=0