mmperf #416
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2022 The IREE Authors | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| # | |
| # Workflow for running `mmperf` (https://github.com/mmperf/mmperf). | |
| # `mmperf` benchmarks matrix-multiply workloads on IREE and other backends such | |
| # as RUY, TVM, Halide, CuBLAS, etc. | |
| # | |
| # The workflow runs benchmarks on CPU and GPU and uploads results to the | |
| # `mmperf-benchmark-artifacts` GC bucket. | |
| name: mmperf | |
| on: | |
| schedule: | |
| - cron: '0 13 * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| # A PR number if a pull request and otherwise the commit hash. This cancels | |
| # queued and in-progress runs for the same PR (presubmit) or commit | |
| # (postsubmit). The workflow name is prepended to avoid conflicts between | |
| # different workflows. | |
| group: ${{ github.workflow }}-${{ github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-20.04 | |
| env: | |
| # The commit being checked out is the merge commit for the PR. Its first | |
| # parent will be the tip of main. | |
| BASE_REF: HEAD^ | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| PR_BODY: ${{ github.event.pull_request.body }} | |
| IREE_SHA: ${{ github.sha }} | |
| outputs: | |
| artifact-upload-dir: ${{ steps.iree.outputs.artifact-upload-dir }} | |
| should-run: ${{ steps.configure.outputs.should-run }} | |
| runner-env: ${{ steps.configure.outputs.runner-env }} | |
| runner-group: ${{ steps.configure.outputs.runner-group }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 | |
| with: | |
| # We need the parent commit to do a diff | |
| fetch-depth: 2 | |
| - name: "Configuring CI options" | |
| id: configure | |
| run: | | |
| # Just informative logging. There should only be two commits in the | |
| # history here, but limiting the depth helps when copying from a local | |
| # repo instead of using checkout, e.g. with | |
| # https://github.com/nektos/act where there will be more. | |
| git log --oneline --graph --max-count=3 | |
| ./build_tools/github_actions/configure_ci.py | |
| - name: "Calculating version info" | |
| id: iree | |
| run: | | |
| export GCS_ARTIFACT_DIR="$(date +'%Y-%m-%d').sha_${IREE_SHA}.timestamp_$(date +'%s')" | |
| echo "artifact-upload-dir=${GCS_ARTIFACT_DIR}" >> $GITHUB_OUTPUT | |
| build_and_benchmark_cpu: | |
| needs: setup | |
| if: needs.setup.outputs.should-run == 'true' | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| IREE_SHA: ${{ github.sha }} | |
| BUILD_DIR: mmperf-build-cpu | |
| RESULTS_DIR: mmperf-results-cpu | |
| GCS_UPLOAD_PARENT_DIR: "gs://mmperf-benchmark-artifacts/cpu" | |
| GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 | |
| - name: "Building mmperf for CPU" | |
| run: | | |
| ./build_tools/github_actions/docker_run.sh \ | |
| gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \ | |
| ./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cpu" "${IREE_SHA}" | |
| - name: "Running mmperf on CPU" | |
| run: | | |
| mkdir ${RESULTS_DIR} | |
| ./build_tools/github_actions/docker_run.sh \ | |
| gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \ | |
| ./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}" | |
| - name: "Uploading results" | |
| run: | | |
| gcloud storage cp "${RESULTS_DIR}/latest/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/" | |
| gcloud storage cp "${RESULTS_DIR}/latest/matmul.png" "${GCS_UPLOAD_PARENT_DIR}/matmul.png" | |
| build_cuda: | |
| needs: setup | |
| if: needs.setup.outputs.should-run == 'true' | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| IREE_SHA: ${{ github.sha }} | |
| BUILD_DIR: mmperf-build-cuda | |
| outputs: | |
| build-dir: ${{ env.BUILD_DIR }} | |
| build-dir-archive: ${{ steps.archive.outputs.build-dir-archive }} | |
| build-dir-gcs-artifact: ${{ steps.upload.outputs.build-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 | |
| - name: "Building mmperf for CUDA" | |
| run: | | |
| ./build_tools/github_actions/docker_run.sh \ | |
| gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \ | |
| ./build_tools/benchmarks/mmperf/build_mmperf.sh "${BUILD_DIR}" "cuda" "${IREE_SHA}" | |
| - name: "Removing unused files" | |
| run: | | |
| find "${BUILD_DIR}" -type f -name "*.a" -o -type f -name "*.o" \ | |
| -print \ | |
| -delete | |
| - name: "Creating build dir archive" | |
| id: archive | |
| env: | |
| BUILD_DIR_ARCHIVE: ${{ env.BUILD_DIR }}.tar.zst | |
| run: | | |
| tar -I 'zstd -T0' \ | |
| -cf ${BUILD_DIR_ARCHIVE} ${BUILD_DIR} | |
| echo "build-dir-archive=${BUILD_DIR_ARCHIVE}" >> "${GITHUB_OUTPUT}" | |
| - name: "Uploading build dir archive" | |
| id: upload | |
| env: | |
| BUILD_DIR_ARCHIVE: ${{ steps.archive.outputs.build-dir-archive }} | |
| BUILD_DIR_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.archive.outputs.build-dir-archive }} | |
| run: | | |
| gcloud storage cp "${BUILD_DIR_ARCHIVE}" "${BUILD_DIR_GCS_ARTIFACT}" | |
| echo "build-dir-gcs-artifact=${BUILD_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| benchmark_cuda: | |
| needs: [setup, build_cuda] | |
| if: needs.setup.outputs.should-run == 'true' | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - machine-type=a2-highgpu-1g | |
| - os-family=Linux | |
| env: | |
| RESULTS_DIR: mmperf-results-cuda | |
| GCS_UPLOAD_PARENT_DIR: "gs://mmperf-benchmark-artifacts/cuda" | |
| GCS_UPLOAD_DIR_NAME: ${{ needs.setup.outputs.artifact-upload-dir }} | |
| BUILD_DIR: ${{ needs.build_cuda.outputs.build-dir }} | |
| BUILD_DIR_ARCHIVE: ${{ needs.build_cuda.outputs.build-dir-archive }} | |
| BUILD_DIR_GCS_ARTIFACT: ${{ needs.build_cuda.outputs.build-dir-gcs-artifact }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 | |
| - name: "Downloading build dir archive" | |
| run: gcloud storage cp "${BUILD_DIR_GCS_ARTIFACT}" "${BUILD_DIR_ARCHIVE}" | |
| - name: "Extracting build dir archive" | |
| run: tar -xf "${BUILD_DIR_ARCHIVE}" | |
| - name: "Running mmperf on CUDA" | |
| run: | | |
| mkdir ${RESULTS_DIR} | |
| ./build_tools/github_actions/docker_run.sh \ | |
| --gpus all \ | |
| gcr.io/iree-oss/mmperf@sha256:dfeebcaf8f417275d44c7b8d9a1360c6a26d280b526779c9e64384798cfd9c92 \ | |
| ./build_tools/benchmarks/mmperf/run_mmperf.sh "${BUILD_DIR}" "${RESULTS_DIR}" | |
| - name: "Uploading results" | |
| run: | | |
| export GCS_DIR_NAME="$(date +'%Y-%m-%d').$(date +'%s')" | |
| gcloud storage cp "${RESULTS_DIR}/latest/**" "${GCS_UPLOAD_PARENT_DIR}/${GCS_UPLOAD_DIR_NAME}/" | |
| gcloud storage cp "${RESULTS_DIR}/latest/matmul.png" "${GCS_UPLOAD_PARENT_DIR}/matmul.png" |