gpu-bench-merge-regression.yml

# Run GPU regression check only when attempting to merge, shown as skipped status check beforehand
name: GPU merge group regression check

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    branches: [main]
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  # Run comparative benchmark against main, open issue on regression
  gpu-benchmark:
    if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
    name: GPU bench regression check
    runs-on: [self-hosted, gpu-bench]
    steps:
      - uses: actions/checkout@v4
        with:
          repository: lurk-lab/ci-workflows
      - uses: ./.github/actions/gpu-setup
        with:
          gpu-framework: 'cuda'
      - uses: ./.github/actions/ci-env
      - uses: actions/checkout@v4
      # Install dependencies
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        with:
          tool: just@1.22
      - name: Install criterion
        run: |
          cargo install cargo-criterion
          cargo install criterion-table
      - name: Set bench output format and base SHA
        run: |
          echo "LURK_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV
          echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV
          GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)
          echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV
          echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV
      # Load Lurk bench env vars used in `justfile`, namely `LURK_BENCH_NOISE_THRESHOLD`
      - uses: cardinalby/export-env-action@v2
        with:
          envFile: 'benches/bench.env'
      # Checkout gh-pages to check for cached bench result
      - name: Checkout gh-pages
        uses: actions/checkout@v4
        with:
          ref: gh-pages
          path: gh-pages
      - name: Check for cached bench result
        id: cached-bench
        run: |
          if [ -f "fibonacci-${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json" ]
          then
            echo "cached=true" | tee -a $GITHUB_OUTPUT
            cp fibonacci-${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json ../fibonacci-${{ env.BASE_COMMIT }}.json
          else
            echo "cached=false" | tee -a $GITHUB_OUTPUT
          fi
        working-directory: ${{ github.workspace }}/gh-pages
      # Checkout base branch for comparative bench
      - uses: actions/checkout@v4
        if: steps.cached-bench.outputs.cached == 'false'
        with:
          ref: main
          path: main
      # Copy the script so the base can bench with the same parameters
      - name: Run GPU bench on base branch
        if: steps.cached-bench.outputs.cached == 'false'
        run: |
          # Copy justfile & env to main, overwriting existing config with that of PR branch
          cp ../benches/justfile ../benches/bench.env .
          # Run benchmark
          just gpu-bench-ci fibonacci
          # Copy bench output to PR branch
          cp fibonacci-${{ env.BASE_COMMIT }}.json ..
        working-directory: ${{ github.workspace }}/main
      - name: Run GPU bench on PR branch
        run: |
          just gpu-bench-ci fibonacci
          cp fibonacci-${{ github.sha }}.json ..
        working-directory: ${{ github.workspace }}/benches
      - name: copy the benchmark template and prepare it with data
        run: |
          cp .github/tables.toml .
          # Get CPU model
          CPU_MODEL=$(grep '^model name' /proc/cpuinfo | head -1 | awk -F ': ' '{ print $2 }')
          # Get vCPU count
          NUM_VCPUS=$(nproc --all)
          # Get total RAM in GB
          TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print int($2), "GB RAM";}')
          WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          # Use conditionals to ensure that only non-empty variables are inserted
          [[ ! -z "${{ env.GPU_NAME }}" ]] && sed -i "/^\"\"\"$/i ${{ env.GPU_NAME }}" tables.toml
          [[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml
          [[ ! -z "$NUM_VCPUS" ]] && sed -i "/^\"\"\"$/i $NUM_VCPUS vCPUs" tables.toml
          [[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml
          sed -i "/^\"\"\"$/i Workflow run: $WORKFLOW_URL" tables.toml
          echo "WORKFLOW_URL=$WORKFLOW_URL" | tee -a $GITHUB_ENV
        working-directory: ${{ github.workspace }}
      # Create a `criterion-table` and write in commit comment
      - name: Run `criterion-table`
        run: cat fibonacci-${{ env.BASE_COMMIT }}.json fibonacci-${{ github.sha }}.json | criterion-table > BENCHMARKS.md
      - name: Write bench on commit comment
        uses: peter-evans/commit-comment@v3
        with:
          body-path: BENCHMARKS.md
      # Check for a slowdown >= `$ARECIBO_BENCH_NOISE_THRESHOLD` (fallback is 10%/1.1x). If so, open an issue but don't block merge
      # Since we are parsing for slowdowns, we simply add 1 to the noise threshold decimal to get the regression factor
      - name: Check for perf regression
        id: regression-check
        run: |
          REGRESSIONS=$(grep -o '[0-9.]*x slower' BENCHMARKS.md | cut -d 'x' -f1)

          echo $REGRESSIONS

          if [ ! -z "${{ env.LURK_BENCH_NOISE_THRESHOLD}}" ]; then
            REGRESSION_FACTOR=$(echo "${{ env.LURK_BENCH_NOISE_THRESHOLD }}+1" | bc)
          else
            REGRESSION_FACTOR=1.1
          fi
          echo "NOISE_THRESHOLD=$(echo "($REGRESSION_FACTOR-1)*100" | bc)" | tee -a $GITHUB_ENV

          for r in $REGRESSIONS
          do
            if (( $(echo "$r >= $REGRESSION_FACTOR" | bc -l) ))
            then
              echo "regression=true" | tee -a $GITHUB_OUTPUT
            fi
          done
      # Not possible to use ${{ github.event.number }} with the `merge_group` trigger
      - name: Get PR number from merge branch
        if: steps.regression-check.outputs.regression == 'true'
        run: |
          echo "PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/')" | tee -a $GITHUB_ENV
      - uses: JasonEtco/create-an-issue@v2
        if: steps.regression-check.outputs.regression == 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ env.PR_NUMBER }}
          GIT_SHA: ${{ github.sha }}
          WORKFLOW_URL: ${{ env.WORKFLOW_URL }}
          NOISE_THRESHOLD: ${{ env.NOISE_THRESHOLD }}
        with:
          filename: .github/PERF_REGRESSION.md
      - name: Remove old main bench
        if: steps.regression-check.outputs.regression != 'true'
        run: |
          rm fibonacci-${{ env.BASE_COMMIT }}.json
          mv fibonacci-${{ github.sha }}.json fibonacci-${{ github.sha }}-${{ env.GPU_ID }}.json
        working-directory: ${{ github.workspace }}
      - name: Commit bench result to `gh-pages` branch if no regression
        if: steps.regression-check.outputs.regression != 'true'
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          branch: gh-pages
          commit_message: '[automated] GPU Benchmark from PR #${{ env.PR_NUMBER }}'
          file_pattern: 'fibonacci-${{ github.sha }}-${{ env.GPU_ID }}.json'