gpu-bench-manual-comparative.yml

# Run GPU benchmark on `main` and a local branch when manually triggered on the latter
# Currently `main` benchmarks are uncached
name: Manual GPU benchmarks compared to main

on:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  gpu-benchmark:
    name: Manual GPU bench comparison
    runs-on: [self-hosted, gpu-bench]
    steps:
      - uses: actions/checkout@v4
        with:
          repository: argumentcomputer/ci-workflows
      - uses: ./.github/actions/gpu-setup
        with:
          gpu-framework: 'cuda'
      - uses: ./.github/actions/ci-env
      - uses: actions/checkout@v4
      - uses: actions/checkout@v4
        with:
          ref: main
          path: main
      # Install dependencies
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        with:
          tool: just@1.22
      - name: Install criterion
        run: |
          cargo install cargo-criterion
          cargo install criterion-table
      - name: Set bench output format type
        run: |
          echo "LURK_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV
          echo "BASE_COMMIT=$(git rev-parse HEAD)" | tee -a $GITHUB_ENV
          echo "GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)" | tee -a $GITHUB_ENV
          echo "GPU_ID=$(echo ${{ env.GPU_NAME }} | awk '{ print $NF }')" | tee -a $GITHUB_ENV
        working-directory: ${{ github.workspace }}/main
      - name: Run GPU bench on base branch
        run: |
          # Copy justfile & env to main, overwriting existing config with that of PR branch
          cp ../benches/justfile ../benches/bench.env .
          just gpu-bench-ci fibonacci
          # Copy bench output to PR branch
          cp fibonacci-${{ env.BASE_COMMIT }}.json ..
        working-directory: ${{ github.workspace }}/main
      - name: Run GPU bench on PR branch
        run: |
          just gpu-bench-ci fibonacci
          cp fibonacci-${{ github.sha }}.json ..
        working-directory: ${{ github.workspace }}/benches
      - name: copy the benchmark template and prepare it with data
        run: |
          cp .github/tables.toml .
          # Get GPU name
          GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1)
          # Get CPU model
          CPU_MODEL=$(grep '^model name' /proc/cpuinfo | head -1 | awk -F ': ' '{ print $2 }')
          # Get vCPU count
          NUM_VCPUS=$(nproc --all)
          # Get total RAM in GB
          TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print $2, "GB RAM";}')
          WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          # Use conditionals to ensure that only non-empty variables are inserted
          [[ ! -z "$GPU_NAME" ]] && sed -i "/^\"\"\"$/i $GPU_NAME" tables.toml
          [[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml
          [[ ! -z "$NUM_VCPUS" ]] && sed -i "/^\"\"\"$/i $NUM_VCPUS vCPUs" tables.toml
          [[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml
          sed -i "/^\"\"\"$/i Workflow run: $WORKFLOW_URL" tables.toml
        working-directory: ${{ github.workspace }}
      # Create a `criterion-table` and write in commit comment
      - name: Run `criterion-table`
        run: cat fibonacci-${{ env.BASE_COMMIT }}.json fibonacci-${{ github.sha }}.json | criterion-table > BENCHMARKS.md
        working-directory: ${{ github.workspace }}
      - name: Write bench on commit comment
        uses: peter-evans/commit-comment@v3
        with:
          body-path: BENCHMARKS.md