bench-pr-comment.yml

# Creates a PR benchmark comment with a comparison to main
name: Benchmark pull requests
on:
  issue_comment:
    types: [created]

env:
  CARGO_TERM_COLOR: always

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  cpu-benchmark:
    name: run fibonacci benchmark
    runs-on: buildjet-32vcpu-ubuntu-2204
    if:
      github.event.issue.pull_request
      && github.event.issue.state == 'open'
      && contains(github.event.comment.body, '!benchmark')
      && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
    steps:
      - uses: actions/checkout@v4
      - name: Checkout PR branch
        run: gh pr checkout $PR_NUMBER
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.issue.number }}
      # Install dependencies
      - name: Install dependencies
        run: sudo apt-get install -y pkg-config libssl-dev
      - uses: actions-rs/toolchain@v1
      - uses: Swatinem/rust-cache@v2
      - name: Load env vars
        run: |
          set -a
          source bench.env
          set +a
          echo "LURK_BENCH_OUTPUT=pr-comment" >> $GITHUB_ENV
          env | grep -E 'LURK|EC_GPU|CUDA'
        working-directory: ${{ github.workspace }}/benches
      # Run the comparative benchmark and comment output on the PR
      - uses: boa-dev/criterion-compare-action@v3
        with:
          # Optional. Compare only this benchmark target
          benchName: "fibonacci"
          # Needed. The name of the branch to compare with
          branchName: ${{ github.ref_name }}

  gpu-benchmark:
    name: run fibonacci benchmark on GPU
    runs-on: [self-hosted, gpu-bench]
    if:
      github.event.issue.pull_request
      && github.event.issue.state == 'open'
      && contains(github.event.comment.body, '!gpu-benchmark')
      && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
    steps:
      - uses: actions/checkout@v4
      - name: Checkout PR branch
        run: gh pr checkout $PR_NUMBER
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.issue.number }}
      # Set up GPU
      # Check we have access to the machine's Nvidia drivers
      - run: nvidia-smi
      # The `compute`/`sm` number corresponds to the Nvidia GPU architecture
      # In this case, the self-hosted machine uses the Ampere architecture, but we want this to be configurable
      # See https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/
      # Writes env vars to `bench.env` to be read by `dotenv` action. This is roundabout but puts all the env vars in one place
      - name: Set env for CUDA compute
        run: echo "CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed 's/\.//g')" >> bench.env
      - name: set env for EC_GPU
        run: echo 'EC_GPU_CUDA_NVCC_ARGS=--fatbin --gpu-architecture=sm_${{ env.CUDA_ARCH }} --generate-code=arch=compute_${{ env.CUDA_ARCH }},code=sm_${{ env.CUDA_ARCH }}' >> bench.env
      # Check that CUDA is installed with a driver-compatible version
      # This must also be compatible with the GPU architecture, see above link
      - run: nvcc --version
      # Install dependencies
      - uses: actions-rs/toolchain@v1
      - uses: Swatinem/rust-cache@v2
      - name: Load env vars
        run: |
          set -a
          source bench.env
          set +a
          echo "LURK_BENCH_OUTPUT=pr-comment" >> $GITHUB_ENV
          env | grep -E 'LURK|EC_GPU|CUDA'
        working-directory: ${{ github.workspace }}/benches
      # Run the comparative benchmark and comment output on the PR
      - uses: boa-dev/criterion-compare-action@v3
        with:
          # Note: Removing `benchName` causes `criterion` errors: https://github.com/boa-dev/criterion-compare-action#troubleshooting
          # Optional. Compare only this benchmark target
          benchName: "fibonacci"
          # Optional. Features activated in the benchmark
          features: "cuda"
          # Needed. The name of the branch to compare with
          branchName: ${{ github.ref_name }}