chore: Rename to argument #604
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run GPU regression check only when attempting to merge, shown as skipped status check beforehand | |
| name: GPU merge group regression check | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: [main] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Run comparative benchmark against main, open issue on regression | |
| gpu-benchmark: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| name: GPU bench regression check | |
| runs-on: [self-hosted, gpu-bench] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/gpu-setup | |
| with: | |
| gpu-framework: 'cuda' | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| # Install dependencies | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: just@1.22 | |
| - name: Install criterion | |
| run: | | |
| cargo install cargo-criterion | |
| cargo install criterion-table | |
| - name: Set bench output format and base SHA | |
| run: | | |
| echo "LURK_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV | |
| echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV | |
| GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1) | |
| echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV | |
| echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV | |
| # Load Lurk bench env vars used in `justfile`, namely `LURK_BENCH_NOISE_THRESHOLD` | |
| - uses: cardinalby/export-env-action@v2 | |
| with: | |
| envFile: 'benches/bench.env' | |
| # Checkout gh-pages to check for cached bench result | |
| - name: Checkout gh-pages | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: gh-pages | |
| - name: Check for cached bench result | |
| id: cached-bench | |
| run: | | |
| if [ -f "fibonacci-${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json" ] | |
| then | |
| echo "cached=true" | tee -a $GITHUB_OUTPUT | |
| cp fibonacci-${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json ../fibonacci-${{ env.BASE_COMMIT }}.json | |
| else | |
| echo "cached=false" | tee -a $GITHUB_OUTPUT | |
| fi | |
| working-directory: ${{ github.workspace }}/gh-pages | |
| # Checkout base branch for comparative bench | |
| - uses: actions/checkout@v4 | |
| if: steps.cached-bench.outputs.cached == 'false' | |
| with: | |
| ref: main | |
| path: main | |
| # Copy the script so the base can bench with the same parameters | |
| - name: Run GPU bench on base branch | |
| if: steps.cached-bench.outputs.cached == 'false' | |
| run: | | |
| # Copy justfile & env to main, overwriting existing config with that of PR branch | |
| cp ../benches/justfile ../benches/bench.env . | |
| # Run benchmark | |
| just gpu-bench-ci fibonacci | |
| # Copy bench output to PR branch | |
| cp fibonacci-${{ env.BASE_COMMIT }}.json .. | |
| working-directory: ${{ github.workspace }}/main | |
| - name: Run GPU bench on PR branch | |
| run: | | |
| just gpu-bench-ci fibonacci | |
| cp fibonacci-${{ github.sha }}.json .. | |
| working-directory: ${{ github.workspace }}/benches | |
| - name: copy the benchmark template and prepare it with data | |
| run: | | |
| cp .github/tables.toml . | |
| # Get CPU model | |
| CPU_MODEL=$(grep '^model name' /proc/cpuinfo | head -1 | awk -F ': ' '{ print $2 }') | |
| # Get vCPU count | |
| NUM_VCPUS=$(nproc --all) | |
| # Get total RAM in GB | |
| TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print int($2), "GB RAM";}') | |
| WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Use conditionals to ensure that only non-empty variables are inserted | |
| [[ ! -z "${{ env.GPU_NAME }}" ]] && sed -i "/^\"\"\"$/i ${{ env.GPU_NAME }}" tables.toml | |
| [[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml | |
| [[ ! -z "$NUM_VCPUS" ]] && sed -i "/^\"\"\"$/i $NUM_VCPUS vCPUs" tables.toml | |
| [[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml | |
| sed -i "/^\"\"\"$/i Workflow run: $WORKFLOW_URL" tables.toml | |
| echo "WORKFLOW_URL=$WORKFLOW_URL" | tee -a $GITHUB_ENV | |
| working-directory: ${{ github.workspace }} | |
| # Create a `criterion-table` and write in commit comment | |
| - name: Run `criterion-table` | |
| run: cat fibonacci-${{ env.BASE_COMMIT }}.json fibonacci-${{ github.sha }}.json | criterion-table > BENCHMARKS.md | |
| - name: Write bench on commit comment | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| body-path: BENCHMARKS.md | |
| # Check for a slowdown >= `$ARECIBO_BENCH_NOISE_THRESHOLD` (fallback is 10%/1.1x). If so, open an issue but don't block merge | |
| # Since we are parsing for slowdowns, we simply add 1 to the noise threshold decimal to get the regression factor | |
| - name: Check for perf regression | |
| id: regression-check | |
| run: | | |
| REGRESSIONS=$(grep -o '[0-9.]*x slower' BENCHMARKS.md | cut -d 'x' -f1) | |
| echo $REGRESSIONS | |
| if [ ! -z "${{ env.LURK_BENCH_NOISE_THRESHOLD}}" ]; then | |
| REGRESSION_FACTOR=$(echo "${{ env.LURK_BENCH_NOISE_THRESHOLD }}+1" | bc) | |
| else | |
| REGRESSION_FACTOR=1.1 | |
| fi | |
| echo "NOISE_THRESHOLD=$(echo "($REGRESSION_FACTOR-1)*100" | bc)" | tee -a $GITHUB_ENV | |
| for r in $REGRESSIONS | |
| do | |
| if (( $(echo "$r >= $REGRESSION_FACTOR" | bc -l) )) | |
| then | |
| echo "regression=true" | tee -a $GITHUB_OUTPUT | |
| fi | |
| done | |
| # Not possible to use ${{ github.event.number }} with the `merge_group` trigger | |
| - name: Get PR number from merge branch | |
| if: steps.regression-check.outputs.regression == 'true' | |
| run: | | |
| echo "PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/')" | tee -a $GITHUB_ENV | |
| - uses: JasonEtco/create-an-issue@v2 | |
| if: steps.regression-check.outputs.regression == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ env.PR_NUMBER }} | |
| GIT_SHA: ${{ github.sha }} | |
| WORKFLOW_URL: ${{ env.WORKFLOW_URL }} | |
| NOISE_THRESHOLD: ${{ env.NOISE_THRESHOLD }} | |
| with: | |
| filename: .github/PERF_REGRESSION.md | |
| - name: Remove old main bench | |
| if: steps.regression-check.outputs.regression != 'true' | |
| run: | | |
| rm fibonacci-${{ env.BASE_COMMIT }}.json | |
| mv fibonacci-${{ github.sha }}.json fibonacci-${{ github.sha }}-${{ env.GPU_ID }}.json | |
| working-directory: ${{ github.workspace }} | |
| - name: Commit bench result to `gh-pages` branch if no regression | |
| if: steps.regression-check.outputs.regression != 'true' | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: gh-pages | |
| commit_message: '[automated] GPU Benchmark from PR #${{ env.PR_NUMBER }}' | |
| file_pattern: 'fibonacci-${{ github.sha }}-${{ env.GPU_ID }}.json' |