chore: Repair fibonacci bench #523
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run final tests only when attempting to merge, shown as skipped status checks beforehand | |
| name: Merge group tests | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: [main] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| linux-ignored: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| runs-on: buildjet-16vcpu-ubuntu-2204 | |
| env: | |
| RUSTFLAGS: -D warnings | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Linux Tests | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored ignored-only -E 'all() - test(test_demo)' | |
| linux-arm: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| runs-on: buildjet-16vcpu-ubuntu-2204-arm | |
| env: | |
| RUSTFLAGS: -D warnings | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Linux Tests | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-ci | |
| - name: Linux Gadget Tests w/o debug assertions | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-no-assertions -E 'test(circuit::gadgets)' | |
| mac-m1: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| runs-on: macos-latest-xlarge | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Linux Tests | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-ci | |
| - name: Linux Gadget Tests w/o debug assertions | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-no-assertions -E 'test(circuit::gadgets)' | |
| # TODO: Make this a required status check | |
| # Run comparative benchmark against main, reject on regression | |
| gpu-benchmark: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| name: Run fibonacci bench on GPU | |
| runs-on: [self-hosted, gpu-bench] | |
| steps: | |
| # TODO: Factor out GPU setup into an action or into justfile, it's used in 4 places | |
| # Set up GPU | |
| # Check we have access to the machine's Nvidia drivers | |
| - run: nvidia-smi | |
| # Check that CUDA is installed with a driver-compatible version | |
| # This must also be compatible with the GPU architecture, see above link | |
| - run: nvcc --version | |
| - uses: actions/checkout@v4 | |
| # Install dependencies | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: just@1.15 | |
| - name: Install criterion | |
| run: | | |
| cargo install cargo-criterion | |
| cargo install criterion-table | |
| - name: Set bench output format and base SHA | |
| run: | | |
| echo "LURK_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV | |
| echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV | |
| GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1) | |
| echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV | |
| echo "GPU_NAME=$GPU_NAME" | tee -a $GITHUB_ENV | |
| # Checkout gh-pages to check for cached bench result | |
| - name: Checkout gh-pages | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: gh-pages | |
| - name: Check for cached bench result | |
| id: cached-bench | |
| run: | | |
| if [ -f "${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json" ] | |
| then | |
| echo "cached=true" | tee -a $GITHUB_OUTPUT | |
| cp ${{ env.BASE_COMMIT }}-${{ env.GPU_ID }}.json ../${{ env.BASE_COMMIT }}.json | |
| else | |
| echo "cached=false" | tee -a $GITHUB_OUTPUT | |
| fi | |
| working-directory: ${{ github.workspace }}/gh-pages | |
| # Checkout base branch for comparative bench | |
| - uses: actions/checkout@v4 | |
| if: steps.cached-bench.outputs.cached == 'false' | |
| with: | |
| ref: main | |
| path: main | |
| # Copy the script so the base can bench with the same parameters | |
| - name: Run GPU bench on base branch | |
| if: steps.cached-bench.outputs.cached == 'false' | |
| run: | | |
| # Copy justfile & env to main, overwriting existing config with that of PR branch | |
| cp ../benches/justfile ../benches/bench.env . | |
| # Run benchmark | |
| just --dotenv-filename bench.env gpu-bench fibonacci | |
| # Copy bench output to PR branch | |
| cp ${{ env.BASE_COMMIT }}.json .. | |
| working-directory: ${{ github.workspace }}/main | |
| - name: Run GPU bench on PR branch | |
| run: | | |
| just --dotenv-filename bench.env gpu-bench fibonacci | |
| cp ${{ github.sha }}.json .. | |
| working-directory: ${{ github.workspace }}/benches | |
| - name: copy the benchmark template and prepare it with data | |
| run: | | |
| cp .github/tables.toml . | |
| # Get CPU model | |
| CPU_MODEL=$(grep '^model name' /proc/cpuinfo | head -1 | awk -F ': ' '{ print $2 }') | |
| # Get total RAM in GB | |
| TOTAL_RAM=$(grep MemTotal /proc/meminfo | awk '{$2=$2/(1024^2); print $2, "GB RAM";}') | |
| # Use conditionals to ensure that only non-empty variables are inserted | |
| [[ ! -z "${{ env.GPU_NAME }}" ]] && sed -i "/^\"\"\"$/i ${{ env.GPU_NAME }}" tables.toml | |
| [[ ! -z "$CPU_MODEL" ]] && sed -i "/^\"\"\"$/i $CPU_MODEL" tables.toml | |
| [[ ! -z "$TOTAL_RAM" ]] && sed -i "/^\"\"\"$/i $TOTAL_RAM" tables.toml | |
| sed -i "/^\"\"\"$/i Workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" tables.toml | |
| working-directory: ${{ github.workspace }} | |
| # Create a `criterion-table` and write in commit comment | |
| - name: Run `criterion-table` | |
| run: cat ${{ env.BASE_COMMIT }}.json ${{ github.sha }}.json | criterion-table > BENCHMARKS.md | |
| - name: Write bench on commit comment | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| body-path: BENCHMARKS.md | |
| # Check for a slowdown >= 10%. If so, open an issue but don't block merge | |
| - name: Check for perf regression | |
| id: regression-check | |
| run: | | |
| regressions=$(awk -F'[*x]' '/slower/{print $12}' BENCHMARKS.md) | |
| echo $regressions | |
| for r in $regressions | |
| do | |
| if (( $(echo "$r >= 1.10" | bc -l) )) | |
| then | |
| exit 1 | |
| fi | |
| done | |
| continue-on-error: true | |
| # Not possible to use ${{ github.event.number }} with the `merge_group` trigger | |
| - name: Get PR number from merge branch | |
| run: | | |
| echo "PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/')" | tee -a $GITHUB_ENV | |
| - name: Create file for issue | |
| if: steps.regression-check.outcome == 'failure' | |
| run: | | |
| printf '%s\n' "Regression >= 10% found during merge for PR #${{ env.PR_NUMBER }} | |
| Commit: ${{ github.sha }} | |
| Workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" > ./_body.md | |
| - name: Open issue on regression | |
| if: steps.regression-check.outcome == 'failure' | |
| uses: peter-evans/create-issue-from-file@v4 | |
| with: | |
| title: ':rotating_light: Performance regression detected for PR #${{ env.PR_NUMBER }}' | |
| content-filepath: ./_body.md | |
| labels: | | |
| P-Performance | |
| automated issue | |
| - name: Remove old main bench | |
| run: | | |
| rm ${{ env.BASE_COMMIT }}.json | |
| mv ${{ github.sha }}.json ${{ github.sha }}-${{ env.GPU_ID }}.json | |
| working-directory: ${{ github.workspace }} | |
| - name: Commit bench result to `gh-pages` branch if no regression | |
| if: steps.regression-check.outcome != 'failure' | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| branch: gh-pages | |
| commit_message: '[automated] GPU Benchmark from PR #${{ env.PR_NUMBER }}' | |
| file_pattern: '${{ github.sha }}-${{ env.GPU_ID }}.json' |