From 280cf87c42ef1723982468f4f9e3144bda404a26 Mon Sep 17 00:00:00 2001 From: Samuel Burnham <45365069+samuelburnham@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:23:52 -0400 Subject: [PATCH 1/2] Reorganize CI to minimize unneeded jobs --- .circleci/config.yml | 212 ------------------ .../{pr-bench.yml => bench_pr_comment.yml} | 1 + .github/workflows/{pr-test.yml => ci.yml} | 6 +- .github/workflows/ci_ignored.yml | 23 ++ .github/workflows/{rust.yml => gpu.yml} | 94 +++----- .github/workflows/nightly.yml | 47 ++++ 6 files changed, 105 insertions(+), 278 deletions(-) rename .github/workflows/{pr-bench.yml => bench_pr_comment.yml} (94%) rename .github/workflows/{pr-test.yml => ci.yml} (97%) create mode 100644 .github/workflows/ci_ignored.yml rename .github/workflows/{rust.yml => gpu.yml} (58%) create mode 100644 .github/workflows/nightly.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 18f89ae5d2..7ef6d59ac8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,16 +8,6 @@ setup-env: &setup-env RUST_LOG: info executors: - default: - machine: - image: ubuntu-2004:202101-01 - working_directory: ~/lurk - resource_class: xlarge - arm64: - machine: - image: ubuntu-2004:202101-01 - working_directory: ~/lurk - resource_class: arm.xlarge darwin: macos: xcode: "14.3.1" @@ -99,128 +89,11 @@ commands: cargo --version rustc --version - save_rustup_cache: - steps: - - save_cache: - name: "Save rustup cache" - key: cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - paths: - - "~/.cargo" - - "~/.rustup" - - restore_rustup_cache: - steps: - - restore_cache: - keys: - - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - install_gpu_deps: steps: - run: sudo apt update && sudo apt install -y ocl-icd-opencl-dev curl build-essential jobs: - cargo_fetch: - executor: default - environment: *setup-env - steps: - - checkout - - run: - name: Update submodules - command: | - git submodule update --init --recursive - - restore_rustup_cache - - set_env_path - - set_versions_n_runners - - run: cargo fetch - - run: rm -rf .git - - persist_to_workspace: - root: ~/ - paths: - - lurk - - save_rustup_cache - - linux_release: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "~/" - - restore_rustup_cache - - set_env_path - - setup-sccache - - install_gpu_deps - - restore-sccache-cache - - run: - name: Linux Tests - command: cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored ignored-only -E 'all() - test(groth16::tests::outer_prove_recursion) - test(test_make_fcomm_examples) - test(test_functional_commitments_demo) - test(test_chained_functional_commitments_demo)' - - run: - name: Benches build successfully - command: cargo bench --no-run --profile dev-ci - - run: - name: Linux Doc Tests - command: cargo test --doc --workspace --profile dev-ci - - save-sccache-cache - - run: - name: Print sccache stats - command: sccache -s - - linux_exhaustive: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "~/" - - restore_rustup_cache - - set_env_path - - install_gpu_deps - - restore-sccache-cache - - run: - name: Linux Tests - command: cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored all - - run: - name: Benches build successfully - command: cargo bench --no-run --profile dev-ci - - run: - name: Linux Doc Tests - command: cargo test --doc --workspace --profile dev-ci - - save-sccache-cache - - run: - name: Print sccache stats - command: sccache -s - - arm64: - executor: arm64 - environment: *setup-env - steps: - - checkout - - run: - name: Update submodules - command: | - git submodule update --init --recursive - - attach_workspace: - at: "~/" - - set_versions_n_runners - - set_env_path - - install_gpu_deps - - setup-sccache - - restore-sccache-cache - - run: - name: Arm64 Tests - command: cargo nextest run --profile ci --workspace --cargo-profile dev-ci - no_output_timeout: 120m - - run: - name: Benches build successfully - command: cargo bench --no-run --profile dev-ci - - run: - name: Arm64 Doc Tests - command: cargo test --doc --workspace --profile dev-ci - - save-sccache-cache - - run: - name: Print sccache stats - command: sccache -s - mac: executor: darwin environment: *setup-env @@ -250,48 +123,9 @@ jobs: name: Print sccache stats command: sccache -s - clippy: - executor: default - environment: *setup-env - steps: - - checkout - - set_env_path - - attach_workspace: - at: "~/" - - restore_rustup_cache - - run: cargo clippy --all -- -D warnings - - rustfmt: - executor: default - environment: *setup-env - steps: - - checkout - - set_env_path - - attach_workspace: - at: "~/" - - restore_rustup_cache - - run: cargo fmt --all -- --check - workflows: test-all: jobs: - - cargo_fetch: - filters: - branches: - ignore: - - gh-pages - - staging.tmp - - trying.tmp - - staging-squash-merge.tmp - - arm64: - filters: - branches: - ignore: - - master - - gh-pages - - staging.tmp - - trying.tmp - - staging-squash-merge.tmp - mac: filters: branches: @@ -301,49 +135,3 @@ workflows: - staging.tmp - trying.tmp - staging-squash-merge.tmp - - linux_release: - requires: - - cargo_fetch - filters: - branches: - ignore: - - master - - gh-pages - - staging.tmp - - trying.tmp - - staging-squash-merge.tmp - - rustfmt: - requires: - - cargo_fetch - filters: - branches: - ignore: - - master - - gh-pages - - staging.tmp - - trying.tmp - - staging-squash-merge.tmp - - clippy: - requires: - - cargo_fetch - filters: - branches: - ignore: - - master - - gh-pages - - staging.tmp - - trying.tmp - - staging-squash-merge.tmp - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - cargo_fetch - - linux_exhaustive: - requires: - - cargo_fetch diff --git a/.github/workflows/pr-bench.yml b/.github/workflows/bench_pr_comment.yml similarity index 94% rename from .github/workflows/pr-bench.yml rename to .github/workflows/bench_pr_comment.yml index 29b944669f..9ace2bd86c 100644 --- a/.github/workflows/pr-bench.yml +++ b/.github/workflows/bench_pr_comment.yml @@ -1,3 +1,4 @@ +# Creates a PR benchmark comment with a comparison to master name: Benchmark pull requests on: issue_comment: diff --git a/.github/workflows/pr-test.yml b/.github/workflows/ci.yml similarity index 97% rename from .github/workflows/pr-test.yml rename to .github/workflows/ci.yml index 39429d0c8d..54977951e2 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Lurk CI tests +name: CI tests on: merge_group: @@ -52,7 +52,7 @@ jobs: - name: Linux Gadget Tests w/o debug assertions run: | cargo nextest run --profile ci --workspace --cargo-profile dev-no-assertions -E 'test(circuit::gadgets)' - + misc: runs-on: ${{ matrix.os }} strategy: @@ -77,7 +77,7 @@ jobs: run: cargo build --benches - name: Doctests run: | - cargo test --doc + cargo test --doc --workspace --profile dev-ci clippy: runs-on: ubuntu-latest diff --git a/.github/workflows/ci_ignored.yml b/.github/workflows/ci_ignored.yml new file mode 100644 index 0000000000..97c3d17861 --- /dev/null +++ b/.github/workflows/ci_ignored.yml @@ -0,0 +1,23 @@ +# Run ignored tests only when attempting to merge +name: Linux ignored tests + +on: + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + linux_ignored: + runs-on: [self-hosted, test] + env: + RUSTFLAGS: -D warnings + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + - uses: taiki-e/install-action@nextest + - uses: Swatinem/rust-cache@v2 + - name: Linux Tests + run: | + cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored ignored-only -E 'all() - test(groth16::tests::outer_prove_recursion) - test(test_make_fcomm_examples) - test(test_functional_commitments_demo) - test(test_chained_functional_commitments_demo)' diff --git a/.github/workflows/rust.yml b/.github/workflows/gpu.yml similarity index 58% rename from .github/workflows/rust.yml rename to .github/workflows/gpu.yml index 52a42fe912..ea4246ffaa 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/gpu.yml @@ -1,9 +1,11 @@ -name: Rust +# Runs the test suite on a self-hosted GPU machine with CUDA and OpenCL enabled +name: GPU tests on: push: branches: - master + env: CARGO_TERM_COLOR: always # Disable incremental compilation. @@ -25,21 +27,19 @@ env: RUSTUP_MAX_RETRIES: 10 # Don't emit giant backtraces in the CI logs. RUST_BACKTRACE: short + RUSTFLAGS: -D warnings concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - ubuntu-latest - fail-fast: false + cuda: + name: Rust tests on CUDA + runs-on: [self-hosted, gpu] env: - RUSTFLAGS: -D warnings + NVIDIA_VISIBLE_DEVICES: all + NVIDIA_DRIVER_CAPABILITITES: compute,utility steps: - uses: actions/checkout@v3 with: @@ -47,34 +47,28 @@ jobs: - uses: actions-rs/toolchain@v1 - uses: taiki-e/install-action@nextest - uses: Swatinem/rust-cache@v2 - # make sure benches don't bit-rot - - name: build benches - run: cargo build --benches --release - - name: cargo test - run: | - cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored all -E 'all() - test(groth16::tests::outer_prove_recursion) - test(test_make_fcomm_examples) - test(test_functional_commitments_demo) - test(test_chained_functional_commitments_demo)' - - name: Doctests + # Check we have access to the machine's Nvidia drivers + - run: nvidia-smi + # The `compute`/`sm` number corresponds to the Nvidia GPU architecture + # In this case, the self-hosted machine uses the Ampere architecture, but we want this to be configurable + # See https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ + - name: Set env for CUDA compute + run: echo "CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed 's/\.//g')" >> $GITHUB_ENV + - name: set env for EC_GPU + run: echo 'EC_GPU_CUDA_NVCC_ARGS=--fatbin --gpu-architecture=sm_${{ env.CUDA_ARCH }} --generate-code=arch=compute_${{ env.CUDA_ARCH }},code=sm_${{ env.CUDA_ARCH }}' >> $GITHUB_ENV + - run: echo "${{ env.EC_GPU_CUDA_NVCC_ARGS}}" + # Check that CUDA is installed with a driver-compatible version + # This must also be compatible with the GPU architecture, see above link + - run: nvcc --version + - name: CUDA tests + env: + EC_GPU_FRAMEWORK: cuda + # Temporarily skipping CLI test due to cudaMallocAsync error: https://github.com/lurk-lab/lurk-rs/issues/596 run: | - cargo test --doc - - msrv: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install rustup - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: Swatinem/rust-cache@v2 - - name: Install cargo-msrv - run: cargo install cargo-msrv - - name: Check Rust MSRV - run: cargo msrv verify + cargo nextest run --profile ci --cargo-profile dev-ci --features cuda -E 'all() - test(test_prove_and_verify)' - # Runs the test suite on a self-hosted GPU machine with CUDA enabled - test-gpu: - name: Rust tests on CUDA + opencl: + name: Rust tests on OpenCL runs-on: [self-hosted, gpu] env: NVIDIA_VISIBLE_DEVICES: all @@ -99,37 +93,11 @@ jobs: # Check that CUDA is installed with a driver-compatible version # This must also be compatible with the GPU architecture, see above link - run: nvcc --version - # The use of `gcc-10`/`g++-10` is a hack to work around a pasta-msm error with CUDA 11.5 and gcc 11.3: - # https://github.com/NVlabs/instant-ngp/issues/119#issuecomment-1231890237 - # We could also update to CUDA 11.8+ but it breaks the self-hosted runner - - name: CUDA tests - env: - EC_GPU_FRAMEWORK: cuda - run: | - CC=gcc-10 CXX=g++-10 cargo nextest run --profile ci --cargo-profile dev-ci --features cuda # Check that we can access the OpenCL headers - run: clinfo - name: OpenCL tests env: EC_GPU_FRAMEWORK: opencl + # Temporarily skipping CLI test due to cudaMallocAsync error: https://github.com/lurk-lab/lurk-rs/issues/596 run: | - CC=gcc-10 CXX=g++-10 cargo nextest run --profile ci --cargo-profile dev-ci --features cuda,opencl - - rustfmt_n_clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - uses: actions-rs/toolchain@v1 - with: - components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 - - name: rustfmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all --check - # See '.cargo/config' for list of enabled/disabled clippy lints - - name: cargo clippy - run: cargo xclippy -D warnings + cargo nextest run --profile ci --cargo-profile dev-ci --features cuda,opencl -E 'all() - test(test_prove_and_verify)' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..c08363ec27 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,47 @@ +name: Nightly tests on master + +on: + schedule: + - cron: '0 0 * * *' + +env: + CARGO_TERM_COLOR: always + # Disable incremental compilation. + # + # Incremental compilation is useful as part of an edit-build-test-edit cycle, + # as it lets the compiler avoid recompiling code that hasn't changed. However, + # on CI, we're not making small edits; we're almost always building the entire + # project from scratch. Thus, incremental compilation on CI actually + # introduces *additional* overhead to support making future builds + # faster...but no future builds will ever occur in any given CI environment. + # + # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow + # for details. + CARGO_INCREMENTAL: 0 + # Allow more retries for network requests in cargo (downloading crates) and + # rustup (installing toolchains). This should help to reduce flaky CI failures + # from transient network timeouts or other issues. + CARGO_NET_RETRY: 10 + RUSTUP_MAX_RETRIES: 10 + # Don't emit giant backtraces in the CI logs. + RUST_BACKTRACE: short + +jobs: + linux_exhaustive: + runs-on: [self-hosted, test] + env: + RUSTFLAGS: -D warnings + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + - uses: taiki-e/install-action@nextest + - uses: Swatinem/rust-cache@v2 + - name: Linux Tests + run: | + cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored all + - name: Benches build successfully + run: | + cargo bench --no-run --profile dev-ci + - name: Linux Doc Tests + run: | + cargo test --doc --workspace --profile dev-ci From 44605c4a9cf193d0d2363208b5b61dc24c3dbf63 Mon Sep 17 00:00:00 2001 From: Samuel Burnham <45365069+samuelburnham@users.noreply.github.com> Date: Sun, 13 Aug 2023 12:18:22 -0400 Subject: [PATCH 2/2] Add back arm64 in matrix job --- .circleci/config.yml | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ef6d59ac8..07be99e0c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,11 @@ setup-env: &setup-env RUST_LOG: info executors: + arm64: + machine: + image: ubuntu-2004:202101-01 + working_directory: ~/lurk + resource_class: arm.xlarge darwin: macos: xcode: "14.3.1" @@ -89,44 +94,47 @@ commands: cargo --version rustc --version - install_gpu_deps: - steps: - - run: sudo apt update && sudo apt install -y ocl-icd-opencl-dev curl build-essential - jobs: - mac: - executor: darwin - environment: *setup-env + test: + parameters: + os: + type: executor + executor: << parameters.os >> steps: - checkout - run: name: Update submodules - command: git submodule update --init --recursive + command: | + git submodule update --init --recursive - attach_workspace: - at: "~/" + at: "~/" - set_versions_n_runners - set_env_path - setup-sccache - restore-sccache-cache - run: - name: MacOS Tests + name: Run cargo tests command: cargo nextest run --profile ci --workspace --cargo-profile dev-ci no_output_timeout: 120m - run: name: Benches build successfully command: cargo bench --no-run --profile dev-ci - run: - name: MacOS Doc Tests + name: Doc Tests command: cargo test --doc --workspace --profile dev-ci - save-sccache-cache - run: name: Print sccache stats command: sccache -s + workflows: test-all: jobs: - - mac: + - test: + matrix: + parameters: + os: [arm64, darwin] filters: branches: ignore: