Merge branch 'branch-24.10' into container-canary

rapidsai · Aug 27, 2024 · c968434 · c968434
2 parents 802f7a6 + b007a92
commit c968434
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 116 deletions.
diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml
@@ -31,6 +31,25 @@ permissions:
   statuses: none
 
 jobs:
+  pr-builder:
+    if: ${{ !cancelled() && inputs.build_type == 'pull-request' }}
+    needs:
+      - checks
+      - compute-matrix
+      - build
+      - build-multiarch-manifest
+      - test
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Run pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit run --all-files
   compute-matrix:
     runs-on: ubuntu-latest
     container:
@@ -119,7 +138,7 @@ jobs:
 
           echo "TEST_MATRIX=$(yq -n -o json 'env(TEST_MATRIX)' | jq -c '{include: .}')" | tee --append "${GITHUB_OUTPUT}"
   build:
-    needs: compute-matrix
+    needs: [checks, compute-matrix]
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
       fail-fast: false
@@ -169,7 +188,6 @@ jobs:
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
         py${{ matrix.PYTHON_VER }}"
   build-multiarch-manifest:
-    if: ${{ !cancelled() && inputs.build_type == 'branch' }}
     needs: [build, compute-matrix]
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
@@ -239,7 +257,7 @@ jobs:
         py${{ matrix.PYTHON_VER }}-\
         ${{ matrix.ARCH }}"
   test:
-    needs: [compute-matrix, build, build-multiarch-manifest]
+    needs: [compute-matrix, build]
     if: inputs.run_tests
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
@@ -259,38 +277,5 @@ jobs:
         ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
         cuda${{ matrix.CUDA_VER }}-\
-        py${{ matrix.PYTHON_VER }}"
-  delete-temp-images:
-    if: ${{ !cancelled() && needs.test.result == 'success' }}
-    needs: [compute-matrix, build-multiarch-manifest, test]
-    strategy:
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Remove temporary images
-        shell: bash
-        env:
-          RAFT_ANN_BENCH_CPU_IMAGE_BUILT: ${{ matrix.BUILD_RAFT_ANN_BENCH_CPU_IMAGE }}
-          BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}
-          BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
-          RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-          ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }}
-          CUDA_TAG: ${{ matrix.CUDA_TAG }}
-          PYTHON_VER: ${{ matrix.PYTHON_VER }}
-          NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}
-          NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
-          RAFT_ANN_BENCH_IMAGE_REPO: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}
-          RAFT_ANN_BENCH_TAG_PREFIX: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
-          RAFT_ANN_BENCH_DATASETS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}
-          RAFT_ANN_BENCH_DATASETS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
-          RAFT_ANN_BENCH_CPU_IMAGE_REPO: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}
-          RAFT_ANN_BENCH_CPU_TAG_PREFIX: ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
-          GPUCIBOT_DOCKERHUB_USER: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
-          GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
-          ARCHES: ${{ toJSON(matrix.ARCHES) }}
-        run: ci/delete-temp-images.sh
+        py${{ matrix.PYTHON_VER }}-\
+        ${{ matrix.ARCH }}"
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -10,26 +10,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pr-builder:
-    needs:
-      - checks
-      - docker
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
-  checks:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-      - name: Run pre-commit
-        run: |
-          pip install pre-commit
-          pre-commit run --all-files
   docker:
-    needs: [checks]
     uses: ./.github/workflows/build-test-publish-images.yml
     with:
       build_type: pull-request

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -15,3 +15,14 @@ To build just the `base` image with default arguments: `docker buildx build --pu
 - `CUDA_VER` - Version of CUDA to use. Should be `major.minor.patch`
 - `PYTHON_VER` - Version of Python to use. Should be `major.minor`
 - `RAPIDS_VER` - Version of RAPIDS to use. Should be `YY.MM`
+
+## Cleaning Up
+
+Every build first writes images to the https://hub.docker.com/r/rapidsai/staging repo on DockerHub,
+then pushes them on to the individual repos like `rapidsai/base`, `rapidsai/notebooks`, etc.
+
+A scheduled job regularly deletes old images from that `rapidsai/staging` repo.
+See https://github.com/rapidsai/workflows/blob/main/.github/workflows/cleanup_staging.yaml for details.
+
+If you come back to a pull requests here after more than a few days and find that jobs are failing with errors
+that suggest that some necessary images don't exist, re-run all of CI on that pull request to produce new images.
diff --git a/ci/delete-temp-images.sh b/ci/delete-temp-images.sh
diff --git a/matrix-test.yaml b/matrix-test.yaml
@@ -1,14 +1,14 @@
 # CUDA_VER is `<major>.<minor>` (e.g. `12.0`)
 
 pull-request:
-  # - { CUDA_VER: '11.8', ARCH: 'amd64', PYTHON_VER: '3.9', GPU: 'v100', DRIVER: 'earliest' }
-  # - { CUDA_VER: '12.0', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'latest' }
+  - { CUDA_VER: '11.8', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'earliest' }
+  - { CUDA_VER: '12.0', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'latest' }
   - { CUDA_VER: '12.2', ARCH: 'arm64', PYTHON_VER: '3.11', GPU: 'a100', DRIVER: 'latest' }
   - { CUDA_VER: '12.5', ARCH: 'amd64', PYTHON_VER: '3.11', GPU: 'v100', DRIVER: 'latest' }
 branch:
-  # - { CUDA_VER: '11.8', ARCH: 'amd64', PYTHON_VER: '3.9', GPU: 'v100', DRIVER: 'earliest' }
-  # - { CUDA_VER: '11.8', ARCH: 'amd64', PYTHON_VER: '3.9', GPU: 'v100', DRIVER: 'latest' }
-  # - { CUDA_VER: '12.0', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'latest' }
-  # - { CUDA_VER: '12.0', ARCH: 'arm64', PYTHON_VER: '3.10', GPU: 'a100', DRIVER: 'latest' }
+  - { CUDA_VER: '11.8', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'earliest' }
+  - { CUDA_VER: '11.8', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'latest' }
+  - { CUDA_VER: '12.0', ARCH: 'amd64', PYTHON_VER: '3.10', GPU: 'v100', DRIVER: 'latest' }
+  - { CUDA_VER: '12.0', ARCH: 'arm64', PYTHON_VER: '3.10', GPU: 'a100', DRIVER: 'latest' }
   - { CUDA_VER: '12.2', ARCH: 'amd64', PYTHON_VER: '3.11', GPU: 'v100', DRIVER: 'latest' }
   - { CUDA_VER: '12.5', ARCH: 'arm64', PYTHON_VER: '3.11', GPU: 'a100', DRIVER: 'latest' }
diff --git a/matrix.yaml b/matrix.yaml
@@ -4,6 +4,5 @@ CUDA_VER: # Should be `<major>.<minor>.<patch>` (e.g. `11.2.2`)
   - "12.2.2"
   - "12.5.1"
 PYTHON_VER:
-  # - "3.9"
-  # - "3.10"
+  - "3.10"
   - "3.11"
diff --git a/raft-ann-bench/cpu/Dockerfile b/raft-ann-bench/cpu/Dockerfile
@@ -23,7 +23,7 @@ EOF
 # we need perl temporarily for the remaining benchmark perl scripts
 RUN apt-get install perl -y
 
-# Install python before updating environment, otherwise Python 3.9 image
+# Install python before updating environment, otherwise Python 3 image
 # runs into a solver conflict with truststore 0.8.0. This avoids the environment installing
 # packages incompatible with python version needed before python itself is pinned to the correct version.
 RUN <<EOF