WIP: [ci] run container-canary tests on built images

.github/workflows/build-image.yml

@@ -120,46 +120,46 @@ jobs:
             PYTHON_VER=${{ inputs.PYTHON_VER }}
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
           tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
-      - name: Build RAFT ANN Benchmarks GPU image
-        uses: docker/build-push-action@v6
-        with:
-          context: context
-          file: raft-ann-bench/gpu/Dockerfile
-          target: raft-ann-bench
-          push: true
-          pull: true
-          build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
-            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-          tags: ${{ inputs.RAFT_ANN_BENCH_TAG }}-${{ matrix.ARCH }}
-      - name: Build RAFT ANN Benchmarks GPU with datasets image
-        uses: docker/build-push-action@v6
-        with:
-          context: context
-          file: raft-ann-bench/gpu/Dockerfile
-          target: raft-ann-bench-datasets
-          push: true
-          pull: true
-          build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
-            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-          tags: ${{ inputs.RAFT_ANN_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }}
-      - name: Build RAFT ANN Benchmarks CPU image
-        if: inputs.BUILD_RAFT_ANN_BENCH_CPU_IMAGE
-        uses: docker/build-push-action@v6
-        with:
-          context: context
-          file: raft-ann-bench/cpu/Dockerfile
-          target: raft-ann-bench-cpu
-          push: true
-          pull: true
-          build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
-            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-          tags: ${{ inputs.RAFT_ANN_BENCH_CPU_TAG }}-${{ matrix.ARCH }}
+      # - name: Build RAFT ANN Benchmarks GPU image
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     context: context
+      #     file: raft-ann-bench/gpu/Dockerfile
+      #     target: raft-ann-bench
+      #     push: true
+      #     pull: true
+      #     build-args: |
+      #       CUDA_VER=${{ inputs.CUDA_VER }}
+      #       LINUX_VER=${{ inputs.LINUX_VER }}
+      #       PYTHON_VER=${{ inputs.PYTHON_VER }}
+      #       RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+      #     tags: ${{ inputs.RAFT_ANN_BENCH_TAG }}-${{ matrix.ARCH }}
+      # - name: Build RAFT ANN Benchmarks GPU with datasets image
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     context: context
+      #     file: raft-ann-bench/gpu/Dockerfile
+      #     target: raft-ann-bench-datasets
+      #     push: true
+      #     pull: true
+      #     build-args: |
+      #       CUDA_VER=${{ inputs.CUDA_VER }}
+      #       LINUX_VER=${{ inputs.LINUX_VER }}
+      #       PYTHON_VER=${{ inputs.PYTHON_VER }}
+      #       RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+      #     tags: ${{ inputs.RAFT_ANN_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }}
+      # - name: Build RAFT ANN Benchmarks CPU image
+      #   if: inputs.BUILD_RAFT_ANN_BENCH_CPU_IMAGE
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     context: context
+      #     file: raft-ann-bench/cpu/Dockerfile
+      #     target: raft-ann-bench-cpu
+      #     push: true
+      #     pull: true
+      #     build-args: |
+      #       CUDA_VER=${{ inputs.CUDA_VER }}
+      #       LINUX_VER=${{ inputs.LINUX_VER }}
+      #       PYTHON_VER=${{ inputs.PYTHON_VER }}
+      #       RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+      #     tags: ${{ inputs.RAFT_ANN_BENCH_CPU_TAG }}-${{ matrix.ARCH }}

.github/workflows/build-test-publish-images.yml

@@ -225,6 +225,37 @@ jobs:
           GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
           ARCHES: ${{ toJSON(matrix.ARCHES) }}
         run: ci/create-multiarch-manifest.sh
+  validate:
+    needs: [build, build-multiarch-manifest]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
+      fail-fast: false
+    secrets: inherit
+    uses: ./.github/workflows/validate.yml
+    with:
+      ARCH: ${{ matrix.ARCH }}
+      CONTAINER_CANARY_VERSION: main
+      CUDA_VER: ${{ matrix.CUDA_VER }}
+      GPU: ${{ matrix.GPU }}
+      DRIVER: ${{ matrix.DRIVER }}
+      PYTHON_VER: ${{ matrix.PYTHON_VER }}
+      # images to test
+      BASE_TAG:
+        "docker.io/rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
+        ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
+        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
+        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
+        cuda${{ matrix.CUDA_VER }}-\
+        py${{ matrix.PYTHON_VER }}-\
+        ${{ matrix.ARCH }}"
+      NOTEBOOKS_TAG:
+        "docker.io/rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
+        ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
+        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
+        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
+        cuda${{ matrix.CUDA_VER }}-\
+        py${{ matrix.PYTHON_VER }}-\
+        ${{ matrix.ARCH }}"
   test:
     needs: [compute-matrix, build]
     if: inputs.run_tests

.github/workflows/validate.yml

@@ -0,0 +1,93 @@
+name: Validate images
+
+on:
+  workflow_call:
+    inputs:
+      ARCH:
+        required: true
+        type: string
+      # a tag from https://github.com/NVIDIA/container-canary/releases
+      CONTAINER_CANARY_VERSION:
+        description: 'tag from https://github.com/NVIDIA/container-canary/releases'
+        required: true
+        type: string
+      CUDA_VER:
+        required: true
+        type: string
+      DRIVER:
+        required: true
+        type: string
+      GPU:
+        required: true
+        type: string
+      PYTHON_VER:
+        required: true
+        type: string
+      BASE_TAG:
+        required: true
+        type: string
+      NOTEBOOKS_TAG:
+        required: true
+        type: string
+
+defaults:
+  run:
+    shell: sh
+
+permissions:
+  actions: read
+  checks: none
+  contents: read
+  deployments: none
+  discussions: none
+  id-token: write
+  issues: none
+  packages: read
+  pages: none
+  pull-requests: read
+  repository-projects: none
+  security-events: none
+  statuses: none
+
+jobs:
+  validate:
+    strategy:
+      matrix:
+        ARCH: ["${{ inputs.ARCH }}"]
+        CUDA_VER: ["${{ inputs.CUDA_VER }}"]
+        PYTHON_VER: ["${{ inputs.PYTHON_VER }}"]
+        GPU: ["${{ inputs.GPU }}"]
+        DRIVER: ["${{ inputs.DRIVER }}"]
+      fail-fast: false
+    runs-on: "linux-${{ inputs.ARCH }}-cpu4"
+    # container:
+    #   image: 'docker:dind'
+    #   options: --privileged
+    #   env:
+    #     NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22.x'
+      - name: Install container-canary
+        run: |
+          GOBIN=/tmp/canary-bin go install github.com/nvidia/container-canary@${{ inputs.CONTAINER_CANARY_VERSION }}
+          /tmp/canary-bin/container-canary version
+      - name: (base) container-canary checks
+        run: |
+          export PATH="/tmp/canary-bin:${PATH}"
+          sh ./ci/container-canary/run-checks.sh \
+            --dask-scheduler \
+            ${{ inputs.BASE_TAG }}
+      - name: (notebooks) container-canary checks
+        run: |
+          export PATH="/tmp/canary-bin:${PATH}"
+          sh ./ci/container-canary/run-checks.sh \
+            --dask-scheduler \
+            --notebooks \
+            ${{ inputs.NOTEBOOK_TAG }}

Dockerfile

@@ -52,7 +52,7 @@ COPY condarc /opt/conda/.condarc
 
 RUN <<EOF
 mamba install -y -n base \
-    "rapids=${RAPIDS_VER}.*" \
+    "cugraph=${RAPIDS_VER}.*" \
     "python=${PYTHON_VER}.*" \
     "cuda-version=${CUDA_VER%.*}.*" \
     ipython

ci/container-canary/README.md

@@ -0,0 +1,24 @@
+# container-canary
+
+Configurations for testing images built from this repo with `container-canary` ([NVIDIA/container-canary](https://github.com/NVIDIA/container-canary)).
+
+## Running the tests
+
+Install `container-canary` following the instructions in that project's repo.
+
+Run the tests against a built image.
+For example:
+
+```shell
+IMAGE_URI="rapidsai/notebooks:24.06a-cuda11.8-py3.11"
+
+# using a config checked in here
+canary validate \
+    --file ./ci/container-canary/rapids.yml \
+    "${IMAGE_URI}"
+
+# usage a config from the container-canary repo
+canary validate \
+    --file https://raw.githubusercontent.com/NVIDIA/container-canary/main/examples/databricks.yaml \
+    "${IMAGE_URI}"
+```

ci/container-canary/base.yml

@@ -0,0 +1,72 @@
+apiVersion: container-canary.nvidia.com/v1
+kind: Validator
+name: rapids-base
+description: |
+  Tests characteristics that the general-purpose RAPIDS images expected to have.
+documentation: https://github.com/rapidsai/docker
+# This command just ensures the container stays up long enough for
+# all checks to complete.
+command:
+  - /bin/sh
+  - -c
+  - "sleep 600"
+checks:
+  - name: tool-conda
+    description: conda can be executed
+    probe:
+      exec:
+        command:
+          - conda
+          - --version
+  - name: tool-dask-cli
+    description: Dask CLI can be executed
+    probe:
+      exec:
+        command:
+          - python
+          - -m
+          - dask
+          - --version
+      timeoutSeconds: 10
+  # ref: https://github.com/rapidsai/docker/issues/668
+  - name: tool-distributed-spec-cli
+    description: Distributed dask_spec CLI can be executed
+    probe:
+      exec:
+        command:
+          - python
+          - -m
+          - distributed.cli.dask_spec
+          - --version
+  - name: user-is-rapids
+    description: Default user is rapids (uid=1001)
+    probe:
+      exec:
+        command:
+          - /bin/sh
+          - -c
+          - 'test "$(id)" = "uid=1001(rapids) gid=1000(conda) groups=1000(conda)"'
+  - name: home-directory
+    description: $HOME is "/home/rapids"
+    probe:
+      exec:
+        command:
+          - /bin/sh
+          - -c
+          - 'test "$HOME" = "/home/rapids"'
+  - name: working-directory
+    description: Working directory is /home/rapids
+    probe:
+      exec:
+        command:
+          - /bin/sh
+          - -c
+          - 'test "$(pwd)" = "/home/rapids"'
+  - name: conda-only-base-env
+    description: The only defined conda env is "base"
+    probe:
+      exec:
+        command:
+          - /bin/bash
+          - -c
+          - "[[ $(conda env list --quiet | grep --count -E '^[A-Za-z]+') == 1 ]];"

ci/container-canary/notebooks.yml

@@ -0,0 +1,16 @@
+apiVersion: container-canary.nvidia.com/v1
+kind: Validator
+name: rapids-notebooks
+description: |
+  Tests characteristics that any RAPIDS images shipping Jupyter
+  are expected to have.
+documentation: https://github.com/rapidsai/docker
+checks:
+  - name: tool-jupyter-lab
+    description: jupyter lab can be executed
+    probe:
+      exec:
+        command:
+          - jupyter
+          - lab
+          - --version

ci/container-canary/run-checks.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e -E -u -o pipefail
+
+IMAGE_URI="${1}"
+
+NUMARGS=$#
+ARGS=$*
+
+function hasArg {
+    (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+# pre-pull
+docker pull "${IMAGE_URI}"
+
+check_configs=(
+    ./ci/container-canary/base.yml
+)
+
+if hasArg '--notebooks'; then
+    check_configs+=(./ci/container-canary/notebooks.yml)
+fi
+
+if hasArg '--dask-scheduler'; then
+    check_configs+=(https://raw.githubusercontent.com/NVIDIA/container-canary/main/examples/dask-scheduler.yaml)
+fi
+
+for check_config in "${check_configs[@]}"; do
+    echo "checking '${IMAGE_URI}' with '${check_config}'"
+    canary validate \
+        --file "${check_config}" \
+        --startup-timeout 60 \
+        "${IMAGE_URI}"
+done
+
+echo "done checking '${IMAGE_URI}' with container-canary"
+
+# usage a config from the container-canary repo
+canary validate \
+    --file https://raw.githubusercontent.com/NVIDIA/container-canary/main/examples/dask-scheduler.yaml \
+    "rapidsai/notebooks:24.10a-cuda12.2-py3.11"

matrix.yaml

@@ -1,6 +1,6 @@
 CUDA_VER: # Should be `<major>.<minor>.<patch>` (e.g. `11.2.2`)
-  - "11.8.0"
-  - "12.0.1"
+  # - "11.8.0"
+  # - "12.0.1"
   - "12.2.2"
   - "12.5.1"
 PYTHON_VER: