From 9981da1a902fa46d69460226e24290b8ecc897ad Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Sep 2024 21:29:15 -0500 Subject: [PATCH 01/15] Update fmt (to 11.0.2) and spdlog (to 1.14.1). --- rapids_config.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rapids_config.cmake b/rapids_config.cmake index f064a5b497..e94ca30876 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -27,6 +27,8 @@ else() endif() if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUML_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") + set(rapids-cmake-repo jameslamb/rapids-cmake) + set(rapids-cmake-branch fmt-and-spdlog) file( DOWNLOAD "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" From 67fbab3c6328cd276164f2b677e4099965a58b53 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 17 Sep 2024 21:59:40 -0500 Subject: [PATCH 02/15] Update fmt (to 11.0.2) and spdlog (to 1.14.1). --- ci/build_cpp.sh | 21 ++++++++++++++- ci/build_python.sh | 25 ++++++++++++++++++ ci/build_wheel.sh | 4 +++ ci/test_cpp.sh | 21 ++++++++++++++- ci/test_notebooks.sh | 21 ++++++++++++++- ci/test_python_common.sh | 21 ++++++++++++++- ci/test_wheel.sh | 4 +++ ci/use_wheels_from_prs.sh | 55 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 ci/use_wheels_from_prs.sh diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 48f4f444b2..566c4ce4d4 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -15,6 +15,25 @@ rapids-print-env rapids-logger "Begin cpp build" -RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcuml +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + +RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${UCXX_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ + conda/recipes/libcuml rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index e1397e3373..7f3a83a366 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -19,11 +19,29 @@ rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${UCXX_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/cuml # Build cuml-cpu only in CUDA 11 jobs since it only depends on python @@ -32,6 +50,13 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${UCXX_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ + --channel "${CUDF_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${RAFT_CHANNEL}" \ conda/recipes/cuml-cpu fi diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index af3a4c124b..cdb6da13d8 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -14,6 +14,10 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_wheels_from_prs.sh +export PIP_CONSTRAINT=/tmp/constraints.txt + rapids-generate-version > ./VERSION cd ${package_dir} diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index ea6d1cdc11..98a1aff7c9 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -11,12 +11,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ - --prepend-channel "${CPP_CHANNEL}" | tee env.yaml + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${LIBRMM_CHANNEL}" \ + --prepend-channel "${RMM_CHANNEL}" \ + --prepend-channel "${UCXX_CHANNEL}" \ + --prepend-channel "${LIBCUDF_CHANNEL}" \ + --prepend-channel "${CUDF_CHANNEL}" \ + --prepend-channel "${LIBRAFT_CHANNEL}" \ + --prepend-channel "${RAFT_CHANNEL}" \ +| tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index d76a754b8b..2b08d9a621 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -8,13 +8,32 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + rapids-logger "Generate Notebook testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_notebooks \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml + --prepend-channel "${PYTHON_CHANNEL}" \ + --prepend-channel "${LIBRMM_CHANNEL}" \ + --prepend-channel "${RMM_CHANNEL}" \ + --prepend-channel "${UCXX_CHANNEL}" \ + --prepend-channel "${LIBCUDF_CHANNEL}" \ + --prepend-channel "${CUDF_CHANNEL}" \ + --prepend-channel "${LIBRAFT_CHANNEL}" \ + --prepend-channel "${RAFT_CHANNEL}" \ +| tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 5f1894356c..6ccc6d74c2 100644 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -9,13 +9,32 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_python \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml + --prepend-channel "${PYTHON_CHANNEL}" \ + --prepend-channel "${LIBRMM_CHANNEL}" \ + --prepend-channel "${RMM_CHANNEL}" \ + --prepend-channel "${UCXX_CHANNEL}" \ + --prepend-channel "${LIBCUDF_CHANNEL}" \ + --prepend-channel "${CUDF_CHANNEL}" \ + --prepend-channel "${LIBRAFT_CHANNEL}" \ + --prepend-channel "${RAFT_CHANNEL}" \ +| tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 9f473b1890..896f2608a8 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,6 +3,10 @@ set -euo pipefail +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_wheels_from_prs.sh +export PIP_CONSTRAINT=/tmp/constraints.txt + mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh new file mode 100644 index 0000000000..940bdadeca --- /dev/null +++ b/ci/use_wheels_from_prs.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +LIBRMM_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp +) +RMM_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python +) + +UCXX_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python +) +LIBUCXX_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=libucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 cpp +) +DISTRIBUTED_UCXX_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python +) + + +CUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=cudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python +) +LIBCUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=libcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 cpp +) +PYLIBCUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=pylibcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python +) +DASK_CUDF_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=dask_cudf_${RAPIDS_PY_CUDA_SUFFIX} \ + RAPIDS_PY_WHEEL_PURE=1 \ + rapids-get-pr-wheel-artifact cudf 16806 python +) + +RAFT_DASK_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=raft_dask_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python +) +PYLIBRAFT_CHANNEL=$( + RAPIDS_PY_WHEEL_NAME=pylibraft_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python +) + +cat > /tmp/constraints.txt < Date: Tue, 17 Sep 2024 22:03:42 -0500 Subject: [PATCH 03/15] add cumlprims_mg --- ci/build_cpp.sh | 3 +++ ci/build_python.sh | 3 +++ ci/test_cpp.sh | 3 +++ ci/test_notebooks.sh | 1 + ci/test_python_common.sh | 3 +++ 5 files changed, 13 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 566c4ce4d4..808a3c24b9 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -26,6 +26,8 @@ CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) +CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) + RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \ --channel "${LIBRMM_CHANNEL}" \ --channel "${RMM_CHANNEL}" \ @@ -34,6 +36,7 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild --channel "${CUDF_CHANNEL}" \ --channel "${LIBRAFT_CHANNEL}" \ --channel "${RAFT_CHANNEL}" \ + --channel "${CUMLPRIMS_CHANNEL}" \ conda/recipes/libcuml rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 7f3a83a366..cb1d174378 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -30,6 +30,8 @@ CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) +CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) + # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ @@ -42,6 +44,7 @@ RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --channel "${CUDF_CHANNEL}" \ --channel "${LIBRAFT_CHANNEL}" \ --channel "${RAFT_CHANNEL}" \ + --channel "${CUMLPRIMS_CHANNEL}" \ conda/recipes/cuml # Build cuml-cpu only in CUDA 11 jobs since it only depends on python diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 98a1aff7c9..ab0e6f8cdf 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -22,6 +22,8 @@ CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) +CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -35,6 +37,7 @@ rapids-dependency-file-generator \ --prepend-channel "${CUDF_CHANNEL}" \ --prepend-channel "${LIBRAFT_CHANNEL}" \ --prepend-channel "${RAFT_CHANNEL}" \ + --prepend-channel "${CUMLPRIMS_CHANNEL}" \ | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 2b08d9a621..74cc439470 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -33,6 +33,7 @@ rapids-dependency-file-generator \ --prepend-channel "${CUDF_CHANNEL}" \ --prepend-channel "${LIBRAFT_CHANNEL}" \ --prepend-channel "${RAFT_CHANNEL}" \ + --prepend-channel "${CUMLPRIMS_CHANNEL}" \ | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 6ccc6d74c2..19fc7ae629 100644 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -20,6 +20,8 @@ CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) +CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -34,6 +36,7 @@ rapids-dependency-file-generator \ --prepend-channel "${CUDF_CHANNEL}" \ --prepend-channel "${LIBRAFT_CHANNEL}" \ --prepend-channel "${RAFT_CHANNEL}" \ + --prepend-channel "${CUMLPRIMS_CHANNEL}" \ | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test From 96876110e795df1a063362679a4d7dc5218c2855 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Sep 2024 09:44:57 -0700 Subject: [PATCH 04/15] move rapids-cmake overrides [skip ci] --- rapids_config.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rapids_config.cmake b/rapids_config.cmake index e94ca30876..4524465a69 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -26,9 +26,9 @@ else() ) endif() +set(rapids-cmake-repo jameslamb/rapids-cmake) +set(rapids-cmake-branch fmt-and-spdlog) if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUML_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") - set(rapids-cmake-repo jameslamb/rapids-cmake) - set(rapids-cmake-branch fmt-and-spdlog) file( DOWNLOAD "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" From 1ca0b5fc3dc85f14853101cb7609a5221ae7dabc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Sep 2024 23:57:19 -0500 Subject: [PATCH 05/15] fix testing --- .github/workflows/pr.yaml | 182 +++++++++++++++--------------- ci/build_cpp.sh | 24 +--- ci/build_python.sh | 31 +---- ci/test_cpp.sh | 23 +--- ci/test_notebooks.sh | 20 +--- ci/test_python_common.sh | 22 +--- ci/use_conda_packages_from_prs.sh | 23 ++++ ci/use_wheels_from_prs.sh | 27 +++-- 8 files changed, 139 insertions(+), 213 deletions(-) create mode 100755 ci/use_conda_packages_from_prs.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d28a073775..c1eb2ec54f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,108 +12,108 @@ concurrency: jobs: pr-builder: needs: - - checks - - clang-tidy + # - checks + # - clang-tidy - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks + # - conda-cpp-tests + # - conda-cpp-checks - conda-python-build - - conda-python-tests-singlegpu - - conda-python-tests-dask - - conda-notebook-tests - - docs-build + # - conda-python-tests-singlegpu + # - conda-python-tests-dask + # - conda-notebook-tests + # - docs-build - wheel-build-cuml - - wheel-tests-cuml + # - wheel-tests-cuml - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 - with: - enable_check_generated_files: false - ignored_pr_jobs: >- - optional-job-conda-python-tests-cudf-pandas-integration - clang-tidy: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 - with: - build_type: pull-request - node_type: "cpu8" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/run_clang_tidy.sh" + # checks: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 + # with: + # enable_check_generated_files: false + # ignored_pr_jobs: >- + # optional-job-conda-python-tests-cudf-pandas-integration + # clang-tidy: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + # with: + # build_type: pull-request + # node_type: "cpu8" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + # run_script: "ci/run_clang_tidy.sh" conda-cpp-build: - needs: checks + # needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: pull-request - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: raft_cutlass + # conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + # with: + # build_type: pull-request + # conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + # with: + # build_type: pull-request + # enable_check_symbols: true + # symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 with: build_type: pull-request - conda-python-tests-singlegpu: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 - with: - build_type: pull-request - script: "ci/test_python_singlegpu.sh" - optional-job-conda-python-tests-cudf-pandas-integration: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 - with: - matrix_filter: map(select(.ARCH == "amd64")) - build_type: pull-request - script: "ci/test_python_integration.sh" - conda-python-tests-dask: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 - with: - build_type: pull-request - script: "ci/test_python_dask.sh" - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" + # conda-python-tests-singlegpu: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + # with: + # build_type: pull-request + # script: "ci/test_python_singlegpu.sh" + # optional-job-conda-python-tests-cudf-pandas-integration: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + # with: + # matrix_filter: map(select(.ARCH == "amd64")) + # build_type: pull-request + # script: "ci/test_python_integration.sh" + # conda-python-tests-dask: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + # with: + # build_type: pull-request + # script: "ci/test_python_dask.sh" + # conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_notebooks.sh" + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" wheel-build-cuml: - needs: checks + # needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: @@ -122,13 +122,13 @@ jobs: extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-24.10 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - wheel-tests-cuml: - needs: wheel-build-cuml - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 - with: - build_type: pull-request - script: ci/test_wheel.sh + # wheel-tests-cuml: + # needs: wheel-build-cuml + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + # with: + # build_type: pull-request + # script: ci/test_wheel.sh devcontainer: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@python-3.12 diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 808a3c24b9..826f4de391 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -11,32 +11,14 @@ source rapids-date-string export CMAKE_GENERATOR=Ninja +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_conda_packages_from_prs.sh + rapids-print-env rapids-logger "Begin cpp build" -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) - -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) -CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) - -LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) -RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) - -CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) - RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ - --channel "${LIBCUDF_CHANNEL}" \ - --channel "${CUDF_CHANNEL}" \ - --channel "${LIBRAFT_CHANNEL}" \ - --channel "${RAFT_CHANNEL}" \ - --channel "${CUMLPRIMS_CHANNEL}" \ conda/recipes/libcuml rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index cb1d174378..b4b1b6bd0b 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,6 +11,9 @@ source rapids-date-string export CMAKE_GENERATOR=Ninja +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ +source ./ci/use_conda_packages_from_prs.sh + rapids-print-env rapids-generate-version > ./VERSION @@ -19,32 +22,11 @@ rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) - -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) -CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) - -LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) -RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) - -CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) - # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ - --channel "${LIBCUDF_CHANNEL}" \ - --channel "${CUDF_CHANNEL}" \ - --channel "${LIBRAFT_CHANNEL}" \ - --channel "${RAFT_CHANNEL}" \ - --channel "${CUMLPRIMS_CHANNEL}" \ conda/recipes/cuml # Build cuml-cpu only in CUDA 11 jobs since it only depends on python @@ -53,13 +35,6 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ --no-test \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ - --channel "${UCXX_CHANNEL}" \ - --channel "${LIBCUDF_CHANNEL}" \ - --channel "${CUDF_CHANNEL}" \ - --channel "${LIBRAFT_CHANNEL}" \ - --channel "${RAFT_CHANNEL}" \ conda/recipes/cuml-cpu fi diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index ab0e6f8cdf..8a7849b4a9 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,36 +8,17 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh +source ./ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) - -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) -CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) - -LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) -RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) - -CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) - rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${LIBRMM_CHANNEL}" \ - --prepend-channel "${RMM_CHANNEL}" \ - --prepend-channel "${UCXX_CHANNEL}" \ - --prepend-channel "${LIBCUDF_CHANNEL}" \ - --prepend-channel "${CUDF_CHANNEL}" \ - --prepend-channel "${LIBRAFT_CHANNEL}" \ - --prepend-channel "${RAFT_CHANNEL}" \ - --prepend-channel "${CUMLPRIMS_CHANNEL}" \ | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 74cc439470..05a38da286 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -3,22 +3,12 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +source ./ci/use_conda_packages_from_prs.sh rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) - -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) -CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) - -LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) -RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) - rapids-logger "Generate Notebook testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -26,14 +16,6 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ - --prepend-channel "${LIBRMM_CHANNEL}" \ - --prepend-channel "${RMM_CHANNEL}" \ - --prepend-channel "${UCXX_CHANNEL}" \ - --prepend-channel "${LIBCUDF_CHANNEL}" \ - --prepend-channel "${CUDF_CHANNEL}" \ - --prepend-channel "${LIBRAFT_CHANNEL}" \ - --prepend-channel "${RAFT_CHANNEL}" \ - --prepend-channel "${CUMLPRIMS_CHANNEL}" \ | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 19fc7ae629..85bef07828 100644 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -4,24 +4,12 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +source ./ci/use_conda_packages_from_prs.sh rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) - -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) -CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) - -LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) -RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) - -CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) - rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ @@ -29,14 +17,6 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ - --prepend-channel "${LIBRMM_CHANNEL}" \ - --prepend-channel "${RMM_CHANNEL}" \ - --prepend-channel "${UCXX_CHANNEL}" \ - --prepend-channel "${LIBCUDF_CHANNEL}" \ - --prepend-channel "${CUDF_CHANNEL}" \ - --prepend-channel "${LIBRAFT_CHANNEL}" \ - --prepend-channel "${RAFT_CHANNEL}" \ - --prepend-channel "${CUMLPRIMS_CHANNEL}" \ | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh new file mode 100755 index 0000000000..9ed18d7c8f --- /dev/null +++ b/ci/use_conda_packages_from_prs.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) + +CUDF_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) +CUDF_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) + +UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) + +LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) +RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) + +CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) + +conda config --system --add channels "${LIBRMM_CHANNEL}" +conda config --system --add channels "${RMM_CHANNEL}" +conda config --system --add channels "${CUDF_CPP_CHANNEL}" +conda config --system --add channels "${CUDF_PYTHON_CHANNEL}" +conda config --system --add channels "${UCXX_CHANNEL}" +conda config --system --add channels "${LIBRAFT_CHANNEL}" +conda config --system --add channels "${RAFT_CHANNEL}" +conda config --system --add channels "${CUMLPRIMS_CHANNEL}" diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh index 940bdadeca..7a49b492fd 100644 --- a/ci/use_wheels_from_prs.sh +++ b/ci/use_wheels_from_prs.sh @@ -1,5 +1,7 @@ #!/bin/bash +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + LIBRMM_CHANNEL=$( RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp ) @@ -17,7 +19,6 @@ DISTRIBUTED_UCXX_CHANNEL=$( RAPIDS_PY_WHEEL_NAME=distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python ) - CUDF_CHANNEL=$( RAPIDS_PY_WHEEL_NAME=cudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python ) @@ -41,15 +42,17 @@ PYLIBRAFT_CHANNEL=$( ) cat > /tmp/constraints.txt < Date: Thu, 19 Sep 2024 00:02:47 -0500 Subject: [PATCH 06/15] branch --- .github/workflows/build.yaml | 12 ++++++------ .github/workflows/pr.yaml | 30 +++++++++++++++--------------- .github/workflows/test.yaml | 10 +++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bf3fb52c50..fcc0aec68c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: [python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -68,7 +68,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cuml: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -84,7 +84,7 @@ jobs: wheel-publish-cuml: needs: wheel-build-cuml secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c1eb2ec54f..4ec3bb6e39 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,10 +26,10 @@ jobs: # - wheel-tests-cuml - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 # checks: # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 # with: # enable_check_generated_files: false # ignored_pr_jobs: >- @@ -37,7 +37,7 @@ jobs: # clang-tidy: # needs: checks # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 # with: # build_type: pull-request # node_type: "cpu8" @@ -47,19 +47,19 @@ jobs: conda-cpp-build: # needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request # conda-cpp-tests: # needs: conda-cpp-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 # with: # build_type: pull-request # conda-cpp-checks: # needs: conda-cpp-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 # with: # build_type: pull-request # enable_check_symbols: true @@ -67,20 +67,20 @@ jobs: conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request # conda-python-tests-singlegpu: # needs: conda-python-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 # with: # build_type: pull-request # script: "ci/test_python_singlegpu.sh" # optional-job-conda-python-tests-cudf-pandas-integration: # needs: conda-python-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 # with: # matrix_filter: map(select(.ARCH == "amd64")) # build_type: pull-request @@ -88,14 +88,14 @@ jobs: # conda-python-tests-dask: # needs: conda-python-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 # with: # build_type: pull-request # script: "ci/test_python_dask.sh" # conda-notebook-tests: # needs: conda-python-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 # with: # build_type: pull-request # node_type: "gpu-v100-latest-1" @@ -105,7 +105,7 @@ jobs: # docs-build: # needs: conda-python-build # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 # with: # build_type: pull-request # node_type: "gpu-v100-latest-1" @@ -115,7 +115,7 @@ jobs: wheel-build-cuml: # needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel.sh @@ -125,13 +125,13 @@ jobs: # wheel-tests-cuml: # needs: wheel-build-cuml # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 # with: # build_type: pull-request # script: ci/test_wheel.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 with: arch: '["amd64"]' cuda: '["12.5"]' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 85f10e134b..9a2c0086ea 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -26,7 +26,7 @@ jobs: symbol_exclusions: raft_cutlass conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests-singlegpu: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -43,7 +43,7 @@ jobs: script: "ci/test_python_singlegpu.sh" conda-python-tests-dask: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -52,7 +52,7 @@ jobs: script: "ci/test_python_dask.sh" wheel-tests-cuml: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} From 557fca6e6ef077378e840d696b1e8c1adcc95830 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 19 Sep 2024 09:26:01 -0500 Subject: [PATCH 07/15] fix clones --- ci/use_conda_packages_from_prs.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh index 9ed18d7c8f..2001081b6c 100755 --- a/ci/use_conda_packages_from_prs.sh +++ b/ci/use_conda_packages_from_prs.sh @@ -11,7 +11,8 @@ UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) -CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) +# NOTE: cloning private repos with rapids-get-pr-conda-artifact doesn't work +# CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) conda config --system --add channels "${LIBRMM_CHANNEL}" conda config --system --add channels "${RMM_CHANNEL}" @@ -20,4 +21,4 @@ conda config --system --add channels "${CUDF_PYTHON_CHANNEL}" conda config --system --add channels "${UCXX_CHANNEL}" conda config --system --add channels "${LIBRAFT_CHANNEL}" conda config --system --add channels "${RAFT_CHANNEL}" -conda config --system --add channels "${CUMLPRIMS_CHANNEL}" +# conda config --system --add channels "${CUMLPRIMS_CHANNEL}" From 8ca0180e2599c6ca1129bee6e5925572f86f6562 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 20 Sep 2024 13:09:19 -0500 Subject: [PATCH 08/15] run all CI --- .github/workflows/pr.yaml | 182 +++++++++++++++--------------- ci/build_cpp.sh | 3 +- ci/test_cpp.sh | 3 +- ci/test_notebooks.sh | 3 +- ci/test_python_common.sh | 3 +- ci/use_conda_packages_from_prs.sh | 1 + ci/use_wheels_from_prs.sh | 1 + 7 files changed, 97 insertions(+), 99 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4ec3bb6e39..c0c0601afb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,108 +12,108 @@ concurrency: jobs: pr-builder: needs: - # - checks - # - clang-tidy + - checks + - clang-tidy - conda-cpp-build - # - conda-cpp-tests - # - conda-cpp-checks + - conda-cpp-tests + - conda-cpp-checks - conda-python-build - # - conda-python-tests-singlegpu - # - conda-python-tests-dask - # - conda-notebook-tests - # - docs-build + - conda-python-tests-singlegpu + - conda-python-tests-dask + - conda-notebook-tests + - docs-build - wheel-build-cuml - # - wheel-tests-cuml + - wheel-tests-cuml - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 - # checks: - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 - # with: - # enable_check_generated_files: false - # ignored_pr_jobs: >- - # optional-job-conda-python-tests-cudf-pandas-integration - # clang-tidy: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "cpu8" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - # run_script: "ci/run_clang_tidy.sh" + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + with: + enable_check_generated_files: false + ignored_pr_jobs: >- + optional-job-conda-python-tests-cudf-pandas-integration + clang-tidy: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "cpu8" + arch: "amd64" + container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + run_script: "ci/run_clang_tidy.sh" conda-cpp-build: - # needs: checks + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request - # conda-cpp-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - # with: - # build_type: pull-request - # conda-cpp-checks: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - # with: - # build_type: pull-request - # enable_check_symbols: true - # symbol_exclusions: raft_cutlass + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request - # conda-python-tests-singlegpu: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # with: - # build_type: pull-request - # script: "ci/test_python_singlegpu.sh" - # optional-job-conda-python-tests-cudf-pandas-integration: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # with: - # matrix_filter: map(select(.ARCH == "amd64")) - # build_type: pull-request - # script: "ci/test_python_integration.sh" - # conda-python-tests-dask: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # with: - # build_type: pull-request - # script: "ci/test_python_dask.sh" - # conda-notebook-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_notebooks.sh" - # docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/build_docs.sh" + conda-python-tests-singlegpu: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/test_python_singlegpu.sh" + optional-job-conda-python-tests-cudf-pandas-integration: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + with: + matrix_filter: map(select(.ARCH == "amd64")) + build_type: pull-request + script: "ci/test_python_integration.sh" + conda-python-tests-dask: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/test_python_dask.sh" + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-cuml: - # needs: checks + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -122,13 +122,13 @@ jobs: extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-24.10 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - # wheel-tests-cuml: - # needs: wheel-build-cuml - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - # with: - # build_type: pull-request - # script: ci/test_wheel.sh + wheel-tests-cuml: + needs: wheel-build-cuml + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + with: + build_type: pull-request + script: ci/test_wheel.sh devcontainer: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 826f4de391..4fdc896d37 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -18,7 +18,6 @@ rapids-print-env rapids-logger "Begin cpp build" -RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild \ - conda/recipes/libcuml +RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcuml rapids-upload-conda-to-s3 cpp diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 8a7849b4a9..f64ac32b0d 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,8 +18,7 @@ rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ - --prepend-channel "${CPP_CHANNEL}" \ -| tee env.yaml + --prepend-channel "${CPP_CHANNEL}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 05a38da286..b4682a26bd 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -15,8 +15,7 @@ rapids-dependency-file-generator \ --file-key test_notebooks \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" \ -| tee env.yaml + --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 85bef07828..1cc93540cd 100644 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -16,8 +16,7 @@ rapids-dependency-file-generator \ --file-key test_python \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ - --prepend-channel "${PYTHON_CHANNEL}" \ -| tee env.yaml + --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh index 2001081b6c..162421396c 100755 --- a/ci/use_conda_packages_from_prs.sh +++ b/ci/use_conda_packages_from_prs.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh index 7a49b492fd..b74fa3a912 100644 --- a/ci/use_wheels_from_prs.sh +++ b/ci/use_wheels_from_prs.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" From 065491912624a7d9bf8f9e494b6dcf87c2a2abd9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 09:50:05 -0500 Subject: [PATCH 09/15] use cumlprims_mg packages --- ci/use_conda_packages_from_prs.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh index 162421396c..274a8e2b56 100755 --- a/ci/use_conda_packages_from_prs.sh +++ b/ci/use_conda_packages_from_prs.sh @@ -12,8 +12,11 @@ UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) -# NOTE: cloning private repos with rapids-get-pr-conda-artifact doesn't work -# CUMLPRIMS_CHANNEL=$(rapids-get-pr-conda-artifact cumlprims_mg 211 cpp) +# NOTE: cloning private repos with rapids-get-pr-conda-artifact doesn't work, +# so need to explicitly set the SHA to use +CUMLPRIMS_CHANNEL=$( + RAPIDS_SHA=6f9f474 rapids-get-pr-conda-artifact cumlprims_mg 211 cpp +) conda config --system --add channels "${LIBRMM_CHANNEL}" conda config --system --add channels "${RMM_CHANNEL}" @@ -22,4 +25,4 @@ conda config --system --add channels "${CUDF_PYTHON_CHANNEL}" conda config --system --add channels "${UCXX_CHANNEL}" conda config --system --add channels "${LIBRAFT_CHANNEL}" conda config --system --add channels "${RAFT_CHANNEL}" -# conda config --system --add channels "${CUMLPRIMS_CHANNEL}" +conda config --system --add channels "${CUMLPRIMS_CHANNEL}" From a1d76d4da25727473a1fd8cdc1762965e9de67f6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 09:56:41 -0500 Subject: [PATCH 10/15] try to avoid cloning cumlprims_mg --- ci/use_conda_packages_from_prs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh index 274a8e2b56..83f1720ca7 100755 --- a/ci/use_conda_packages_from_prs.sh +++ b/ci/use_conda_packages_from_prs.sh @@ -15,7 +15,7 @@ RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) # NOTE: cloning private repos with rapids-get-pr-conda-artifact doesn't work, # so need to explicitly set the SHA to use CUMLPRIMS_CHANNEL=$( - RAPIDS_SHA=6f9f474 rapids-get-pr-conda-artifact cumlprims_mg 211 cpp + RAPIDS_SHA=6f9f474 rapids-get-pr-conda-artifact cumlprims_mg 211 cpp 6f9f474 ) conda config --system --add channels "${LIBRMM_CHANNEL}" From 5b812f0e966f2d8998ef62437cdcf01139103d8d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 09:31:23 -0700 Subject: [PATCH 11/15] add host dependencies on fmt and spdlog in libcuml --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 ++ conda/environments/all_cuda-125_arch-x86_64.yaml | 2 ++ conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml | 2 ++ conda/environments/cpp_all_cuda-118_arch-x86_64.yaml | 2 ++ conda/environments/cpp_all_cuda-125_arch-x86_64.yaml | 2 ++ conda/recipes/libcuml/conda_build_config.yaml | 6 ++++++ conda/recipes/libcuml/meta.yaml | 2 ++ dependencies.yaml | 2 ++ 8 files changed, 20 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index e7dcb0a323..0540f469d8 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,6 +20,7 @@ dependencies: - dask-cudf==24.10.*,>=0.0.0a0 - dask-ml - doxygen=1.9.1 +- fmt>=11.0.2,<12 - gcc_linux-64=11.* - graphviz - hdbscan>=0.8.38,<0.8.39 @@ -69,6 +70,7 @@ dependencies: - scipy>=1.8.0 - seaborn - setuptools +- spdlog>=1.14.1,<1.15 - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 2340040085..ad8d12f1a3 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -22,6 +22,7 @@ dependencies: - dask-cudf==24.10.*,>=0.0.0a0 - dask-ml - doxygen=1.9.1 +- fmt>=11.0.2,<12 - gcc_linux-64=11.* - graphviz - hdbscan>=0.8.38,<0.8.39 @@ -65,6 +66,7 @@ dependencies: - scipy>=1.8.0 - seaborn - setuptools +- spdlog>=1.14.1,<1.15 - sphinx-copybutton - sphinx-markdown-tables - sphinx<6 diff --git a/conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml b/conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml index 195ddcd8cd..f332c206d9 100644 --- a/conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml +++ b/conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml @@ -14,6 +14,7 @@ dependencies: - cuda-version=11.8 - cudatoolkit - cxx-compiler +- fmt>=11.0.2,<12 - gcc_linux-64=11.* - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 @@ -31,6 +32,7 @@ dependencies: - librmm==24.10.*,>=0.0.0a0 - ninja - nvcc_linux-64=11.8 +- spdlog>=1.14.1,<1.15 - sysroot_linux-64==2.17 - tomli name: clang_tidy_cuda-118_arch-x86_64 diff --git a/conda/environments/cpp_all_cuda-118_arch-x86_64.yaml b/conda/environments/cpp_all_cuda-118_arch-x86_64.yaml index 53535e39c4..66291a21ec 100644 --- a/conda/environments/cpp_all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/cpp_all_cuda-118_arch-x86_64.yaml @@ -12,6 +12,7 @@ dependencies: - cuda-version=11.8 - cudatoolkit - cxx-compiler +- fmt>=11.0.2,<12 - gcc_linux-64=11.* - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 @@ -29,5 +30,6 @@ dependencies: - librmm==24.10.*,>=0.0.0a0 - ninja - nvcc_linux-64=11.8 +- spdlog>=1.14.1,<1.15 - sysroot_linux-64==2.17 name: cpp_all_cuda-118_arch-x86_64 diff --git a/conda/environments/cpp_all_cuda-125_arch-x86_64.yaml b/conda/environments/cpp_all_cuda-125_arch-x86_64.yaml index 2973c81db2..90bdefa75e 100644 --- a/conda/environments/cpp_all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/cpp_all_cuda-125_arch-x86_64.yaml @@ -14,6 +14,7 @@ dependencies: - cuda-profiler-api - cuda-version=12.5 - cxx-compiler +- fmt>=11.0.2,<12 - gcc_linux-64=11.* - libcublas-dev - libcufft-dev @@ -25,5 +26,6 @@ dependencies: - libraft==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - ninja +- spdlog>=1.14.1,<1.15 - sysroot_linux-64==2.17 name: cpp_all_cuda-125_arch-x86_64 diff --git a/conda/recipes/libcuml/conda_build_config.yaml b/conda/recipes/libcuml/conda_build_config.yaml index 42b1cf2e86..f802440354 100644 --- a/conda/recipes/libcuml/conda_build_config.yaml +++ b/conda/recipes/libcuml/conda_build_config.yaml @@ -19,6 +19,12 @@ c_stdlib_version: cmake_version: - ">=3.26.4,!=3.30.0" +fmt_version: + - ">=11.0.2,<12" + +spdlog_version: + - ">=1.14.1,<1.15" + treelite_version: - "=4.3.0" diff --git a/conda/recipes/libcuml/meta.yaml b/conda/recipes/libcuml/meta.yaml index 718375b198..0737da6969 100644 --- a/conda/recipes/libcuml/meta.yaml +++ b/conda/recipes/libcuml/meta.yaml @@ -68,10 +68,12 @@ requirements: - libcusolver-dev - libcusparse-dev {% endif %} + - fmt {{ fmt_version }} - libcumlprims ={{ minor_version }} - libraft ={{ minor_version }} - libraft-headers ={{ minor_version }} - librmm ={{ minor_version }} + - spdlog {{ spdlog_version }} - treelite {{ treelite_version }} outputs: diff --git a/dependencies.yaml b/dependencies.yaml index 23a72d1db8..d176e382ad 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -140,10 +140,12 @@ dependencies: packages: - c-compiler - cxx-compiler + - fmt>=11.0.2,<12 - libcumlprims==24.10.*,>=0.0.0a0 - libraft==24.10.*,>=0.0.0a0 - libraft-headers==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 + - spdlog>=1.14.1,<1.15 specific: - output_types: conda matrices: From 77241136ad60d59ef462a8c506bdfd8bf216e388 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 11:25:16 -0700 Subject: [PATCH 12/15] explicitly pin fmt and spdlog in test environment --- dependencies.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index d176e382ad..d40d90807c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -140,12 +140,12 @@ dependencies: packages: - c-compiler - cxx-compiler - - fmt>=11.0.2,<12 + - &fmt fmt>=11.0.2,<12 - libcumlprims==24.10.*,>=0.0.0a0 - libraft==24.10.*,>=0.0.0a0 - libraft-headers==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - - spdlog>=1.14.1,<1.15 + - &spdlog spdlog>=1.14.1,<1.15 specific: - output_types: conda matrices: @@ -520,6 +520,8 @@ dependencies: - setuptools # Needed on Python 3.12 for dask-glm, which requires pkg_resources but Python 3.12 doesn't have setuptools by default - output_types: conda packages: + - *fmt + - *spdlog - pip - pip: - dask-glm==0.3.0 @@ -538,3 +540,7 @@ dependencies: - *scikit_learn - seaborn - xgboost + - output_types: conda + packages: + - *fmt + - *spdlog From af19c93f76ef2e1f7103be346bff109d87e04ca3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 17:45:49 -0500 Subject: [PATCH 13/15] remove testing-only details --- ci/build_cpp.sh | 3 --- ci/build_python.sh | 3 --- ci/build_wheel.sh | 4 ---- ci/test_cpp.sh | 2 -- ci/test_notebooks.sh | 1 - ci/test_python_common.sh | 1 - ci/test_wheel.sh | 4 ---- rapids_config.cmake | 2 -- 8 files changed, 20 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 4fdc896d37..48f4f444b2 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -11,9 +11,6 @@ source rapids-date-string export CMAKE_GENERATOR=Ninja -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ -source ./ci/use_conda_packages_from_prs.sh - rapids-print-env rapids-logger "Begin cpp build" diff --git a/ci/build_python.sh b/ci/build_python.sh index b4b1b6bd0b..e1397e3373 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,9 +11,6 @@ source rapids-date-string export CMAKE_GENERATOR=Ninja -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ -source ./ci/use_conda_packages_from_prs.sh - rapids-print-env rapids-generate-version > ./VERSION diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index cdb6da13d8..af3a4c124b 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -14,10 +14,6 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # everywhere except in the final wheel name. PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ -source ./ci/use_wheels_from_prs.sh -export PIP_CONSTRAINT=/tmp/constraints.txt - rapids-generate-version > ./VERSION cd ${package_dir} diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index f64ac32b0d..ea6d1cdc11 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -8,8 +8,6 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -source ./ci/use_conda_packages_from_prs.sh - rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index b4682a26bd..d76a754b8b 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -3,7 +3,6 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -source ./ci/use_conda_packages_from_prs.sh rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 1cc93540cd..5f1894356c 100644 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -4,7 +4,6 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -source ./ci/use_conda_packages_from_prs.sh rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 96615dc9a0..86eef035cd 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,10 +3,6 @@ set -euo pipefail -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ -source ./ci/use_wheels_from_prs.sh -export PIP_CONSTRAINT=/tmp/constraints.txt - mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist diff --git a/rapids_config.cmake b/rapids_config.cmake index 4524465a69..f064a5b497 100644 --- a/rapids_config.cmake +++ b/rapids_config.cmake @@ -26,8 +26,6 @@ else() ) endif() -set(rapids-cmake-repo jameslamb/rapids-cmake) -set(rapids-cmake-branch fmt-and-spdlog) if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/CUML_RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") file( DOWNLOAD From 9d3de8db216a7502dba7c90cc4dcfe6d408981cf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 17:48:17 -0500 Subject: [PATCH 14/15] revert testing changes in dependencies.yaml --- dependencies.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index d40d90807c..d176e382ad 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -140,12 +140,12 @@ dependencies: packages: - c-compiler - cxx-compiler - - &fmt fmt>=11.0.2,<12 + - fmt>=11.0.2,<12 - libcumlprims==24.10.*,>=0.0.0a0 - libraft==24.10.*,>=0.0.0a0 - libraft-headers==24.10.*,>=0.0.0a0 - librmm==24.10.*,>=0.0.0a0 - - &spdlog spdlog>=1.14.1,<1.15 + - spdlog>=1.14.1,<1.15 specific: - output_types: conda matrices: @@ -520,8 +520,6 @@ dependencies: - setuptools # Needed on Python 3.12 for dask-glm, which requires pkg_resources but Python 3.12 doesn't have setuptools by default - output_types: conda packages: - - *fmt - - *spdlog - pip - pip: - dask-glm==0.3.0 @@ -540,7 +538,3 @@ dependencies: - *scikit_learn - seaborn - xgboost - - output_types: conda - packages: - - *fmt - - *spdlog From c58738941522b588361cba720905904154b36c11 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Sep 2024 18:02:13 -0500 Subject: [PATCH 15/15] remove scripts --- ci/use_conda_packages_from_prs.sh | 28 --------------- ci/use_wheels_from_prs.sh | 59 ------------------------------- 2 files changed, 87 deletions(-) delete mode 100755 ci/use_conda_packages_from_prs.sh delete mode 100644 ci/use_wheels_from_prs.sh diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh deleted file mode 100755 index 83f1720ca7..0000000000 --- a/ci/use_conda_packages_from_prs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1678 python) - -CUDF_CPP_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 cpp) -CUDF_PYTHON_CHANNEL=$(rapids-get-pr-conda-artifact cudf 16806 python) - -UCXX_CHANNEL=$(rapids-get-pr-conda-artifact ucxx 278 cpp) - -LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 cpp) -RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 2433 python) - -# NOTE: cloning private repos with rapids-get-pr-conda-artifact doesn't work, -# so need to explicitly set the SHA to use -CUMLPRIMS_CHANNEL=$( - RAPIDS_SHA=6f9f474 rapids-get-pr-conda-artifact cumlprims_mg 211 cpp 6f9f474 -) - -conda config --system --add channels "${LIBRMM_CHANNEL}" -conda config --system --add channels "${RMM_CHANNEL}" -conda config --system --add channels "${CUDF_CPP_CHANNEL}" -conda config --system --add channels "${CUDF_PYTHON_CHANNEL}" -conda config --system --add channels "${UCXX_CHANNEL}" -conda config --system --add channels "${LIBRAFT_CHANNEL}" -conda config --system --add channels "${RAFT_CHANNEL}" -conda config --system --add channels "${CUMLPRIMS_CHANNEL}" diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh deleted file mode 100644 index b74fa3a912..0000000000 --- a/ci/use_wheels_from_prs.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -LIBRMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 cpp -) -RMM_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=rmm_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact rmm 1678 python -) - -UCXX_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python -) -LIBUCXX_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=libucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 cpp -) -DISTRIBUTED_UCXX_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact ucxx 278 python -) - -CUDF_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=cudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python -) -LIBCUDF_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=libcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 cpp -) -PYLIBCUDF_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=pylibcudf_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact cudf 16806 python -) -DASK_CUDF_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=dask_cudf_${RAPIDS_PY_CUDA_SUFFIX} \ - RAPIDS_PY_WHEEL_PURE=1 \ - rapids-get-pr-wheel-artifact cudf 16806 python -) - -RAFT_DASK_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=raft_dask_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python -) -PYLIBRAFT_CHANNEL=$( - RAPIDS_PY_WHEEL_NAME=pylibraft_${RAPIDS_PY_CUDA_SUFFIX} rapids-get-pr-wheel-artifact raft 2433 python -) - -cat > /tmp/constraints.txt <