Merge branch 'branch-23.08' into mfg_capi

rapidsai · Jul 21, 2023 · 37c08e8 · 37c08e8
2 parents bc5d3e1 + cab8368
commit 37c08e8
Show file tree

Hide file tree

Showing 18 changed files with 241 additions and 213 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -30,7 +30,6 @@ jobs:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
@@ -40,7 +39,6 @@ jobs:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
@@ -63,7 +61,7 @@ jobs:
       arch: "amd64"
       branch: ${{ inputs.branch }}
       build_type: ${{ inputs.build_type || 'branch' }}
-      container_image: "rapidsai/ci:cuda11.8.0-ubuntu22.04-py3.10"
+      container_image: "rapidsai/ci:latest"
       date: ${{ inputs.date }}
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -35,29 +35,25 @@ jobs:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: pull-request
       node_type: cpu16
   conda-cpp-tests:
     needs: conda-cpp-build
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: pull-request
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: pull-request
   conda-notebook-tests:
     needs: conda-python-build
@@ -67,7 +63,7 @@ jobs:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
       arch: "amd64"
-      container_image: "rapidsai/ci:cuda11.8.0-ubuntu22.04-py3.10"
+      container_image: "rapidsai/ci:latest"
       run_script: "ci/test_notebooks.sh"
   docs-build:
     needs: conda-python-build
@@ -77,7 +73,7 @@ jobs:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
       arch: "amd64"
-      container_image: "rapidsai/ci:cuda11.8.0-ubuntu22.04-py3.10"
+      container_image: "rapidsai/ci:latest"
       run_script: "ci/build_docs.sh"
   wheel-build-pylibcugraph:
     needs: checks

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -18,7 +18,6 @@ jobs:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
@@ -27,7 +26,6 @@ jobs:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
     with:
-      matrix_filter: map(select(.CUDA_VER | startswith("11")))
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}

diff --git a/build.sh b/build.sh
@@ -60,7 +60,7 @@ HELP="$0 [<target> ...] [<flag> ...]
  and <flag> is:
    -v                         - verbose build mode
    -g                         - build for debug
-   -n                         - do not install after a successful build
+   -n                         - do not install after a successful build (does not affect Python packages)
    --pydevelop                - use setup.py develop instead of install
    --allgpuarch               - build for all supported GPU architectures
    --skip_cpp_tests           - do not build the SG test binaries as part of the libcugraph and libcugraph_etl targets
@@ -104,7 +104,7 @@ BUILD_CPP_MG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
 BUILD_WITH_CUGRAPHOPS=ON
 CMAKE_GENERATOR_OPTION="-G Ninja"
-PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps ."
+PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback
@@ -178,6 +178,12 @@ if hasArg --pydevelop; then
     PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps -e ."
 fi
 
+# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
+SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}"
+  if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUGRAPH_CPP"* ]]; then
+      SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_CUGRAPH_CPP=ON"
+  fi
+
 # If clean or uninstall targets given, run them prior to any other steps
 if hasArg uninstall; then
     if [[ "$INSTALL_PREFIX" != "" ]]; then
@@ -296,21 +302,9 @@ if buildAll || hasArg pylibcugraph; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/pylibcugraph
     else
-        cd ${REPODIR}/python/pylibcugraph
-        # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
-        # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
-        CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
-        python setup.py build_ext \
-	       --inplace \
-	       -- \
-	       -DFIND_CUGRAPH_CPP=ON \
-	       -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \
-               -Dcugraph_ROOT=${LIBCUGRAPH_BUILD_DIR} \
-	       -- \
-	       -j${PARALLEL_LEVEL:-1}
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        SKBUILD_CONFIGURE_OPTIONS="${SKBUILD_EXTRA_CMAKE_ARGS} -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS}" \
+            SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \
+            python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/pylibcugraph
     fi
 fi
 
@@ -319,22 +313,9 @@ if buildAll || hasArg cugraph; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph
     else
-        cd ${REPODIR}/python/cugraph
-        # FIXME: this needs to eventually reference the pylibcugraph build
-        # setup.py references an env var CUGRAPH_BUILD_PATH to find the libcugraph
-        # build. If not set by the user, set it to LIBCUGRAPH_BUILD_DIR
-        CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH:=${LIBCUGRAPH_BUILD_DIR}}
-        python setup.py build_ext \
-	       --inplace \
-	       -- \
-	       -DFIND_CUGRAPH_CPP=ON \
-	       -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \
-               -Dcugraph_ROOT=${LIBCUGRAPH_BUILD_DIR} \
-	       -- \
-	       -j${PARALLEL_LEVEL:-1}
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            env CUGRAPH_BUILD_PATH=${CUGRAPH_BUILD_PATH} python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        SKBUILD_CONFIGURE_OPTIONS="${SKBUILD_EXTRA_CMAKE_ARGS} -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS}" \
+            SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \
+            python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph
     fi
 fi
 
@@ -343,12 +324,8 @@ if hasArg cugraph-service; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-service
     else
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            cd ${REPODIR}/python/cugraph-service/client
-            python ${PYTHON_ARGS_FOR_INSTALL}
-            cd ${REPODIR}/python/cugraph-service/server
-            python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-service/client
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-service/server
     fi
 fi
 
@@ -357,10 +334,7 @@ if hasArg cugraph-pyg; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-pyg
     else
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            cd ${REPODIR}/python/cugraph-pyg
-            python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-pyg
     fi
 fi
 
@@ -369,10 +343,7 @@ if hasArg cugraph-dgl; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph-dgl
     else
-        if [[ ${INSTALL_TARGET} != "" ]]; then
-            cd ${REPODIR}/python/cugraph-dgl
-            python ${PYTHON_ARGS_FOR_INSTALL}
-        fi
+        python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph-dgl
     fi
 fi
 

diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -26,28 +26,36 @@ rapids-mamba-retry mambabuild \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/cugraph
 
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  conda/recipes/cugraph-service
-
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel pyg \
-  --channel pytorch \
-  --channel pytorch-nightly \
-  conda/recipes/cugraph-pyg
-
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  --channel dglteam \
-  --channel pytorch \
-  --channel pytorch-nightly \
-  conda/recipes/cugraph-dgl
+RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
+
+if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
+  # Only one CUDA configuration is needed, so we choose CUDA 11 arbitrarily.
+  # Nothing in the cugraph-service packages is CUDA-specific.
+  rapids-mamba-retry mambabuild \
+    --no-test \
+    --channel "${CPP_CHANNEL}" \
+    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+    conda/recipes/cugraph-service
+
+  # Only CUDA 11 is supported right now due to PyTorch requirement.
+  rapids-mamba-retry mambabuild \
+    --no-test \
+    --channel "${CPP_CHANNEL}" \
+    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+    --channel pyg \
+    --channel pytorch \
+    --channel pytorch-nightly \
+    conda/recipes/cugraph-pyg
+
+  # Only CUDA 11 is supported right now due to PyTorch requirement.
+  rapids-mamba-retry mambabuild \
+    --no-test \
+    --channel "${CPP_CHANNEL}" \
+    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+    --channel dglteam \
+    --channel pytorch \
+    --channel pytorch-nightly \
+    conda/recipes/cugraph-dgl
+fi
 
 rapids-upload-conda-to-s3 python
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -4,15 +4,16 @@ channels:
 - rapidsai
 - rapidsai-nightly
 - dask/label/dev
-- conda-forge
-- nvidia
 - pytorch
 - dglteam/label/cu118
+- conda-forge
+- nvidia
 dependencies:
 - aiohttp
 - c-compiler
 - cmake>=3.26.4
-- cudatoolkit=11.8
+- cuda-version=11.8
+- cudatoolkit
 - cudf==23.8.*
 - cupy>=12.0.0
 - cxx-compiler

diff --git a/conda/recipes/cugraph/conda_build_config.yaml b/conda/recipes/cugraph/conda_build_config.yaml
@@ -5,6 +5,9 @@ cxx_compiler_version:
   - 11
 
 cuda_compiler:
+  - cuda-nvcc
+
+cuda11_compiler:
   - nvcc
 
 cmake_version:
@@ -15,18 +18,3 @@ sysroot_version:
 
 ucx_py_version:
   - "0.33.*"
-
-# The CTK libraries below are missing from the conda-forge::cudatoolkit
-# package. The "*_host_*" version specifiers correspond to `11.8` packages.
-
-libcublas_host_version:
-  - "=11.11.3.6"
-
-libcurand_host_version:
-  - "=10.3.0.86"
-
-libcusolver_host_version:
-  - "=11.4.1.48"
-
-libcusparse_host_version:
-  - "=11.7.5.86"
diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml
@@ -39,36 +39,38 @@ build:
 requirements:
   build:
     - {{ compiler('c') }}
-    - {{ compiler('cuda') }} {{ cuda_version }}
     - {{ compiler('cxx') }}
+    {% if cuda_major == "11" %}
+    - {{ compiler('cuda11') }} {{ cuda_version }}
+    {% else %}
+    - {{ compiler('cuda') }}
+    {% endif %}
+    - cuda-version ={{ cuda_version }}
     - cmake {{ cmake_version }}
     - ninja
     - sysroot_{{ target_platform }} {{ sysroot_version }}
   host:
-    - cudatoolkit ={{ cuda_version }}
+    - cuda-version ={{ cuda_version }}
+    {% if cuda_major == "11" %}
+    - cudatoolkit
+    {% endif %}
     - cudf ={{ minor_version }}
     - cython >=0.29,<0.30
-    - libcublas {{ libcublas_host_version }}
-    - libcublas-dev {{ libcublas_host_version }}
     - libcugraph ={{ version }}
-    - libcurand {{ libcurand_host_version }}
-    - libcurand-dev {{ libcurand_host_version }}
-    - libcusolver {{ libcusolver_host_version }}
-    - libcusolver-dev {{ libcusolver_host_version }}
-    - libcusparse {{ libcusparse_host_version }}
-    - libcusparse-dev {{ libcusparse_host_version }}
-    - libraft ={{ minor_version }}
-    - libraft-headers ={{ minor_version }}
-    - pylibraft ={{ minor_version}}
+    - pylibraft ={{ minor_version }}
     - python
     - raft-dask ={{ minor_version }}
+    - rmm ={{ minor_version }}
     - scikit-build >=0.13.1
     - setuptools
-    - ucx-proc=*=gpu
-    - ucx-py {{ ucx_py_version }}
   run:
-    - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
-    - cuda-python >=11.7.1,<12.0
+    - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
+    {% if cuda_major == "11" %}
+    - cudatoolkit
+    - cuda-python >=11.7.1,<12.0a0
+    {% else %}
+    - cuda-python >=12.0,<13.0a0
+    {% endif %}
     - cudf ={{ minor_version }}
     - cupy >=12.0.0
     - dask-cuda ={{ minor_version }}
@@ -77,8 +79,6 @@ requirements:
     - dask-core >=2023.5.1
     - distributed >=2023.5.1
     - libcugraph ={{ version }}
-    - libraft ={{ minor_version }}
-    - libraft-headers ={{ minor_version }}
     - pylibcugraph ={{ version }}
     - pylibraft ={{ minor_version }}
     - python
@@ -88,7 +88,7 @@ requirements:
 
 tests:
   requirements:
-    - cudatoolkit ={{ cuda_version }}
+    - cuda-version ={{ cuda_version }}
   imports:
     - cugraph