Build wheels alongside conda CI (#12427)

This PR adds pip wheel CI to the Conda CI, instead of having them work separately. Authors: - Sevag H (https://github.com/sevagh) - AJ Schmidt (https://github.com/ajschmidt8) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Vyas Ramasubramani (https://github.com/vyasr) URL: #12427
rapidsai · Jan 10, 2023 · 66b846a · 66b846a
1 parent 4704130
commit 66b846a
Show file tree

Hide file tree

Showing 8 changed files with 128 additions and 82 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -52,3 +52,45 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-build-cudf:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cudf
+      package-dir: python/cudf
+      skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF"
+  wheel-publish-cudf:
+    needs: wheel-build-cudf
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cudf
+  wheel-build-dask-cudf:
+    needs: wheel-publish-cudf
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@main
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: dask_cudf
+      package-dir: python/dask_cudf
+  wheel-publish-dask-cudf:
+    needs: wheel-build-dask-cudf
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@main
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: dask_cudf
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -20,6 +20,10 @@ jobs:
       - conda-python-other-tests
       - conda-java-tests
       - conda-notebook-tests
+      - wheel-build-cudf
+      - wheel-tests-cudf
+      - wheel-build-dask-cudf
+      - wheel-tests-dask-cudf
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-118
   checks:
@@ -78,3 +82,40 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci:latest"
       run_script: "ci/test_notebooks.sh"
+  wheel-build-cudf:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main
+    with:
+      build_type: pull-request
+      package-name: cudf
+      package-dir: python/cudf
+      skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF"
+  wheel-tests-cudf:
+    needs: wheel-build-cudf
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main
+    with:
+      build_type: pull-request
+      package-name: cudf
+      test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64"
+      test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests"
+      test-smoketest: "python ./ci/wheel_smoke_test_cudf.py"
+  wheel-build-dask-cudf:
+    needs: wheel-build-cudf
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@main
+    with:
+      build_type: pull-request
+      package-name: dask_cudf
+      package-dir: python/dask_cudf
+      before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf && pip install --no-deps ./local-cudf/cudf*.whl"
+  wheel-tests-dask-cudf:
+    needs: wheel-build-dask-cudf
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@main
+    with:
+      build_type: pull-request
+      package-name: dask_cudf
+      test-before: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf-dep && pip install --no-deps ./local-cudf-dep/cudf*.whl"
+      test-unittest: "pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests"
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -65,3 +65,24 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci:latest"
       run_script: "ci/test_notebooks.sh"
+  wheel-tests-cudf:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      package-name: cudf
+      test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64"
+      test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests"
+  wheel-tests-dask-cudf:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@main
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      package-name: dask_cudf
+      test-unittest: "pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests"
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
diff --git a/ci/wheel_smoke_test_cudf.py b/ci/wheel_smoke_test_cudf.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+
+import cudf
+import pyarrow as pa
+
+if __name__ == '__main__':
+    n_legs = pa.array([2, 4, 5, 100])
+    animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"])
+    names = ["n_legs", "animals"]
+    foo = pa.table([n_legs, animals], names=names)
+    df = cudf.DataFrame.from_arrow(foo)
+    assert df.loc[df["animals"] == "Centipede"]["n_legs"].iloc[0] == 100
+    assert df.loc[df["animals"] == "Flamingo"]["n_legs"].iloc[0] == 2
diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake
@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -25,8 +25,14 @@ include_guard(GLOBAL)
 # Generate a FindArrow module for the case where we need to search for arrow within a pip install
 # pyarrow.
 function(find_libarrow_in_python_wheel PYARROW_VERSION)
-  string(REPLACE "." "" PYARROW_SO_VER "${PYARROW_VERSION}")
-  set(PYARROW_LIB libarrow.so.${PYARROW_SO_VER})
+  string(REPLACE "." ";" PYARROW_VER_COMPONENTS "${PYARROW_VERSION}")
+  list(GET PYARROW_VER_COMPONENTS 0 PYARROW_SO_VER)
+  # The soname for Arrow libraries is constructed using the major version plus "00". Note that,
+  # although it may seem like it due to Arrow almost exclusively releasing new major versions (i.e.
+  # `${MINOR_VERSION}${PATCH_VERSION}` is almost always equivalent to "00"),
+  # the soname is not generated by concatenating the major, minor, and patch versions into a single
+  # version number soname, just `${MAJOR_VERSION}00`
+  set(PYARROW_LIB "libarrow.so.${PYARROW_SO_VER}00")
 
   find_package(Python REQUIRED)
   execute_process(

diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml
@@ -10,7 +10,7 @@ requires = [
     "cmake>=3.23.1,!=3.25.0",
     "ninja",
     "numpy",
-    "pyarrow==10",
+    "pyarrow==10.0.1",
     "protoc-wheel",
     "versioneer",
 ]

diff --git a/python/cudf/setup.py b/python/cudf/setup.py
@@ -19,7 +19,7 @@
     "pandas>=1.0,<1.6.0dev0",
     "protobuf==4.21",
     "typing_extensions",
-    "pyarrow==10",
+    "pyarrow==10.0.1",
     f"rmm{cuda_suffix}",
     f"ptxcompiler{cuda_suffix}",
     f"cubinlinker{cuda_suffix}",