Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Build wheels alongside conda CI #12427

Merged
merged 30 commits into from
Jan 10, 2023
Merged

Build wheels alongside conda CI #12427

Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
628de05
Build wheels alongside conda CI
sevagh Dec 20, 2022
c51c692
Retrigger build
sevagh Dec 20, 2022
72ea6a4
Retrigger build
sevagh Dec 20, 2022
5cbaac4
Put wheel name at end of job name
sevagh Dec 20, 2022
f83d795
Use unified publish workflow
sevagh Dec 21, 2022
5eec7dd
Add wheel parameters to build.yaml inputs
sevagh Dec 21, 2022
fba1239
Separate publish workflows
sevagh Dec 21, 2022
a993a3f
Use bigger Pyarrow interop smoke test
sevagh Dec 21, 2022
0124cd1
Try escaped double-quotes instead
sevagh Dec 21, 2022
782b2af
Move smoke test to its own file
sevagh Dec 21, 2022
bb610d1
Add copyright to smoke test file
sevagh Dec 21, 2022
77d3694
Merge branch 'branch-23.02' into feat/wheels/py310-ctk12-wheels
ajschmidt8 Jan 5, 2023
bc578d8
fix copyright headers
ajschmidt8 Jan 5, 2023
1794af8
make wheel builds depend on `checks`
ajschmidt8 Jan 5, 2023
04e5add
Install pyarrow 10.0.1
sevagh Jan 6, 2023
68f3ee4
Add one more assert to smoke test; drop print statement
sevagh Jan 6, 2023
0aeabe2
Copyright fix in cudf/setup.py
sevagh Jan 6, 2023
4608733
Merge branch 'branch-23.02' into feat/wheels/py310-ctk12-wheels
sevagh Jan 6, 2023
5ea3a5f
Removed some wheel parameters
sevagh Jan 6, 2023
037ef3d
Update arrow versioning logic.
vyasr Jan 6, 2023
35a6c30
More accurate comment.
vyasr Jan 6, 2023
c070aa5
Use main version of improved wheel workflows
sevagh Jan 6, 2023
f81bc92
Use current-workflow cudf wheel for dask-cudf
sevagh Jan 9, 2023
5157326
Use branch of shared-action-workflows
sevagh Jan 9, 2023
22e3951
Pure wheels use test-before no arch
sevagh Jan 9, 2023
c364ec8
Use correct RAPIDS_PY_WHEEL_NAME
sevagh Jan 9, 2023
caa95a7
Trigger build
sevagh Jan 9, 2023
acf191e
Remove unneeded repo: keys
sevagh Jan 9, 2023
e214f83
Merge branch 'branch-23.02' into feat/wheels/py310-ctk12-wheels
ajschmidt8 Jan 9, 2023
6492d1a
before-wheel param is now supported by main workflow
sevagh Jan 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,49 @@ jobs:
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
wheel-build-cudf:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main
with:
build_type: ${{ inputs.build_type || 'branch' }}
repo: rapidsai/cudf
sevagh marked this conversation as resolved.
Show resolved Hide resolved
branch: ${{ inputs.branch }}
sha: ${{ inputs.sha }}
date: ${{ inputs.date }}
package-name: cudf
package-dir: python/cudf
skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can leave this for now (something I will raise more broadly in the new year) but AFAIK this variable doesn't do anything right now. The question would be whether it should be set to do anything. That's a rapids-wide discussion though; I think only cuML is actively making use of this now and that's partly because it needs to forward along to deps like treelite.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rapidsai/rapids-cmake#334

wheel-publish-cudf:
needs: wheel-build-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main
with:
build_type: ${{ inputs.build_type || 'branch' }}
repo: rapidsai/cudf
branch: ${{ inputs.branch }}
sha: ${{ inputs.sha }}
date: ${{ inputs.date }}
package-name: cudf
wheel-build-dask-cudf:
needs: wheel-publish-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@main
with:
build_type: ${{ inputs.build_type || 'branch' }}
repo: rapidsai/cudf
branch: ${{ inputs.branch }}
sha: ${{ inputs.sha }}
date: ${{ inputs.date }}
package-name: dask_cudf
package-dir: python/dask_cudf
wheel-publish-dask-cudf:
needs: wheel-build-dask-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@main
with:
build_type: ${{ inputs.build_type || 'branch' }}
repo: rapidsai/cudf
branch: ${{ inputs.branch }}
sha: ${{ inputs.sha }}
date: ${{ inputs.date }}
package-name: dask_cudf
39 changes: 39 additions & 0 deletions .github/workflows/pr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ jobs:
- conda-python-other-tests
- conda-java-tests
- conda-notebook-tests
- wheel-build-cudf
- wheel-tests-cudf
- wheel-build-dask-cudf
- wheel-tests-dask-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main
checks:
Expand Down Expand Up @@ -78,3 +82,38 @@ jobs:
arch: "amd64"
container_image: "rapidsai/ci:latest"
run_script: "ci/test_notebooks.sh"
wheel-build-cudf:
needs: checks
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main
with:
build_type: pull-request
package-name: cudf
package-dir: python/cudf
skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF"
wheel-tests-cudf:
needs: wheel-build-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main
with:
build_type: pull-request
package-name: cudf
test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since it's already working we can leave it for now, but I'll note that in #12377 we actually removed tokenizers from arm tests and only test on x86 for conda packages.

test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests"
test-smoketest: "python ./ci/wheel_smoke_test_cudf.py"
wheel-build-dask-cudf:
needs: wheel-build-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@main
with:
build_type: pull-request
package-name: dask_cudf
package-dir: python/dask_cudf
wheel-tests-dask-cudf:
needs: wheel-build-dask-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@main
with:
build_type: pull-request
package-name: dask_cudf
test-unittest: "pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests"
23 changes: 23 additions & 0 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,26 @@ jobs:
arch: "amd64"
container_image: "rapidsai/ci:latest"
run_script: "ci/test_notebooks.sh"
wheel-tests-cudf:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main
with:
build_type: nightly
repo: rapidsai/cudf
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
package-name: cudf
test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64"
test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests"
wheel-tests-dask-cudf:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@main
with:
build_type: nightly
repo: rapidsai/cudf
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
package-name: dask_cudf
test-unittest: "pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests"
77 changes: 0 additions & 77 deletions .github/workflows/wheels.yml
View file
Open in desktop

This file was deleted.

13 changes: 13 additions & 0 deletions ci/wheel_smoke_test_cudf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (c) 2022-2023, NVIDIA CORPORATION.

import cudf
import pyarrow as pa

if __name__ == '__main__':
n_legs = pa.array([2, 4, 5, 100])
animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"])
names = ["n_legs", "animals"]
foo = pa.table([n_legs, animals], names=names)
df = cudf.DataFrame.from_arrow(foo)
assert df.loc[df["animals"] == "Centipede"]["n_legs"].iloc[0] == 100
assert df.loc[df["animals"] == "Flamingo"]["n_legs"].iloc[0] == 2
12 changes: 9 additions & 3 deletions cpp/cmake/thirdparty/get_arrow.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# =============================================================================
# Copyright (c) 2020-2022, NVIDIA CORPORATION.
# Copyright (c) 2020-2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
Expand All @@ -25,8 +25,14 @@ include_guard(GLOBAL)
# Generate a FindArrow module for the case where we need to search for arrow within a pip install
# pyarrow.
function(find_libarrow_in_python_wheel PYARROW_VERSION)
string(REPLACE "." "" PYARROW_SO_VER "${PYARROW_VERSION}")
set(PYARROW_LIB libarrow.so.${PYARROW_SO_VER})
string(REPLACE "." ";" PYARROW_VER_COMPONENTS "${PYARROW_VERSION}")
list(GET PYARROW_VER_COMPONENTS 0 PYARROW_SO_VER)
# The soname for Arrow libraries is constructed using the major version plus "00". Note that,
# although it may seem like it due to Arrow almost exclusively releasing new major versions (i.e.
# `${MINOR_VERSION}${PATCH_VERSION}` is almost always equivalent to "00"),
# the soname is not generated by concatenating the major, minor, and patch versions into a single
# version number soname, just `${MAJOR_VERSION}00`
set(PYARROW_LIB "libarrow.so.${PYARROW_SO_VER}00")

find_package(Python REQUIRED)
execute_process(
Expand Down
2 changes: 1 addition & 1 deletion python/cudf/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ requires = [
"cmake>=3.23.1,!=3.25.0",
"ninja",
"numpy",
"pyarrow==10",
"pyarrow==10.0.1",
"protoc-wheel",
"versioneer",
]
Expand Down
4 changes: 2 additions & 2 deletions python/cudf/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2018-2022, NVIDIA CORPORATION.
# Copyright (c) 2018-2023, NVIDIA CORPORATION.

import os

Expand All @@ -19,7 +19,7 @@
"pandas>=1.0,<1.6.0dev0",
"protobuf==4.21",
"typing_extensions",
"pyarrow==10",
"pyarrow==10.0.1",
f"rmm{cuda_suffix}",
f"ptxcompiler{cuda_suffix}",
f"cubinlinker{cuda_suffix}",
Expand Down