From f8b513b53b44a3f7d7ee7507250965eaffb88d1c Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 30 Aug 2024 13:50:24 +0100 Subject: [PATCH] CI: GHA conda build & upload (#1914) - skip most of conda matrix on PR - update docs (replace CI badges, describe new matrix) - fixes #1870 - part of #1598 - part of #1560 - related to #1875 - related to #1899 <- #1842 <- #1835 - related to #1430 - related to #1416 - related to #1205 - it's my birthday and I'll merge if I wanna --- .github/workflows/README.md | 8 +++-- .github/workflows/build.yml | 66 ++++++++++++++++++++++++++----------- CHANGELOG.md | 1 + README.md | 4 +-- recipe/meta.yaml | 30 ++++------------- 5 files changed, 61 insertions(+), 48 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index cf395ad959..5acd379cbb 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -23,12 +23,16 @@ Details on some of these jobs are given below. ## conda -When opening or modifying a pull request to `master`, a single variant is built and tested. This variant is for linux with `python=3.11` and `numpy=1.25`. +When opening or modifying a pull request to `master`, two variants are built and tested (for linux with minimum & maximum supported `python` & `numpy` versions). > [!NOTE] -> The action does not publish to conda, instead this is done by jenkins. We will eventually move from jenkins to conda-forge instead. > When pushing to `master` or creating an [annotated tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging), *all* variants are built and tested. + + +> [!NOTE] +> The action publishes `ccpi` as well as `https://tomography.stfc.ac.uk/conda/` conda channels. We will eventually move to conda-forge instead. + It looks for conda-build dependencies in the channels listed [here](./build.yml#L118). If you add any new dependencies, the appropriate channels need to be added to this line. > [!TIP] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4915cdb127..575281464c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,16 +1,3 @@ -# Copyright 2021 United Kingdom Research and Innovation -# Copyright 2021 The University of Manchester -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt name: build on: release: @@ -100,13 +87,32 @@ jobs: cmake --build ./build --target install - name: test run: python -m unittest discover -v ./Wrappers/Python/test + conda-matrix: + runs-on: ubuntu-latest + outputs: + python-version: ${{ steps.matrix.outputs.python-version }} + numpy-version: ${{ steps.matrix.outputs.numpy-version }} + steps: + - id: matrix + run: | + if ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') }}; then + echo "python-version=['3.10', 3.11]" >> $GITHUB_OUTPUT + echo "numpy-version=[1.23, 1.24, 1.25, 1.26]" >> $GITHUB_OUTPUT + else + echo "python-version=['3.10']" >> $GITHUB_OUTPUT + echo "numpy-version=[1.23]" >> $GITHUB_OUTPUT + fi conda: defaults: {run: {shell: 'bash -el {0}'}} runs-on: ubuntu-latest + needs: conda-matrix strategy: matrix: - python-version: [3.11] - numpy-version: [1.25] + python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }} + numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }} + include: + - python-version: 3.12 + numpy-version: 1.26 steps: - uses: actions/checkout@v4 with: @@ -122,12 +128,34 @@ jobs: working-directory: recipe run: | conda install boa - conda mambabuild . -c conda-forge -c https://software.repos.intel.com/python/conda -c ccpi --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder . - - name: Upload artifact of the conda package - uses: actions/upload-artifact@v4 + conda mambabuild . -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder . + - uses: actions/upload-artifact@v4 with: - name: cil-package + name: cil-package-py${{ matrix.python-version }}-np${{ matrix.numpy-version }} path: recipe/linux-64/cil* + - name: anaconda upload -c ccpi + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') + run: | + conda install anaconda-client + anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} recipe/linux-64/cil* + - name: conda upload -c tomography.stfc.ac.uk/conda + if: startsWith(github.ref, 'refs/tags') + run: | + echo '${{ secrets.STFC_SSH_KEY }}' > ./key + chmod 600 ./key + rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -P recipe/linux-64/cil* \ + '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}/linux-64/' + conda-reindex: + if: startsWith(github.ref, 'refs/tags') + needs: conda + runs-on: ubuntu-latest + steps: + - name: conda index tomography.stfc.ac.uk/conda + run: | + echo '${{ secrets.STFC_SSH_KEY }}' > ./key + chmod 600 ./key + ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \ + 'conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}' docs: defaults: {run: {shell: 'bash -el {0}', working-directory: docs}} runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index fd0885792e..bb2d98df2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Testing: - New unit tests for operators and functions to check for in place errors and the behaviour of `out` (#1805) - Updates in SPDHG vs PDHG unit test to reduce test time and adjustments to parameters (#1898) + - Drop Jenkins in favour of GHA for conda builds (#1914) - Bug fixes: - `ImageData` removes dimensions of size 1 from the input array. This fixes an issue where single slice reconstructions from 3D data would fail due to shape mismatches (#1885) - Make Binner accept accelerated=False (#1887) diff --git a/README.md b/README.md index 315cdff911..0c4937c318 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # CIL - Core Imaging Library -Master | Development | Conda binaries --|-|- -[![CI-master](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework) | [![CI-dev](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework-dev)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework-dev) | ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil) +[![CI-master](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml/badge.svg)](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml) ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TomographicImaging/CIL-Demos/HEAD?urlpath=lab/tree/binder%2Findex.ipynb) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5e19c411a8..3355fee710 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,21 +1,5 @@ -# Copyright 2018 United Kingdom Research and Innovation -# Copyright 2018 The University of Manchester -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt -# Kyle Pidgeon (UKRI-STFC) - package: name: cil - # use the latest stable release version version: {{ environ.get('GIT_DESCRIBE_TAG','v')[1:].split('-')[0] }} source: @@ -33,20 +17,18 @@ build: test: requires: - python-wget - - cvxpy # [ linux ] + - cvxpy # [linux] - scikit-image - - tomophantom=2.0.0 # [ linux ] + - tomophantom=2.0.0 # [linux] - tigre=2.6 - packaging - - ccpi-regulariser=24.0.1 # [ not osx ] - - astra-toolbox=2.1=cuda* # [ not osx ] + - ccpi-regulariser=24.0.1 # [not osx] + - astra-toolbox=2.1=cuda* # [not osx] source_files: - - ./Wrappers/Python/test # [win] - - ./Wrappers/Python/test # [not win] + - ./Wrappers/Python/test commands: - - python -c "import os; print ('TESTING IN THIS DIRECTORY' , os.getcwd())" - python -m unittest discover -v -s Wrappers/Python/test # [win] - python -m unittest discover -s Wrappers/Python/test -v # [not win] @@ -98,6 +80,6 @@ requirements: - ipywidgets about: - home: https://ccpi.ac.uk/cil + home: https://TomographicImaging.github.io/CIL license: Apache 2.0 License summary: 'CCPi Core Imaging Library'