CI: GHA conda build & upload (#1914)

- skip most of conda matrix on PR
- update docs (replace CI badges, describe new matrix)
- fixes #1870
- part of #1598
- part of #1560
- related to #1875
- related to #1899 <- #1842 <- #1835
- related to #1430
- related to #1416
- related to #1205
- it's my birthday and I'll merge if I wanna

.github/workflows/README.md

@@ -23,12 +23,16 @@ Details on some of these jobs are given below.
 
 ## conda
 
-When opening or modifying a pull request to `master`, a single variant is built and tested. This variant is for linux with `python=3.11` and `numpy=1.25`.
+When opening or modifying a pull request to `master`, two variants are built and tested (for linux with minimum & maximum supported `python` & `numpy` versions).
 
 > [!NOTE]
-> The action does not publish to conda, instead this is done by jenkins. We will eventually move from jenkins to conda-forge instead.
 > When pushing to `master` or creating an [annotated tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging), *all* variants are built and tested.
 
+<!-- <br/> -->
+
+> [!NOTE]
+> The action publishes `ccpi` as well as `https://tomography.stfc.ac.uk/conda/` conda channels. We will eventually move to conda-forge instead.
+
 It looks for conda-build dependencies in the channels listed [here](./build.yml#L118). If you add any new dependencies, the appropriate channels need to be added to this line.
 
 > [!TIP]

.github/workflows/build.yml

@@ -1,16 +1,3 @@
-# Copyright 2021 United Kingdom Research and Innovation
-# Copyright 2021 The University of Manchester
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# Authors:
-# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt
 name: build
 on:
   release:
@@ -100,13 +87,32 @@ jobs:
         cmake --build ./build --target install
     - name: test
       run: python -m unittest discover -v ./Wrappers/Python/test
+  conda-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      python-version: ${{ steps.matrix.outputs.python-version }}
+      numpy-version: ${{ steps.matrix.outputs.numpy-version }}
+    steps:
+    - id: matrix
+      run: |
+        if ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') }}; then
+          echo "python-version=['3.10', 3.11]" >> $GITHUB_OUTPUT
+          echo "numpy-version=[1.23, 1.24, 1.25, 1.26]" >> $GITHUB_OUTPUT
+        else
+          echo "python-version=['3.10']" >> $GITHUB_OUTPUT
+          echo "numpy-version=[1.23]" >> $GITHUB_OUTPUT
+        fi
   conda:
     defaults: {run: {shell: 'bash -el {0}'}}
     runs-on: ubuntu-latest
+    needs: conda-matrix
     strategy:
       matrix:
-        python-version: [3.11]
-        numpy-version: [1.25]
+        python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }}
+        numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }}
+        include:
+        - python-version: 3.12
+          numpy-version: 1.26
     steps:
     - uses: actions/checkout@v4
       with:
@@ -122,12 +128,34 @@ jobs:
       working-directory: recipe
       run: |
         conda install boa
-        conda mambabuild . -c conda-forge -c https://software.repos.intel.com/python/conda -c ccpi --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder .
-    - name: Upload artifact of the conda package
-      uses: actions/upload-artifact@v4
+        conda mambabuild . -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder .
+    - uses: actions/upload-artifact@v4
       with:
-        name: cil-package
+        name: cil-package-py${{ matrix.python-version }}-np${{ matrix.numpy-version }}
         path: recipe/linux-64/cil*
+    - name: anaconda upload -c ccpi
+      if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')
+      run: |
+        conda install anaconda-client
+        anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} recipe/linux-64/cil*
+    - name: conda upload -c tomography.stfc.ac.uk/conda
+      if: startsWith(github.ref, 'refs/tags')
+      run: |
+        echo '${{ secrets.STFC_SSH_KEY }}' > ./key
+        chmod 600 ./key
+        rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -P recipe/linux-64/cil* \
+          '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}/linux-64/'
+  conda-reindex:
+    if: startsWith(github.ref, 'refs/tags')
+    needs: conda
+    runs-on: ubuntu-latest
+    steps:
+    - name: conda index tomography.stfc.ac.uk/conda
+      run: |
+        echo '${{ secrets.STFC_SSH_KEY }}' > ./key
+        chmod 600 ./key
+        ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \
+          'conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}'
   docs:
     defaults: {run: {shell: 'bash -el {0}', working-directory: docs}}
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -19,6 +19,7 @@
   - Testing:
     - New unit tests for operators and functions to check for in place errors and the behaviour of `out` (#1805)
     - Updates in SPDHG vs PDHG unit test to reduce test time and adjustments to parameters (#1898)
+    - Drop Jenkins in favour of GHA for conda builds (#1914)
   - Bug fixes:
     - `ImageData` removes dimensions of size 1 from the input array. This fixes an issue where single slice reconstructions from 3D data would fail due to shape mismatches (#1885)
     - Make Binner accept accelerated=False (#1887)

README.md

@@ -1,8 +1,6 @@
 # CIL - Core Imaging Library
 
-Master | Development | Conda binaries
--|-|-
-[![CI-master](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework) | [![CI-dev](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework-dev)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework-dev) | ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil)
+[![CI-master](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml/badge.svg)](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml) ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil)
 
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TomographicImaging/CIL-Demos/HEAD?urlpath=lab/tree/binder%2Findex.ipynb)
 

recipe/meta.yaml

@@ -1,21 +1,5 @@
-# Copyright 2018 United Kingdom Research and Innovation
-# Copyright 2018 The University of Manchester
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# Authors:
-# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt
-# Kyle Pidgeon (UKRI-STFC)
-
 package:
   name: cil
-  # use the latest stable release version
   version: {{ environ.get('GIT_DESCRIBE_TAG','v')[1:].split('-')[0] }}
 
 source:
@@ -33,20 +17,18 @@ build:
 test:
   requires:
     - python-wget
-    - cvxpy                   # [ linux ]
+    - cvxpy                   # [linux]
     - scikit-image
-    - tomophantom=2.0.0       # [ linux ]
+    - tomophantom=2.0.0       # [linux]
     - tigre=2.6
     - packaging
-    - ccpi-regulariser=24.0.1 # [ not osx ]
-    - astra-toolbox=2.1=cuda* # [ not osx ]
+    - ccpi-regulariser=24.0.1 # [not osx]
+    - astra-toolbox=2.1=cuda* # [not osx]
 
   source_files:
-    - ./Wrappers/Python/test # [win]
-    - ./Wrappers/Python/test # [not win]
+    - ./Wrappers/Python/test
 
   commands:
-    - python -c "import os; print ('TESTING IN THIS DIRECTORY' , os.getcwd())"
     - python -m unittest discover -v -s Wrappers/Python/test  # [win]
     - python -m unittest discover -s Wrappers/Python/test -v  # [not win]
 
@@ -98,6 +80,6 @@ requirements:
     - ipywidgets
 
 about:
-  home: https://ccpi.ac.uk/cil
+  home: https://TomographicImaging.github.io/CIL
   license:  Apache 2.0 License
   summary: 'CCPi Core Imaging Library'