update CIL and CCPi-Regularisation version and reinstate tests

CHANGES.md

@@ -1,11 +1,15 @@
 # ChangeLog
 
 ## v3.7.0
-
+- Adds CCPi-Regularisation-Toolkit as prerequisite for CIL and reinstate unit tests
+- Docker:
+  - Handle BUILD_CIL correctly and not install CIL from conda if BUILD_CIL=ON
 - CMake:
   - only pass through `OpenMP*LIBRARY` variables if set, and add OpenMP_iomp_LIBRARY to the list (for Intel compiler) https://github.com/SyneRBI/SIRF-SuperBuild/pull/879
-- updated versions:
+- Updated Versions:
   - SIRF: v3.7.0
+  - CIL: 501726d8f09c16faef19ceb69b85c212db2eeca6 # 16 May 2024
+  - CCPi-Regularisation-Toolkit: fix_3D_regularisers branch
   - STIR: v6.1.0
 
 ## v3.6.0

Dockerfile

@@ -21,12 +21,16 @@ RUN bash /opt/scripts/build_system-ubuntu.sh
 
 # SIRF python deps
 ARG BUILD_GPU=0
+ARG BUILD_CIL="OFF"
 COPY docker/requirements.yml /opt/scripts/
 # https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#conda-environments
 # https://github.com/TomographicImaging/CIL/blob/master/Dockerfile
 RUN if test "$BUILD_GPU" != 0; then \
   sed -ri 's/^(\s*)#\s*(- \S+.*#.*GPU.*)$/\1\2/' /opt/scripts/requirements.yml; \
  fi \
+ && if test "$BUILD_CIL" != "OFF"; then \
+  sed -r -i -e '/^\s*- (cil).*/d' /opt/scripts/requirements.yml; \
+ fi \
  && conda config --env --set channel_priority strict \
  && for ch in defaults ccpi intel conda-forge; do conda config --env --add channels $ch; done \
  && mamba env update -n base -f /opt/scripts/requirements.yml \
@@ -66,7 +70,8 @@ ARG USE_NiftyPET="OFF"
 ARG BUILD_siemens_to_ismrmrd="ON"
 ARG BUILD_pet_rd_tools="ON"
 ARG Gadgetron_USE_CUDA="ON"
-ARG BUILD_CIL="OFF"
+# BUILD_CIL is defined in the previous stage
+ARG BUILD_CIL
 ARG EXTRA_BUILD_FLAGS=""
 
 # build, install in /opt/SIRF-SuperBuild/{INSTALL,sources/SIRF}, test (if RUN_CTEST)
@@ -99,7 +104,7 @@ RUN apt update -qq && apt install -yq --no-install-recommends \
   && mkdir -p /usr/share/X11/xkb \
   && test -e /usr/bin/X || ln -s /usr/bin/Xorg /usr/bin/X
 
-RUN echo 'test -z "$OMP_NUM_THREADS" && export OMP_NUM_THREADS=$(python -c "import multiprocessing as mc; print(mc.cpu_count() // 2)")' > /usr/local/bin/before-notebook.d/omp_num_threads.sh
+RUN echo 'test -z "$OMP_NUM_THREADS" && export OMP_NUM_THREADS=$(python -c "import multiprocessing as mc; print(mc.cpu_count() - 2)")' > /usr/local/bin/before-notebook.d/omp_num_threads.sh
 
 COPY --chown=${NB_USER} --chmod=644 --link docker/.bashrc /home/${NB_USER}/
 # RUN sed -i s:PYTHON_INSTALL_DIR:${CONDA_DIR}:g /home/${NB_USER}/.bashrc
@@ -110,8 +115,9 @@ COPY --from=build --link --chown=${NB_USER} /opt/SIRF-SuperBuild/INSTALL/ /opt/S
 #COPY --from=build --link /opt/conda/ /opt/conda/
 
 # install {SIRF-Exercises,CIL-Demos}
+ARG BUILD_CIL
 COPY docker/user_demos.sh /opt/scripts/
-RUN bash /opt/scripts/user_demos.sh \
+RUN BUILD_CIL="${BUILD_CIL}" bash /opt/scripts/user_demos.sh \
  && fix-permissions /opt/SIRF-Exercises /opt/CIL-Demos "${CONDA_DIR}" /home/${NB_USER}
 
 # docker-stacks notebook
@@ -126,4 +132,6 @@ ENV GADGETRON_RELAY_HOST="0.0.0.0"
 
 # run gadgetron in the background before start-notebook.py
 COPY --link --chown=${NB_USER} docker/start-gadgetron-notebook.sh /opt/scripts/
+# COPY --from=build --link --chown=${NB_USER} /opt/SIRF-SuperBuild/INSTALL/lib /opt/conda/lib
+COPY --from=build --link --chown=${NB_USER} /opt/SIRF-SuperBuild/INSTALL/bin/env_sirf.sh /opt/conda/etc/conda/activate.d
 CMD ["/opt/scripts/start-gadgetron-notebook.sh"]

SuperBuild.cmake

@@ -325,7 +325,7 @@ if ("${PYTHON_STRATEGY}" STREQUAL "CONDA")
   set (BUILD_CIL OFF)
 endif()
 if (BUILD_CIL)
-  list(APPEND ${PRIMARY_PROJECT_NAME}_DEPENDENCIES CIL TomoPhantom)
+  list(APPEND ${PRIMARY_PROJECT_NAME}_DEPENDENCIES CIL CCPi-Regularisation-Toolkit TomoPhantom)
 endif()
 if (BUILD_ASTRA)
   list(APPEND ${PRIMARY_PROJECT_NAME}_DEPENDENCIES astra-python-wrapper)

SuperBuild/External_CCPi-Regularisation-Toolkit.cmake

@@ -72,7 +72,8 @@ if(NOT ( DEFINED "USE_SYSTEM_${externalProjName}" AND "${USE_SYSTEM_${externalPr
         -DPYTHON_DEST_DIR:PATH=${PYTHON_DEST}
 
       BUILD_COMMAND ${CMAKE_COMMAND} -E env CIL_VERSION=${${proj}_TAG} ${CMAKE_COMMAND} --build .
-      INSTALL_COMMAND ${CMAKE_COMMAND} -E env CIL_VERSION=${${proj}_TAG} ${CMAKE_COMMAND} --build . --target install
+      INSTALL_COMMAND ${CMAKE_COMMAND} -E env CIL_VERSION=${${proj}_TAG} ${CMAKE_COMMAND} --build . --target install && 
+       ${PYTHON_EXECUTABLE} -m pip install ${${proj}_SOURCE_DIR}/src/Python
       #TEST_COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -s ${${proj}_SOURCE_DIR}/test/ -p test*.py
       DEPENDS
         ${${proj}_DEPENDENCIES}

SuperBuild/External_CIL.cmake

@@ -115,6 +115,9 @@ if(NOT ( DEFINED "USE_SYSTEM_${externalProjName}" AND "${USE_SYSTEM_${externalPr
   add_test(NAME CIL_FRAMEWORK_TESTS_9
            COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_Operator.py
            WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
+  add_test(NAME CIL_FRAMEWORK_TESTS_10
+           COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_PluginsRegularisation.py
+           WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
   add_test(NAME CIL_FRAMEWORK_TESTS_11
            COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_quality_measures.py
            WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
@@ -130,6 +133,7 @@ if(NOT ( DEFINED "USE_SYSTEM_${externalProjName}" AND "${USE_SYSTEM_${externalPr
   add_test(NAME CIL_FRAMEWORK_TESTS_15
            COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_TranslateFunction.py
            WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
+
   # add_test(NAME CIL_FRAMEWORK_TESTS_ALL
   #          COMMAND ${PYTHON_EXECUTABLE} -m unittest discover
   #          WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)

docker/user_demos.sh

@@ -22,7 +22,16 @@ if [ "$PYTHON" = "miniconda" ]; then
       # delete GPU deps
       sed -r -e '/^\s*- (astra-toolbox|tigre).*/d' -e '/^\s*- \S+.*#.*GPU/d' environment.yml > environment-sirf.yml
     fi
-    conda env update --file environment-sirf.yml
+    # do not install CIL from conda if BUILD_CIL is set
+if test "${BUILD_CIL:-OFF}" != "OFF"; then
+      # delete CIL package from the environment file
+      echo "Deleting CIL from the environment file BUILD_CIL is set to >${BUILD_CIL}<"
+      sed -r -i -e '/^\s*- (cil).*/d' environment-sirf.yml
+      cat environment-sirf.yml
+    else
+      echo "Not deleting CIL from the environment file BUILD_CIL is set to >${BUILD_CIL}<"
+    fi
+    conda env update --file environment-sirf.yml -v
   else
     if [ -f requirements.txt ]; then
       cat requirements.txt

env_sirf.sh.in

@@ -2,6 +2,7 @@
 # Use it like this		
 #    source /path/to/whereever/env_ccppetmr.sh		
 # Preferably add this line to your .basrhc, .profile or whatever file is appropriate for your shell		
+test -n "$SIRF_ENV_SOURCED" && return; SIRF_ENV_SOURCED=1 # pragma once
 
 SIRF_PATH=@SIRF_SOURCE_DIR@
 export SIRF_PATH

version_config.cmake

@@ -161,10 +161,10 @@ set(DEFAULT_JSON_TAG v3.11.3)
 # CCPi CIL
 # minimum supported version of CIL supported is > 22.1.0 or from commit a6062410028c9872c5b355be40b96ed1497fed2a
 set(DEFAULT_CIL_URL https://github.com/TomographicImaging/CIL)
-set(DEFAULT_CIL_TAG db5a2a6cd3bddfbbf53e65f0549ac206096e5b44) # 13 Feb 2024
+set(DEFAULT_CIL_TAG 501726d8f09c16faef19ceb69b85c212db2eeca6) # 16 May 2024
 
-set(DEFAULT_CCPi-Regularisation-Toolkit_URL https://github.com/vais-ral/CCPi-Regularisation-Toolkit)
-set(DEFAULT_CCPi-Regularisation-Toolkit_TAG "71f8d304d804b54d378f0ed05539f01aaaf13758")
+set(DEFAULT_CCPi-Regularisation-Toolkit_URL https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit)
+set(DEFAULT_CCPi-Regularisation-Toolkit_TAG "fix_3D_regularisers")
 
 # CERN ROOT
 set(DEFAULT_ROOT_URL https://github.com/root-project/root)