(closes #2629) Fix NEMOv5 issues

.github/workflows/lfric_test.yml

@@ -46,7 +46,7 @@ jobs:
     if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
     runs-on: self-hosted
     env:
-      LFRIC_APPS_REV: 1192
+      LFRIC_APPS_REV: 3269
       PYTHON_VERSION: 3.12.5
 
     steps:
@@ -79,21 +79,21 @@ jobs:
     - name: LFRic passthrough (with DistributedMemory)
       run: |
         # Set up environment
-        source /apps/spack/spack-upstream/share/spack/setup-env.sh
-        spack load lfric-buildenv%gcc
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack load lfric-build-environment%gcc@14
         source .runner_venv/bin/activate
         export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
         export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
-        export LFRIC_DIR=${HOME}/LFRic/lfric_apps
+        export LFRIC_DIR=${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV}
         cd ${LFRIC_DIR}
         # Clean previous version and compile again
         rm -rf applications/gungho_model/working
         ./build/local_build.py -a gungho_model -v
         # Run
         cd applications/gungho_model/example
         cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml
-        mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
-        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_output.txt output.txt
+        mpirun -n 6 ../bin/gungho_model configuration.nml |& tee output.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_checksums.txt gungho_model-checksums.txt
         cat timer.txt
         export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
         export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
@@ -105,7 +105,7 @@ jobs:
           --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
           github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
           ci_test: "LFRic Passthrough with DM", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos",
-          compiler:"spack-gfortran-11", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          compiler:"spack-gfortran-14", date: new Date(), elapsed_time: '"$VAR_TIME"',
           num_of_halo_exchanges: '"$VAR_HALOS"'})'
 
     - name: Upload LFRic passthrough results
@@ -119,12 +119,12 @@ jobs:
     - name: LFRic with all transformations
       run: |
         # Set up environment
-        source /apps/spack/spack-upstream/share/spack/setup-env.sh
-        spack load lfric-buildenv%gcc
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack load lfric-build-environment%gcc@14
         source .runner_venv/bin/activate
         export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
         export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
-        export LFRIC_DIR=${HOME}/LFRic/lfric_apps
+        export LFRIC_DIR=${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV}
         export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
         cd ${LFRIC_DIR}
         # Psyclone scripts must now be under 'optimisation' and be called 'global.py'
@@ -138,7 +138,7 @@ jobs:
         cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml
         export OMP_NUM_THREADS=6
         mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
-        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_output.txt output.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_checksums.txt gungho_model-checksums.txt
         cat timer.txt
         export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
         export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
@@ -150,7 +150,7 @@ jobs:
           --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
           github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
           ci_test: "LFRic all transformations", lfric_version: '"$LFRIC_APPS_REV"', omp_threads: '"$OMP_NUM_THREADS"',
-          system: "GlaDos", compiler:"spack-gfortran-11", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          system: "GlaDos", compiler:"spack-gfortran-14", date: new Date(), elapsed_time: '"$VAR_TIME"',
           num_of_halo_exchanges: '"$VAR_HALOS"'})'
 
     - name: Upload LFRic optimised results

.github/workflows/nemo_tests.yml

@@ -36,7 +36,7 @@
 # This workflow will use a self-hosted runner to perform the more expensive
 # integrations tests that are not run on GHA systems.
 
-name: NEMO Integration Tests
+name: NEMOv4 Integration Tests
 
 on:
   push
@@ -82,28 +82,6 @@ jobs:
         cd lib/profiling/nvidia/
         F90=nvfortran make
 
-    # PSyclone passthrough for 5.0-beta of NEMO.
-    - name: NEMO 5.0 beta passthrough without optimisation
-      run: |
-        . .runner_venv/bin/activate
-        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
-        # PSYCLONE_HOME has `/bin` appended to it by the build system.
-        export PSYCLONE_HOME=${PWD}/.runner_venv
-        export NEMO_DIR=${HOME}/NEMOv5
-        cd $NEMO_DIR
-        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
-        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
-        module load perl/${PERL_VERSION}
-        # We compile at -O1 to permit comparison of the results. (N.B. this test
-        # passes at -O3 with the Intel ifx compiler.)
-        ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p passthrough del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" clean -y
-        ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p passthrough del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4
-        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH/EXP00
-        mpirun -np 4 ./nemo
-        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
-        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
-        echo "Time-stepping duration = " $VAR_TIME
-
     # PSyclone passthrough for MetOffice NEMO
     - name: NEMO MetOffice Passthrough
       run: |

.github/workflows/nemo_v5_tests.yml

@@ -0,0 +1,221 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2024, Science and Technology Facilities Council.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Author S. Siso, STFC Daresbury Lab
+
+# This workflow will use a self-hosted runner to perform the more expensive
+# NEMOv5 integrations tests that are not run on GHA systems.
+
+name: NEMOv5 Integration Tests
+
+on:
+  push
+
+jobs:
+  run_if_on_mirror:
+    if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
+    runs-on: self-hosted
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        # This is required to get the commit history for merge commits for
+        # the ci-skip check below.
+        fetch-depth: '0'
+    - name: Check for [skip ci] in commit message
+      uses: mstachniuk/ci-skip@v1
+      with:
+        # This setting causes the tests to 'fail' if [skip ci] is specified
+        fail-fast: true
+        commit-filter: '[skip ci]'
+    - name: Install dependencies
+      run: |
+        module load python/${PYTHON_VERSION}
+        python -m venv .runner_venv
+        source .runner_venv/bin/activate
+        python -m pip install --upgrade pip
+        # Uncomment the below to use the submodule version of fparser rather
+        # than the latest release from pypi.
+        # pip install external/fparser
+        pip install .
+
+    # PSyclone passthrough for 5.0-beta of NEMO.
+    - name: NEMO 5.0 gfortran passthrough
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%gcc@14
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-fdefault-real-8 -O2 -fcray-pointer -ffree-line-length-none -g"
+
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \
+            add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run test
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00
+        cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg
+        mpirun -np 4 ./nemo
+        tail run.stat
+        # This was produced with gfortran, so we can do an exact diff
+        diff $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+
+    - name: NEMO 5.0 nvidia passthrough
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%nvhpc@24.5
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -Mr8 -O2 -Minline -Mcray=pointer -Mpre -g"
+
+        # Clean up and compile
+        # Without key_mpi_off it fails to compile (even without psyclone)
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \
+          add_key "key_mpi_off key_nosignedzero" -j 4 -v 1
+
+        # Run test
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00
+        cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg
+        ./nemo
+        python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat
+        tail run.stat
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+
+    - name: NEMO 5.0 Intel passthrough
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%oneapi
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g"
+
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/passthrough.py \
+          add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run test
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00
+        cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg
+        mpirun -np 6 ./nemo
+        tail run.stat
+        python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+
+    - name: NEMO 5.0 gfortran OpenMP for CPUs
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%gcc@14
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-fdefault-real-8 -O2 -fcray-pointer -ffree-line-length-none -g -fopenmp"
+
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_GCC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_GCC -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \
+          add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run test
+        cd $NEMO_DIR/tests/BENCH_OMP_THREADING_GCC/EXP00
+        cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg
+        OMP_NUM_THREADS=4 mpirun -np 1 ./nemo
+        tail run.stat
+        python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+
+    - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory)
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%nvhpc@24.5
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        # We compile at -O1 to permit comparison of the results.
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed"
+
+        # Clean up and compile
+        # Without key_mpi_off it fails to compile (even without psyclone)
+        ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \
+          add_key "key_mpi_off key_nosignedzero" -j 4 -v 1
+
+        # Run test
+        cd $NEMO_DIR/tests/BENCH_OMP_OFFLOAD_NVHPC/EXP00
+        cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg
+        ./nemo
+        tail run.stat
+        python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME

changelog

@@ -213,6 +213,9 @@
 
 	73) PR #2700 for #2699. Fix docstring error for Config class.
 
+	74) PR #2698 for #2629. Fix NEMOv5 issues and extend related
+	integration tests.
+
 release 2.5.0 14th of February 2024
 
 	1) PR #2199 for #2189. Fix bugs with missing maps in enter data