#2692 Add separate NEMOv5 integration test workflow

stfc · Aug 27, 2024 · 1647dc7 · 1647dc7
1 parent c58853a
commit 1647dc7
Show file tree

Hide file tree

Showing 3 changed files with 231 additions and 23 deletions.
diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml
@@ -36,7 +36,7 @@
 # This workflow will use a self-hosted runner to perform the more expensive
 # integrations tests that are not run on GHA systems.
 
-name: NEMO Integration Tests
+name: NEMOv4 Integration Tests
 
 on:
   push
@@ -82,28 +82,6 @@ jobs:
         cd lib/profiling/nvidia/
         F90=nvfortran make
 
-    # PSyclone passthrough for 5.0-beta of NEMO.
-    - name: NEMO 5.0 beta passthrough without optimisation
-      run: |
-        . .runner_venv/bin/activate
-        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
-        # PSYCLONE_HOME has `/bin` appended to it by the build system.
-        export PSYCLONE_HOME=${PWD}/.runner_venv
-        export NEMO_DIR=${HOME}/NEMOv5
-        cd $NEMO_DIR
-        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
-        module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION}
-        module load perl/${PERL_VERSION}
-        # We compile at -O1 to permit comparison of the results. (N.B. this test
-        # passes at -O3 with the Intel ifx compiler.)
-        ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p passthrough del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" clean -y
-        ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p passthrough del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4
-        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH/EXP00
-        mpirun -np 4 ./nemo
-        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
-        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
-        echo "Time-stepping duration = " $VAR_TIME
-
     # PSyclone passthrough for MetOffice NEMO
     - name: NEMO MetOffice Passthrough
       run: |

diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml
@@ -0,0 +1,211 @@
+# -----------------------------------------------------------------------------
+# BSD 3-Clause License
+#
+# Copyright (c) 2024, Science and Technology Facilities Council.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+# Author S. Siso, STFC Daresbury Lab
+
+# This workflow will use a self-hosted runner to perform the more expensive
+# integrations tests that are not run on GHA systems.
+
+name: NEMOv5 Integration Tests
+
+on:
+  push
+
+jobs:
+  run_if_on_mirror:
+    if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
+    runs-on: self-hosted
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+        # This is required to get the commit history for merge commits for
+        # the ci-skip check below.
+        fetch-depth: '0'
+    - name: Check for [skip ci] in commit message
+      uses: mstachniuk/ci-skip@v1
+      with:
+        # This setting causes the tests to 'fail' if [skip ci] is specified
+        fail-fast: true
+        commit-filter: '[skip ci]'
+    - name: Install dependencies
+      run: |
+        module load python/${PYTHON_VERSION}
+        python -m venv .runner_venv
+        source .runner_venv/bin/activate
+        python -m pip install --upgrade pip
+        # Uncomment the below to use the submodule version of fparser rather
+        # than the latest release from pypi.
+        # pip install external/fparser
+        pip install .
+
+    # PSyclone passthrough for 5.0-beta of NEMO.
+    - name: NEMO 5.0 gcc passthrough
+      run: |
+        # Set up environment
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%gcc@14
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-O2 -fcray-pointer -ffree-line-length-none -g"
+
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \
+            del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run Orca1
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00
+        mpirun -np 4 ./nemo
+        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+
+    - name: NEMO 5.0 nvidia passthrough
+      run: |
+        # Set up environment
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%nvhpc@24.5
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+        
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        # We compile at -O1 to permit comparison of the results.
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g"
+        
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \
+          del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run Orca1
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00
+        mpirun -np 4 ./nemo
+        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+
+    - name: NEMO 5.0 Intel passthrough
+      run: |
+        # Set up environment
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%oneapi@2024.04
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+        
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        # We compile at -O1 to permit comparison of the results.
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g"
+        
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/passthrough.py \
+          del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run Orca1
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00
+        mpirun -np 4 ./nemo
+        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+        
+    - name: NEMO 5.0 Intel OpenMP for CPUs
+      run: |
+        # Set up environment
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%oneapi@2024.04
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+        
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        # We compile at -O1 to permit comparison of the results.
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g -qopenmp"
+        
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \
+          del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run Orca1
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00
+        OMP_NUM_THREADS=4 mpirun -np 1 ./nemo
+        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
+        
+    - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory)
+      run: |
+        # Set up environment
+        source /apps/spack/spack-upstream/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment%nvhpc@24.5
+        source .runner_venv/bin/activate
+        export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
+        export PSYCLONE_HOME=${PWD}/.runner_venv
+        export NEMO_DIR=${HOME}/NEMOv5
+        
+        # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS
+        # We compile at -O1 to permit comparison of the results.
+        cd $NEMO_DIR
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed"
+        
+        # Clean up and compile
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y
+        ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \
+          del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1
+
+        # Run Orca1
+        cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00
+        mpirun -np 4 ./nemo
+        diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps
+        export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//')
+        echo "Time-stepping duration = " $VAR_TIME
diff --git a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm
@@ -0,0 +1,19 @@
+%PSYCLONE_HOME       ${PSYCLONE_HOME}
+%NCDF_INC            -I${NCDF_F_HOME}/include -I${NCDF_C_HOME}/include -I${HDF5_HOME}/include
+%NCDF_LIB            -L${NCDF_F_HOME}/lib -lnetcdff -L${NCDF_C_HOME}/lib -lnetcdf
+
+%CPP	             cpp
+%FC	                 ${MPIF90} -c
+
+%FFLAGS              ${FCFLAGS}
+%LD                  ${MPIF90}
+%LDFLAGS             
+%FPPFLAGS            -P -traditional
+%AR                  ar
+%ARFLAGS             rs
+%MK                  make
+%USER_INC            %NCDF_INC
+%USER_LIB            %NCDF_LIB
+
+%CC                  ${CC}
+%CFLAGS              -O2