Merge branch 'master' into 2704_access_improvements

.github/workflows/lfric_test.yml

@@ -75,6 +75,58 @@ jobs:
         pip install .[test]
         pip install jinja2
 
+    # PSyclone, compile and run MetOffice gungho_model on GPU
+    - name: LFRic GungHo with OpenMP offload
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack load lfric-build-environment%nvhpc
+        source .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
+        # The LFRic source must be patched to workaround bugs in the NVIDIA
+        # compiler's namelist handling.
+        rm -rf ${HOME}/LFRic/gpu_build
+        mkdir -p ${HOME}/LFRic/gpu_build
+        cp -r ${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV} ${HOME}/LFRic/gpu_build/lfric_apps
+        cp -r ${HOME}/LFRic/lfric_core_50869 ${HOME}/LFRic/gpu_build/lfric
+        cd ${HOME}/LFRic/gpu_build
+        patch -p1 < ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_${LFRIC_APPS_REV}_nvidia.patch
+        # Update the compiler definitions to build for GPU
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvfortran_acc.mk lfric/infrastructure/build/fortran/nvfortran.mk
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvc++.mk lfric/infrastructure/build/cxx/.
+        # Update the PSyclone commands to ensure transformed kernels are written
+        # to working directory.
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/psyclone.mk lfric/infrastructure/build/psyclone/.
+        # Update dependencies.sh to point to our patched lfric core.
+        sed -i -e 's/export lfric_core_sources=.*$/export lfric_core_sources\=\/home\/gh_runner\/LFRic\/gpu_build\/lfric/' lfric_apps/dependencies.sh
+        export LFRIC_DIR=${HOME}/LFRic/gpu_build/lfric_apps
+        export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
+        cd ${LFRIC_DIR}
+        # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
+        mkdir -p ${OPT_DIR}
+        cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
+        # Clean previous version and compile again
+        rm -rf applications/gungho_model/working
+        LFRIC_OFFLOAD_DIRECTIVES=omp ./build/local_build.py -a gungho_model -p psyclone-test
+        cd applications/gungho_model/example
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
+        mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its_checksums.txt gungho_model-checksums.txt
+        cat timer.txt
+        export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
+        export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
+        echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME $VAR_HALOS >> ${HOME}/store_results/lfric_omp_performance_history
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "LFRic OpenMP offloading", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos",
+          compiler:"spack-nvhpc-24.5", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          num_of_halo_exchanges: '"$VAR_HALOS"'})'
+
     # PSyclone, compile and run MetOffice gungho_model on GPU
     - name: LFRic GungHo with OpenACC offload
       run: |
@@ -105,10 +157,10 @@ jobs:
         cd ${LFRIC_DIR}
         # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
         mkdir -p ${OPT_DIR}
-        cp ${PSYCLONE_LFRIC_DIR}/acc_parallel.py ${OPT_DIR}/global.py
+        cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf applications/gungho_model/working
-        ./build/local_build.py -a gungho_model -p psyclone-test
+        LFRIC_OFFLOAD_DIRECTIVES=acc ./build/local_build.py -a gungho_model -p psyclone-test
         cd applications/gungho_model/example
         cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
         mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt

changelog

@@ -235,7 +235,10 @@
 	symbols from their parent scope.
 
 	81) PR #2725 to close #717. Removes some TODOs and associated utility code
-        from the fparser2 frontend that is now unused.
+	from the fparser2 frontend that is now unused.
+
+	82) PR #2733 for #2730. Adds OpenMP offloading support for LFRic plus
+	associated integration test.
 
 release 2.5.0 14th of February 2024
 

examples/lfric/scripts/KGOs/lfric_3269_nvidia.patch

@@ -57,65 +57,6 @@ index 19c9cff9..b5cd3014 100644
  	$(call MESSAGE,Compiled,$<)
 
 
-diff --git a/lfric/infrastructure/build/cxx/nvc++.mk b/lfric/infrastructure/build/cxx/nvc++.mk
-new file mode 100644
-index 00000000..13b17a10
---- /dev/null
-+++ b/lfric/infrastructure/build/cxx/nvc++.mk
-@@ -0,0 +1,9 @@
-+##############################################################################
-+# (c) Crown copyright 2017 Met Office. All rights reserved.
-+# The file LICENCE, distributed with this code, contains details of the terms
-+# under which the code may be used.
-+##############################################################################
-+
-+$(info ** Chosen NVC++ compiler)
-+
-+CXX_RUNTIME_LIBRARY=stdc++
-diff --git a/lfric/infrastructure/build/fortran/nvfortran.mk b/lfric/infrastructure/build/fortran/nvfortran.mk
-new file mode 100644
-index 00000000..cfed52c1
---- /dev/null
-+++ b/lfric/infrastructure/build/fortran/nvfortran.mk
-@@ -0,0 +1,38 @@
-+##############################################################################
-+# Copyright (c) 2017,  Met Office, on behalf of HMSO and Queen's Printer
-+# For further details please refer to the file LICENCE.original which you
-+# should have received as part of this distribution.
-+##############################################################################
-+# Various things specific to the Portland Fortran compiler.
-+##############################################################################
-+#
-+# This macro is evaluated now (:= syntax) so it may be used as many times as
-+# desired without wasting time rerunning it.
-+#
-+F_MOD_DESTINATION_ARG = -module$(SPACE)
-+OPENMP_ARG            = -mp
-+
-+FFLAGS_COMPILER           =
-+FFLAGS_NO_OPTIMISATION    = -O0
-+FFLAGS_SAFE_OPTIMISATION  = -O2
-+FFLAGS_RISKY_OPTIMISATION = -O4
-+FFLAGS_DEBUG              = -g -traceback
-+FFLAGS_RUNTIME            = -Mchkptr -Mchkstk
-+# Option for checking code meets Fortran standard (not available for PGI)
-+FFLAGS_FORTRAN_STANDARD   =
-+
-+LDFLAGS_COMPILER = -g
-+
-+FPP = cpp -traditional-cpp
-+FPPFLAGS = -P
-+FC = mpif90
-+
-+# FS#34981 (nvbug 4648082)
-+science/src/um/src/atmosphere/large_scale_precipitation/ls_ppnc.o: private FFLAGS_RUNTIME = -Mchkstk
-+
-+# FS#35751
-+mesh/create_mesh_mod.o: private FFLAGS_RUNTIME = -Mchkstk
-+
-+# 24.3
-+science/src/socrates/src/cosp_github/subsample_and_optics_example/optics/quickbeam_optics/optics_lib.o: private FFLAGS_SAFE_OPTIMISATION = -O1
-+science/src/socrates/src/cosp_github/subsample_and_optics_example/optics/quickbeam_optics/optics_lib.o: private FFLAGS_RISKY_OPTIMISATION = -O1
 diff --git a/lfric/infrastructure/build/tools/DependencyRules b/lfric/infrastructure/build/tools/DependencyRules
 index 9d4db390..e37384fc 100755
 --- a/lfric/infrastructure/build/tools/DependencyRules

examples/lfric/scripts/KGOs/nvfortran_acc.mk

@@ -20,9 +20,20 @@ FFLAGS_DEBUG              = -g -traceback
 FFLAGS_RUNTIME            = -Mchkptr -Mchkstk
 # Option for checking code meets Fortran standard (not available for PGI)
 FFLAGS_FORTRAN_STANDARD   =
-OPENMP_ARG = -acc=gpu -gpu=managed -mp=multicore
 
-LDFLAGS_COMPILER = -g -acc=gpu -gpu=managed -mp=multicore -cuda
+# Flags for OpenMP threading / OpenMP offloading / OpenACC Offloading
+# The LFRIC_OFFLOAD_DIRECTIVES env_variable is also queried in the PSyclone
+# script to generate matching directives
+ifeq ("$(LFRIC_OFFLOAD_DIRECTIVES)", "omp")
+	OPENMP_ARG = -mp=gpu -gpu=managed
+	LDFLAGS_COMPILER = -mp=gpu -gpu=managed -cuda
+else ifeq ("$(LFRIC_OFFLOAD_DIRECTIVES)", "acc")
+	OPENMP_ARG = -acc=gpu -gpu=managed -mp=multicore
+	LDFLAGS_COMPILER = -acc=gpu -gpu=managed -mp=multicore -cuda
+else
+	OPENMP_ARG = -mp
+	LDFLAGS_COMPILER = -mp
+endif
 
 FPP = cpp -traditional-cpp
 FPPFLAGS = -P

examples/lfric/scripts/Makefile

@@ -45,7 +45,7 @@ transform: ${SCRIPTS}
 .PHONY: ${SCRIPTS}
 
 ${SCRIPTS}:
-	${PSYCLONE} -api lfric -s ./$@ ../eg3/solver_mod.x90 -oalg /dev/null -opsy /dev/null
+	LFRIC_OFFLOAD_DIRECTIVES=acc ${PSYCLONE} -api lfric -s ./$@ ../eg3/solver_mod.x90 -oalg /dev/null -opsy /dev/null
 
 compile: transform
 	@echo "No compilation supported for lfric/scripts examples"