Merge remote-tracking branch 'origin/master' into 2690_value_verifica…

…tion

.github/workflows/lfric_test.yml

@@ -75,6 +75,110 @@ jobs:
         pip install .[test]
         pip install jinja2
 
+    # PSyclone, compile and run MetOffice gungho_model on GPU
+    - name: LFRic GungHo with OpenMP offload
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack load lfric-build-environment%nvhpc
+        source .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
+        # The LFRic source must be patched to workaround bugs in the NVIDIA
+        # compiler's namelist handling.
+        rm -rf ${HOME}/LFRic/gpu_build
+        mkdir -p ${HOME}/LFRic/gpu_build
+        cp -r ${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV} ${HOME}/LFRic/gpu_build/lfric_apps
+        cp -r ${HOME}/LFRic/lfric_core_50869 ${HOME}/LFRic/gpu_build/lfric
+        cd ${HOME}/LFRic/gpu_build
+        patch -p1 < ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_${LFRIC_APPS_REV}_nvidia.patch
+        # Update the compiler definitions to build for GPU
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvfortran_acc.mk lfric/infrastructure/build/fortran/nvfortran.mk
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvc++.mk lfric/infrastructure/build/cxx/.
+        # Update the PSyclone commands to ensure transformed kernels are written
+        # to working directory.
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/psyclone.mk lfric/infrastructure/build/psyclone/.
+        # Update dependencies.sh to point to our patched lfric core.
+        sed -i -e 's/export lfric_core_sources=.*$/export lfric_core_sources\=\/home\/gh_runner\/LFRic\/gpu_build\/lfric/' lfric_apps/dependencies.sh
+        export LFRIC_DIR=${HOME}/LFRic/gpu_build/lfric_apps
+        export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
+        cd ${LFRIC_DIR}
+        # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
+        mkdir -p ${OPT_DIR}
+        cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
+        # Clean previous version and compile again
+        rm -rf applications/gungho_model/working
+        LFRIC_OFFLOAD_DIRECTIVES=omp ./build/local_build.py -a gungho_model -p psyclone-test
+        cd applications/gungho_model/example
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
+        mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its_checksums.txt gungho_model-checksums.txt
+        cat timer.txt
+        export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
+        export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
+        echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME $VAR_HALOS >> ${HOME}/store_results/lfric_omp_performance_history
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "LFRic OpenMP offloading", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos",
+          compiler:"spack-nvhpc-24.5", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          num_of_halo_exchanges: '"$VAR_HALOS"'})'
+
+    # PSyclone, compile and run MetOffice gungho_model on GPU
+    - name: LFRic GungHo with OpenACC offload
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack load lfric-build-environment%nvhpc
+        source .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
+        # The LFRic source must be patched to workaround bugs in the NVIDIA
+        # compiler's namelist handling.
+        rm -rf ${HOME}/LFRic/gpu_build
+        mkdir -p ${HOME}/LFRic/gpu_build
+        cp -r ${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV} ${HOME}/LFRic/gpu_build/lfric_apps
+        cp -r ${HOME}/LFRic/lfric_core_50869 ${HOME}/LFRic/gpu_build/lfric
+        cd ${HOME}/LFRic/gpu_build
+        patch -p1 < ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_${LFRIC_APPS_REV}_nvidia.patch
+        # Update the compiler definitions to build for GPU
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvfortran_acc.mk lfric/infrastructure/build/fortran/nvfortran.mk
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvc++.mk lfric/infrastructure/build/cxx/.
+        # Update the PSyclone commands to ensure transformed kernels are written
+        # to working directory.
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/psyclone.mk lfric/infrastructure/build/psyclone/.
+        # Update dependencies.sh to point to our patched lfric core.
+        sed -i -e 's/export lfric_core_sources=.*$/export lfric_core_sources\=\/home\/gh_runner\/LFRic\/gpu_build\/lfric/' lfric_apps/dependencies.sh
+        export LFRIC_DIR=${HOME}/LFRic/gpu_build/lfric_apps
+        export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
+        cd ${LFRIC_DIR}
+        # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
+        mkdir -p ${OPT_DIR}
+        cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
+        # Clean previous version and compile again
+        rm -rf applications/gungho_model/working
+        LFRIC_OFFLOAD_DIRECTIVES=acc ./build/local_build.py -a gungho_model -p psyclone-test
+        cd applications/gungho_model/example
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
+        mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its_checksums.txt gungho_model-checksums.txt
+        cat timer.txt
+        export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
+        export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
+        echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME $VAR_HALOS >> ${HOME}/store_results/lfric_acc_performance_history
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "LFRic OpenACC", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos",
+          compiler:"spack-nvhpc-24.5", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          num_of_halo_exchanges: '"$VAR_HALOS"'})'
+
     # PSyclone, compile and run MetOffice LFRic with 6 MPI ranks
     - name: LFRic passthrough (with DistributedMemory)
       run: |
@@ -91,9 +195,9 @@ jobs:
         ./build/local_build.py -a gungho_model -v
         # Run
         cd applications/gungho_model/example
-        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
         mpirun -n 6 ../bin/gungho_model configuration.nml |& tee output.txt
-        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_checksums.txt gungho_model-checksums.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its_checksums.txt gungho_model-checksums.txt
         cat timer.txt
         export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
         export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
@@ -108,13 +212,6 @@ jobs:
           compiler:"spack-gfortran-14", date: new Date(), elapsed_time: '"$VAR_TIME"',
           num_of_halo_exchanges: '"$VAR_HALOS"'})'
 
-    - name: Upload LFRic passthrough results
-      uses: exuanbo/actions-deploy-gist@v1
-      with:
-        token: ${{ secrets.GIST_TOKEN }}
-        gist_id: a4049a0fc0a0a11651a5ce6a04d76160
-        file_path: ../../../../store_results/lfric_passthrough_performance_history
-
     # PSyclone, compile and run MetOffice LFRic with all optimisations and 6 OpenMP threads
     - name: LFRic with all transformations
       run: |
@@ -127,18 +224,18 @@ jobs:
         export LFRIC_DIR=${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV}
         export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
         cd ${LFRIC_DIR}
-        # Psyclone scripts must now be under 'optimisation' and be called 'global.py'
+        # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
         mkdir -p applications/gungho_model/optimisation/psyclone-test
         cp ${PSYCLONE_LFRIC_DIR}/everything_everywhere_all_at_once.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf applications/gungho_model/working
         ./build/local_build.py -a gungho_model -p psyclone-test -v
         # Run
         cd applications/gungho_model/example
-        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
         export OMP_NUM_THREADS=6
         mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
-        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_checksums.txt gungho_model-checksums.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its_checksums.txt gungho_model-checksums.txt
         cat timer.txt
         export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
         export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
@@ -152,10 +249,3 @@ jobs:
           ci_test: "LFRic all transformations", lfric_version: '"$LFRIC_APPS_REV"', omp_threads: '"$OMP_NUM_THREADS"',
           system: "GlaDos", compiler:"spack-gfortran-14", date: new Date(), elapsed_time: '"$VAR_TIME"',
           num_of_halo_exchanges: '"$VAR_HALOS"'})'
-
-    - name: Upload LFRic optimised results
-      uses: exuanbo/actions-deploy-gist@v1
-      with:
-        token: ${{ secrets.GIST_TOKEN }}
-        gist_id: a4049a0fc0a0a11651a5ce6a04d76160
-        file_path: ../../../../store_results/lfric_optimised_performance_history

changelog

@@ -94,7 +94,7 @@
 	34) PR #2574 for #2196. Add support for use-intrinsic imports.
 
 	35) PR #2566 towards #2156. Bug fixes for routines declared as
-	ELEMENTAL or IMPURE. (Alters the Fortran backend and 
+	ELEMENTAL or IMPURE. (Alters the Fortran backend and
 	Routine.copy().)
 
 	36) PR #2572 for #2195. Improves SymbolTable.__str__ such that
@@ -110,7 +110,7 @@
 	40) PR #2598 towards #2543. Refresh the PSyKAl-related documentation
 	in both the User and Developer Guides.
 
-	41) PR #2611. Update Integration tests nvhpc mpi library path. 
+	41) PR #2611. Update Integration tests nvhpc mpi library path.
 
 	42) PR #2614 towards #2604. Fixes broken links to the docs in the
 	various tutorial READMEs.
@@ -216,6 +216,30 @@
 	74) PR #2698 for #2629. Fix NEMOv5 issues and extend related
 	integration tests.
 
+	75) PR #2710 for #2665. Improves error messages for line-length and
+	kernel arguments.
+
+	76) PR #2692 for #2691. Increase recursion limit when handling
+	conditional statements with many branches. Disable final tree copy
+	when generating code.
+
+	77) PR #2708 for #2663. Add LFRic OpenACC in the integration tests.
+
+	78) PR #2678 for #2636. Alter LFRic PSy-layer to lookup nlevels for
+	each kernel
+
+	79) PR #2687 for #2684. Add support for WHERE constructs that contain
+	references without range-notation (and prevent some incorrect cases).
+
+	80) PR #2596 for #2592 and #2582. Routine nodes manage their own
+	symbols from their parent scope.
+
+	81) PR #2725 to close #717. Removes some TODOs and associated utility code
+	from the fparser2 frontend that is now unused.
+
+	82) PR #2733 for #2730. Adds OpenMP offloading support for LFRic plus
+	associated integration test.
+
 release 2.5.0 14th of February 2024
 
 	1) PR #2199 for #2189. Fix bugs with missing maps in enter data
@@ -302,7 +326,7 @@ release 2.5.0 14th of February 2024
 	validating LFRic kernel arguments.
 
 	31) PR #2388 for #2323. Rename the array reduction 2code transformations
-	to use the 2loop postfix and improve their implementation. 
+	to use the 2loop postfix and improve their implementation.
 
 	32) PR #2400 for #1508. Add flag/config option to disable backend checks.
 
@@ -330,7 +354,7 @@ release 2.5.0 14th of February 2024
 
 	41) PR #2393 for #2392. Split and rename DynKernMetadata into
 	into domain/lfric module (LFRicKernMetadata class).
-	
+
 	42) PR #2240 for #2132. Update Integration test software stack.
 
 	43) PR #2454 for #2418. Fix OMPTaskTrans attempts to inline IntrinsicCalls.
@@ -372,7 +396,7 @@ release 2.5.0 14th of February 2024
 	55) PR #2465 towards #2004. Move same_range functionality to
 	ArrayMixin and other improvements.
 
-	56) PR #2372 towards #1960. Fix bugs on Fortran WHERE canonicalisation. 
+	56) PR #2372 towards #1960. Fix bugs on Fortran WHERE canonicalisation.
 
 	57) PR #2489 for #2488. Unify generic LoopFuseTrans and allow renaming
 	the second loop variable for certain cases.
@@ -930,7 +954,7 @@ release 2.4.0 29th of September 2023
 	script.
 
 	183) PR #2182 for #2179. Fix LFRic eg14 nvfortran compilation and add
-	nvfortran examples and tutorials steps in the Integration tests. 
+	nvfortran examples and tutorials steps in the Integration tests.
 
 	184) PR #2250 for #2248 and #1575. Fixes bug in line-length limiter
 	to prevent it breaking a line before the first non-whitepsace
@@ -1085,7 +1109,7 @@ release 2.3.0 9th June 2022
 	17) PR #1702 for #1695. Correction to LFRic infrastructure ('emdi') to
 	bring up-to-date with current LFRic usage.
 
-	18) PR #1705. Corrections for the NEMO tutorial. 
+	18) PR #1705. Corrections for the NEMO tutorial.
 
 	19) PR #1692 for #1688. Prevents scoping regions from sharing symbol
 	tables. Adds attach() and detach() methods to SymbolTable for
@@ -1101,7 +1125,7 @@ release 2.3.0 9th June 2022
 
 	23) PR #1725. Allows parsing an empty Fortran file.
 
-	24) PR #1716 for #1713. Fix bug in matmul transformation. 
+	24) PR #1716 for #1713. Fix bug in matmul transformation.
 
 	25) PR #1693 for #1661. Improve dependency analysis tools.
 
@@ -1119,10 +1143,10 @@ release 2.3.0 9th June 2022
 	30) PR #1749 for #1748. Updates version of fparser specified in setup.py
 	to 0.0.15.
 
-	31) PR #1740 for #1738. Update the operator implementation in the 
+	31) PR #1740 for #1738. Update the operator implementation in the
 	cut-down LFRic infrastructure and change the USE statements generated
 	for CMA operators.
- 
+
 release 2.2.0 17th March 2022
 
 	1) PR #1439 for #1074. Adds a GOcean example for the use of the NaN-
@@ -2051,7 +2075,7 @@ release 2.0.0 28th April 2021
 	156) PR #1156 for #874. Removes support for old-style LFRic arg_type
 	descriptor. Primitive type of all kernel arguments must now be
 	specified.
-	
+
 	157) PR #1182 for #1181. Add OpenACC directives support to the PSyIR
 	Fortran backend.
 

doc/user_guide/dynamo0p3.rst

@@ -1934,7 +1934,9 @@ conventions, are:
 1) If an LMA operator is passed then include the ``cells`` argument.
    ``cells`` is an ``integer`` of kind ``i_def`` and has intent ``in``.
 2) Include ``nlayers``, the number of layers in a column. ``nlayers``
-   is an ``integer`` of kind ``i_def`` and has intent ``in``.
+   is an ``integer`` of kind ``i_def`` and has intent ``in``. PSyclone
+   will obtain the value of ``nlayers`` to use for a particular kernel
+   from the first field (in the argument list) that is written to.
 3) For each scalar/field/vector_field/operator in the order specified by
    the meta_args metadata: