From eeb32b29be3e6e709fb73f3645f4488704cf7c92 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Mon, 26 Aug 2024 16:08:33 +0100 Subject: [PATCH 01/16] #2692 Fix NEMOv5 issues --- src/psyclone/psyir/backend/fortran.py | 9 +++-- src/psyclone/psyir/frontend/fparser2.py | 2 +- .../psyir/symbols/generic_interface_symbol.py | 6 ++-- src/psyclone/psyir/symbols/symbol_table.py | 2 ++ .../tests/psyir/backend/fortran_test.py | 24 +++++++++++++ .../frontend/fparser2_interface_block_test.py | 36 +++++++++++++++++-- 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/psyclone/psyir/backend/fortran.py b/src/psyclone/psyir/backend/fortran.py index 7190090c9a..b1c2a00e7a 100644 --- a/src/psyclone/psyir/backend/fortran.py +++ b/src/psyclone/psyir/backend/fortran.py @@ -803,6 +803,11 @@ def gen_access_stmts(self, symbol_table): if isinstance(symbol, RoutineSymbol) and symbol is itself: continue + # Skip _PSYCLONE_INTERNAL_* symbols + if (isinstance(symbol, RoutineSymbol) and + symbol.name.startswith("_PSYCLONE_INTERNAL_")): + continue + # It doesn't matter whether this symbol has a local or import # interface - its accessibility in *this* context is determined # by the local accessibility statements. e.g. if we are @@ -1171,8 +1176,8 @@ def routine_node(self, node): container = node.ancestor(Container) rsym = None if container: - rsym = container.symbol_table.get_symbols().get( - node.name, None) + # TODO: Will be node.symbol + rsym = container.symbol_table.lookup(node.name, otherwise=None) prefix = "" if rsym: if rsym.is_elemental: diff --git a/src/psyclone/psyir/frontend/fparser2.py b/src/psyclone/psyir/frontend/fparser2.py index 1423dc2bde..e25fd1600d 100644 --- a/src/psyclone/psyir/frontend/fparser2.py +++ b/src/psyclone/psyir/frontend/fparser2.py @@ -2450,7 +2450,7 @@ def _process_interface_block(self, node, symbol_table, visibility_map): # UnsupportedFortranType with an internal name as we do # for unnamed interfaces. symbol_table.new_symbol( - root_name=f"_psyclone_internal_{name}", + root_name=f"_PSYCLONE_INTERNAL_{name}", symbol_type=RoutineSymbol, datatype=UnsupportedFortranType(str(node).lower()), visibility=vis) diff --git a/src/psyclone/psyir/symbols/generic_interface_symbol.py b/src/psyclone/psyir/symbols/generic_interface_symbol.py index 9a25e642bb..c0427918b8 100644 --- a/src/psyclone/psyir/symbols/generic_interface_symbol.py +++ b/src/psyclone/psyir/symbols/generic_interface_symbol.py @@ -45,8 +45,10 @@ class GenericInterfaceSymbol(RoutineSymbol): different callable routines. :param str name: name of the interface. - :param routines: the routines that this interface provides access to. - :type routines: list[:py:class:`psyclone.psyir.symbols.RoutineSymbol`] + :param routines: the routines that this interface provides access to, + nd whether or not each of them is a module procedure. + :type routines: list[ + tuple[:py:class:`psyclone.psyir.symbols.RoutineSymbol`, bool]] :param kwargs: additional keyword arguments provided by :py:class:`psyclone.psyir.symbols.TypedSymbol` :type kwargs: unwrapped dict. diff --git a/src/psyclone/psyir/symbols/symbol_table.py b/src/psyclone/psyir/symbols/symbol_table.py index 76335ec635..b84439b256 100644 --- a/src/psyclone/psyir/symbols/symbol_table.py +++ b/src/psyclone/psyir/symbols/symbol_table.py @@ -568,6 +568,8 @@ def add(self, new_symbol, tag=None): if not isinstance(new_symbol, Symbol): raise InternalError(f"Symbol '{new_symbol}' is not a symbol, but " f"'{type(new_symbol).__name__}'.'") + if new_symbol.name == "_psyclone_internal_div_hor": + import pdb; pdb.set_trace() key = self._normalize(new_symbol.name) if key in self._symbols: diff --git a/src/psyclone/tests/psyir/backend/fortran_test.py b/src/psyclone/tests/psyir/backend/fortran_test.py index 1513674d33..ccdb8bbef9 100644 --- a/src/psyclone/tests/psyir/backend/fortran_test.py +++ b/src/psyclone/tests/psyir/backend/fortran_test.py @@ -832,6 +832,30 @@ def test_gen_access_stmts(fortran_writer): assert code.strip() == "public :: my_sub1, some_var, overloaded" +def test_gen_access_stmts_avoids_internal(fortran_reader, fortran_writer): + ''' + Tests for the gen_access_stmts does not list the psyclone internal symbols + ''' + test_module = ''' + module test_mod + private + interface test + module procedure test, test_code + end interface test + public test + contains + subroutine test_code() + end subroutine test_code + subroutine test() + end subroutine test + end module test_mod + ''' + psyir = fortran_reader.psyir_from_source(test_module) + # Check that the internal symbol exists but is not listed + assert psyir.children[0].symbol_table.lookup("_PSYCLONE_INTERNAL_test") + assert "_PSYCLONE_INTERNAL_test" not in fortran_writer(psyir) + + def test_fw_exception(fortran_writer): '''Check the FortranWriter class instance raises an exception if an unsupported PSyIR node is found. diff --git a/src/psyclone/tests/psyir/frontend/fparser2_interface_block_test.py b/src/psyclone/tests/psyir/frontend/fparser2_interface_block_test.py index 270b319e1a..32f64ec1cb 100644 --- a/src/psyclone/tests/psyir/frontend/fparser2_interface_block_test.py +++ b/src/psyclone/tests/psyir/frontend/fparser2_interface_block_test.py @@ -97,11 +97,11 @@ def test_named_interface(fortran_reader, mod_txt): @pytest.mark.parametrize("mod_txt", ["", "module "]) -def test_named_interface_declared(fortran_reader, mod_txt): +def test_named_interface_declared(fortran_reader, fortran_writer, mod_txt): ''' Test that the frontend creates a RoutineSymbol of UnsupportedFortranType for a named interface block when the symbol name has already been declared (as will be the case for a structure - constructor). ''' + constructor or a subroutine with the same name). ''' test_module = f''' module test_mod type test @@ -132,6 +132,38 @@ def test_named_interface_declared(fortran_reader, mod_txt): f"end interface test") assert test_symbol.visibility == Symbol.Visibility.PUBLIC + test_module = f''' + module test_mod + private + interface test + {mod_txt}procedure test, test_code + end interface test + public test + contains + subroutine test_code() + end subroutine test_code + subroutine test() + end subroutine test + end module test_mod + ''' + file_container = fortran_reader.psyir_from_source(test_module) + container = file_container.children[0] + assert isinstance(container, Container) + # routine symbol + assert container.symbol_table.lookup("test") + test_symbol = container.symbol_table.lookup("test") + assert isinstance(test_symbol, RoutineSymbol) + # interface symbol + assert container.symbol_table.lookup("_psyclone_internal_test") + test_symbol = container.symbol_table.lookup("_psyclone_internal_test") + assert isinstance(test_symbol, RoutineSymbol) + assert isinstance(test_symbol.datatype, UnsupportedFortranType) + assert test_symbol.datatype.declaration == ( + f"interface test\n" + f" {mod_txt}procedure test, test_code\n" + f"end interface test") + assert test_symbol.visibility == Symbol.Visibility.PUBLIC + def test_named_interface_no_routine_symbol(fortran_reader): ''' From c58853ab970580075ee341df79e82c98b396f3b1 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Mon, 26 Aug 2024 16:48:59 +0100 Subject: [PATCH 02/16] #2692 Fix flake8 and TODO --- src/psyclone/psyir/backend/fortran.py | 2 +- src/psyclone/psyir/symbols/symbol_table.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/psyclone/psyir/backend/fortran.py b/src/psyclone/psyir/backend/fortran.py index b1c2a00e7a..ee760bb484 100644 --- a/src/psyclone/psyir/backend/fortran.py +++ b/src/psyclone/psyir/backend/fortran.py @@ -1176,7 +1176,7 @@ def routine_node(self, node): container = node.ancestor(Container) rsym = None if container: - # TODO: Will be node.symbol + # TODO #2592: When this is implemented it will be node.symbol rsym = container.symbol_table.lookup(node.name, otherwise=None) prefix = "" if rsym: diff --git a/src/psyclone/psyir/symbols/symbol_table.py b/src/psyclone/psyir/symbols/symbol_table.py index b84439b256..76335ec635 100644 --- a/src/psyclone/psyir/symbols/symbol_table.py +++ b/src/psyclone/psyir/symbols/symbol_table.py @@ -568,8 +568,6 @@ def add(self, new_symbol, tag=None): if not isinstance(new_symbol, Symbol): raise InternalError(f"Symbol '{new_symbol}' is not a symbol, but " f"'{type(new_symbol).__name__}'.'") - if new_symbol.name == "_psyclone_internal_div_hor": - import pdb; pdb.set_trace() key = self._normalize(new_symbol.name) if key in self._symbols: From 1647dc75c6853c8e4ad9a523092308e4dfd1cd94 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Tue, 27 Aug 2024 15:59:47 +0100 Subject: [PATCH 03/16] #2692 Add separate NEMOv5 integration test workflow --- .github/workflows/nemo_tests.yml | 24 +- .github/workflows/nemo_v5_tests.yml | 211 ++++++++++++++++++ .../nemo/scripts/KGOs/arch-linux_spack.fcm | 19 ++ 3 files changed, 231 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/nemo_v5_tests.yml create mode 100644 examples/nemo/scripts/KGOs/arch-linux_spack.fcm diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml index a051263430..bbcd2bf468 100644 --- a/.github/workflows/nemo_tests.yml +++ b/.github/workflows/nemo_tests.yml @@ -36,7 +36,7 @@ # This workflow will use a self-hosted runner to perform the more expensive # integrations tests that are not run on GHA systems. -name: NEMO Integration Tests +name: NEMOv4 Integration Tests on: push @@ -82,28 +82,6 @@ jobs: cd lib/profiling/nvidia/ F90=nvfortran make - # PSyclone passthrough for 5.0-beta of NEMO. - - name: NEMO 5.0 beta passthrough without optimisation - run: | - . .runner_venv/bin/activate - export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts - # PSYCLONE_HOME has `/bin` appended to it by the build system. - export PSYCLONE_HOME=${PWD}/.runner_venv - export NEMO_DIR=${HOME}/NEMOv5 - cd $NEMO_DIR - module load nvidia-hpcsdk/${NVFORTRAN_VERSION} - module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION} - module load perl/${PERL_VERSION} - # We compile at -O1 to permit comparison of the results. (N.B. this test - # passes at -O3 with the Intel ifx compiler.) - ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p passthrough del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" clean -y - ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p passthrough del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 - cd $NEMO_DIR/tests/BENCH_PASSTHROUGH/EXP00 - mpirun -np 4 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - echo "Time-stepping duration = " $VAR_TIME - # PSyclone passthrough for MetOffice NEMO - name: NEMO MetOffice Passthrough run: | diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml new file mode 100644 index 0000000000..13453e8eb5 --- /dev/null +++ b/.github/workflows/nemo_v5_tests.yml @@ -0,0 +1,211 @@ +# ----------------------------------------------------------------------------- +# BSD 3-Clause License +# +# Copyright (c) 2024, Science and Technology Facilities Council. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ----------------------------------------------------------------------------- +# Author S. Siso, STFC Daresbury Lab + +# This workflow will use a self-hosted runner to perform the more expensive +# integrations tests that are not run on GHA systems. + +name: NEMOv5 Integration Tests + +on: + push + +jobs: + run_if_on_mirror: + if: ${{ github.repository == 'stfc/PSyclone-mirror' }} + runs-on: self-hosted + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + # This is required to get the commit history for merge commits for + # the ci-skip check below. + fetch-depth: '0' + - name: Check for [skip ci] in commit message + uses: mstachniuk/ci-skip@v1 + with: + # This setting causes the tests to 'fail' if [skip ci] is specified + fail-fast: true + commit-filter: '[skip ci]' + - name: Install dependencies + run: | + module load python/${PYTHON_VERSION} + python -m venv .runner_venv + source .runner_venv/bin/activate + python -m pip install --upgrade pip + # Uncomment the below to use the submodule version of fparser rather + # than the latest release from pypi. + # pip install external/fparser + pip install . + + # PSyclone passthrough for 5.0-beta of NEMO. + - name: NEMO 5.0 gcc passthrough + run: | + # Set up environment + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%gcc@14 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 + + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-O2 -fcray-pointer -ffree-line-length-none -g" + + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ + del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + + # Run Orca1 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00 + mpirun -np 4 ./nemo + diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME + + - name: NEMO 5.0 nvidia passthrough + run: | + # Set up environment + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%nvhpc@24.5 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 + + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # We compile at -O1 to permit comparison of the results. + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" + + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ + del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + + # Run Orca1 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + mpirun -np 4 ./nemo + diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME + + - name: NEMO 5.0 Intel passthrough + run: | + # Set up environment + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%oneapi@2024.04 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 + + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # We compile at -O1 to permit comparison of the results. + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g" + + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ + del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + + # Run Orca1 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 + mpirun -np 4 ./nemo + diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME + + - name: NEMO 5.0 Intel OpenMP for CPUs + run: | + # Set up environment + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%oneapi@2024.04 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 + + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # We compile at -O1 to permit comparison of the results. + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g -qopenmp" + + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ + del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + + # Run Orca1 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 + OMP_NUM_THREADS=4 mpirun -np 1 ./nemo + diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME + + - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory) + run: | + # Set up environment + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%nvhpc@24.5 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 + + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # We compile at -O1 to permit comparison of the results. + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed" + + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ + del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + + # Run Orca1 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + mpirun -np 4 ./nemo + diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm new file mode 100644 index 0000000000..3559796441 --- /dev/null +++ b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm @@ -0,0 +1,19 @@ +%PSYCLONE_HOME ${PSYCLONE_HOME} +%NCDF_INC -I${NCDF_F_HOME}/include -I${NCDF_C_HOME}/include -I${HDF5_HOME}/include +%NCDF_LIB -L${NCDF_F_HOME}/lib -lnetcdff -L${NCDF_C_HOME}/lib -lnetcdf + +%CPP cpp +%FC ${MPIF90} -c + +%FFLAGS ${FCFLAGS} +%LD ${MPIF90} +%LDFLAGS +%FPPFLAGS -P -traditional +%AR ar +%ARFLAGS rs +%MK make +%USER_INC %NCDF_INC +%USER_LIB %NCDF_LIB + +%CC ${CC} +%CFLAGS -O2 From 9184963481103801baf8558105914fd86088b440 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Tue, 27 Aug 2024 16:08:26 +0100 Subject: [PATCH 04/16] #2692 Upgrade integration test Spack version --- .github/workflows/lfric_test.yml | 8 ++++---- .github/workflows/nemo_v5_tests.yml | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml index 49cff9c194..ffd9674c3a 100644 --- a/.github/workflows/lfric_test.yml +++ b/.github/workflows/lfric_test.yml @@ -79,8 +79,8 @@ jobs: - name: LFRic passthrough (with DistributedMemory) run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh - spack load lfric-buildenv%gcc + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + spack load lfric-build-envirnoment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg @@ -119,8 +119,8 @@ jobs: - name: LFRic with all transformations run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh - spack load lfric-buildenv%gcc + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + spack load lfric-build-envirnoment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index 13453e8eb5..b44247fa11 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -74,7 +74,7 @@ jobs: - name: NEMO 5.0 gcc passthrough run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh spack unload && spack load nemo-build-environment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts @@ -101,7 +101,7 @@ jobs: - name: NEMO 5.0 nvidia passthrough run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh spack unload && spack load nemo-build-environment%nvhpc@24.5 source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts @@ -129,7 +129,7 @@ jobs: - name: NEMO 5.0 Intel passthrough run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh spack unload && spack load nemo-build-environment%oneapi@2024.04 source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts @@ -157,7 +157,7 @@ jobs: - name: NEMO 5.0 Intel OpenMP for CPUs run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh spack unload && spack load nemo-build-environment%oneapi@2024.04 source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts @@ -185,7 +185,7 @@ jobs: - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory) run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh spack unload && spack load nemo-build-environment%nvhpc@24.5 source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts From 7da0d158f5fee190e89cc26a8754ed03adbd3b10 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 28 Aug 2024 07:43:32 +0100 Subject: [PATCH 05/16] #2692 Fix integration test issues --- .github/workflows/lfric_test.yml | 4 ++-- examples/nemo/scripts/KGOs/arch-linux_spack.fcm | 4 ++-- examples/nemo/scripts/omp_cpu_trans.py | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml index ffd9674c3a..cf353dc121 100644 --- a/.github/workflows/lfric_test.yml +++ b/.github/workflows/lfric_test.yml @@ -80,7 +80,7 @@ jobs: run: | # Set up environment source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack load lfric-build-envirnoment%gcc@14 + spack load lfric-build-environment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg @@ -120,7 +120,7 @@ jobs: run: | # Set up environment source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack load lfric-build-envirnoment%gcc@14 + spack load lfric-build-environment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg diff --git a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm index 3559796441..1a4f06227c 100644 --- a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm +++ b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm @@ -4,8 +4,8 @@ %CPP cpp %FC ${MPIF90} -c - -%FFLAGS ${FCFLAGS} +%FCFLAGS ${FCFLAGS} +%FFFLAGS %FCFLAGS %LD ${MPIF90} %LDFLAGS %FPPFLAGS -P -traditional diff --git a/examples/nemo/scripts/omp_cpu_trans.py b/examples/nemo/scripts/omp_cpu_trans.py index 818f8788c7..9530c470ec 100755 --- a/examples/nemo/scripts/omp_cpu_trans.py +++ b/examples/nemo/scripts/omp_cpu_trans.py @@ -68,17 +68,19 @@ def trans(psyir): return for subroutine in psyir.walk(Routine): - print(f"Transforming subroutine: {subroutine.name}") + print(f"Adding OpenMP threading to subroutine: {subroutine.name}") if PROFILING_ENABLED: add_profiling(subroutine.children) enhance_tree_information(subroutine) - if subroutine.name in ("eos_rprof", ): - # TODO #1959: This subroutines make the ECMWF compilation fail + if subroutine.name in ("eos_rprof", "load_nml", "prt_ctl_write_sum", + "sbc_blk"): + # TODO #1959: 'eos_rprof' make the ECMWF compilation fail # because it moves a statement function outside of the # specification part. + # The rest are due to Subroutine wrongly parsed as Arrays? print("Skipping normalisation for ", subroutine.name) else: From 170de3dc1c51b41a71056615d62f05296cb73e14 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 28 Aug 2024 08:28:11 +0100 Subject: [PATCH 06/16] #2692 Fix NEMOv5 integration test issues --- .github/workflows/nemo_v5_tests.yml | 2 +- examples/nemo/scripts/omp_cpu_trans.py | 3 +- .../transformations/parallel_loop_trans.py | 15 ++++++- .../parallel_loop_trans_test.py | 39 +++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index b44247fa11..0d810aa427 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -84,7 +84,7 @@ jobs: # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS cd $NEMO_DIR cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - export FCFLAGS="-O2 -fcray-pointer -ffree-line-length-none -g" + export FCFLAGS="-fdefault-real-8 -O2 -fcray-pointer -ffree-line-length-none -g" # Clean up and compile ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC clean -y diff --git a/examples/nemo/scripts/omp_cpu_trans.py b/examples/nemo/scripts/omp_cpu_trans.py index 9530c470ec..102a2c185a 100755 --- a/examples/nemo/scripts/omp_cpu_trans.py +++ b/examples/nemo/scripts/omp_cpu_trans.py @@ -76,7 +76,8 @@ def trans(psyir): enhance_tree_information(subroutine) if subroutine.name in ("eos_rprof", "load_nml", "prt_ctl_write_sum", - "sbc_blk"): + "sbc_blk", "lbc_lnk_pt2pt_sp", + "lbc_lnk_neicoll_sp"): # TODO #1959: 'eos_rprof' make the ECMWF compilation fail # because it moves a statement function outside of the # specification part. diff --git a/src/psyclone/psyir/transformations/parallel_loop_trans.py b/src/psyclone/psyir/transformations/parallel_loop_trans.py index 33cb1472f0..94f9b32ea5 100644 --- a/src/psyclone/psyir/transformations/parallel_loop_trans.py +++ b/src/psyclone/psyir/transformations/parallel_loop_trans.py @@ -45,7 +45,7 @@ from psyclone.core import Signature from psyclone.domain.common.psylayer import PSyLoop from psyclone.psyir import nodes -from psyclone.psyir.nodes import Loop, Reference, Call +from psyclone.psyir.nodes import Loop, Reference, Call, Routine from psyclone.psyir.tools import DependencyTools, DTCode from psyclone.psyir.transformations.loop_trans import LoopTrans from psyclone.psyir.transformations.transformation_error import \ @@ -140,6 +140,19 @@ def validate(self, node, options=None): f"The 'collapse' argument must be an integer or a bool but" f" got an object of type {type(collapse)}") + routine = node.ancestor(Routine) + if routine is not None and routine.parent is not None: + try: + rsym = routine.parent.symbol_table.lookup(routine.name) + if rsym.is_pure or rsym.is_elemental: + raise TransformationError( + f"Loops inside a pure (or elemental) routine cannot be" + f" parallelised, but attempted to parallelise loop " + f"inside '{routine.name}'" + ) + except KeyError: + pass + # If it's sequential or we 'force' the transformation, the validations # below this point are skipped if sequential or force: diff --git a/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py b/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py index 083d59a441..2a2f69ab07 100644 --- a/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py +++ b/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py @@ -118,6 +118,45 @@ def test_paralooptrans_validate_pure_calls(fortran_reader): trans.validate(loop) +def test_paralooptrans_validate_loop_inside_pure(fortran_reader): + ''' Test that the validation checks that we don't attempt to parallelise + a loop inside a pure (or elemental) routine + ''' + psyir = fortran_reader.psyir_from_source(''' + module test + contains + elemental function my_func(a) + real, intent(in) :: a(10) + real, intent(out) :: my_func(10) + integer :: i + + do i = LBOUND(a), UBOUND(a) + my_func(i) = a(i) + 1 + end do + end function + + pure subroutine my_sub(a,b) + real, intent(in) :: a(10) + real, intent(out) :: b(10) + integer :: i + + do i = LBOUND(a), UBOUND(a) + b(i) = a(i) + 1 + end do + end subroutine + end module + ''') + + trans = ParaTrans() + for loop in psyir.walk(Loop): + # Check that we reject parallelisng inside a pure routine + with pytest.raises(TransformationError) as err: + trans.validate(loop, {"verbose": True}) + assert ("Loops inside a pure (or elemental) routine cannot be " + "parallelised, but attempted to parallelise loop inside '" + in str(err.value)) + + def test_paralooptrans_validate_ignore_dependencies_for(fortran_reader, fortran_writer): ''' From 85b1fbc1d4b8516e78a24b21c11c2b5d1a28057f Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 28 Aug 2024 11:32:20 +0100 Subject: [PATCH 07/16] #2692 Fix issues with NEMOv5 and LFRic integration tests --- .github/workflows/nemo_v5_tests.yml | 32 ++- .../KGOs/lfric_gunho_configuration_4its.nml | 2 +- .../nemo/scripts/KGOs/arch-linux_spack.fcm | 6 +- .../nemo/scripts/KGOs/namelist_cfg_orca1_like | 226 ++++++++++++++++++ .../nemo/scripts/KGOs/namelist_cfg_orca2_like | 226 ++++++++++++++++++ examples/nemo/scripts/omp_cpu_trans.py | 23 +- 6 files changed, 498 insertions(+), 17 deletions(-) create mode 100644 examples/nemo/scripts/KGOs/namelist_cfg_orca1_like create mode 100644 examples/nemo/scripts/KGOs/namelist_cfg_orca2_like diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index 0d810aa427..7eea66e430 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -91,10 +91,12 @@ jobs: ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca1 + # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00 + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg mpirun -np 4 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -119,10 +121,12 @@ jobs: ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca1 + # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg mpirun -np 4 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -147,10 +151,12 @@ jobs: ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca1 + # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg mpirun -np 4 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -175,10 +181,12 @@ jobs: ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca1 + # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg OMP_NUM_THREADS=4 mpirun -np 1 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -203,9 +211,11 @@ jobs: ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca1 + # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - mpirun -np 4 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + mpirun -np 1 ./nemo + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml index 856d4111d9..c47047c2d1 100644 --- a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml +++ b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml @@ -105,7 +105,7 @@ stretching_method='linear', / &io - use_xios_io = .true. + use_xios_io = .false. write_conservation_diag = .true. diag_active_files = 'lfric_diag' diag_always_on_sampling = .false. diff --git a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm index 1a4f06227c..db066ca63a 100644 --- a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm +++ b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm @@ -2,12 +2,12 @@ %NCDF_INC -I${NCDF_F_HOME}/include -I${NCDF_C_HOME}/include -I${HDF5_HOME}/include %NCDF_LIB -L${NCDF_F_HOME}/lib -lnetcdff -L${NCDF_C_HOME}/lib -lnetcdf -%CPP cpp +%CPP cpp -Dkey_nosignedzero %FC ${MPIF90} -c %FCFLAGS ${FCFLAGS} -%FFFLAGS %FCFLAGS +%FFLAGS %FCFLAGS %LD ${MPIF90} -%LDFLAGS +%LDFLAGS ${FCFLAGS} %FPPFLAGS -P -traditional %AR ar %ARFLAGS rs diff --git a/examples/nemo/scripts/KGOs/namelist_cfg_orca1_like b/examples/nemo/scripts/KGOs/namelist_cfg_orca1_like new file mode 100644 index 0000000000..dabc93656c --- /dev/null +++ b/examples/nemo/scripts/KGOs/namelist_cfg_orca1_like @@ -0,0 +1,226 @@ +!!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +!! NEMO/OPA BENCH Configuration namelist : overwrite some defaults values defined in SHARED/namelist_ref +!!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +! +!----------------------------------------------------------------------- +&namrun ! parameters of the run +!----------------------------------------------------------------------- + cn_exp = 'BENCH' ! experience name + nn_it000 = 1 ! first time step + nn_itend = 10 ! last time step + nn_stock = -1 ! frequency of creation of a restart file (modulo referenced to 1) + nn_write = -1 ! frequency of write in the output file (modulo referenced to nn_it000) +/ +!----------------------------------------------------------------------- +&namusr_def ! User defined : BENCH configuration: Flat bottom, beta-plane +!----------------------------------------------------------------------- + nn_isize = 360 ! number of point in i-direction of global(local) domain if >0 (<0) + nn_jsize = 331 ! number of point in j-direction of global(local) domain if >0 (<0) + nn_ksize = 75 ! total number of point in k-direction + ln_Iperio = .true. ! i-periodicity + ln_Jperio = .false. ! j-periodicity + ln_NFold = .true. ! North pole folding + cn_NFtype = 'F' ! Folding type: T or F +/ +!----------------------------------------------------------------------- +&nammpp ! Massively Parallel Processing +!----------------------------------------------------------------------- + ln_nnogather= .true. ! activate code to avoid mpi_allgather use at the northfold + jpni = 0 ! jpni number of processors following i (set automatically if < 1) + jpnj = 0 ! jpnj number of processors following j (set automatically if < 1) +/ +!----------------------------------------------------------------------- +&namctl ! Control prints (default: OFF) +!----------------------------------------------------------------------- + ln_timing = .true. ! timing by routine write out in timing.output file + sn_cfctl%l_runstat = .true. ! Output run.stat +/ +!----------------------------------------------------------------------- +&namdom ! time and space domain +!----------------------------------------------------------------------- + rn_Dt = 360. ! time step for the dynamics (and tracer if nn_acc=0) + rn_atfp = 0.05 ! asselin time filter parameter + ln_meshmask = .false. ! =T create a mesh file +/ + +!!====================================================================== +!! *** Surface Boundary Condition namelists *** !! +!! !! +!! namsbc surface boundary condition manager (default: NO selection) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namsbc ! Surface Boundary Condition (surface module) +!----------------------------------------------------------------------- + ln_usr = .true. ! user defined formulation (T => check usrdef_sbc) + nn_ice = 2 ! =0 no ice boundary condition + ! ! =1 use observed ice-cover ( => fill namsbc_iif ) + ! ! =2 or 3 for SI3 and CICE, respectively + ln_traqsr = .true. ! Light penetration in the ocean (T => fill namtra_qsr) +/ + +! +!!====================================================================== +!! *** Lateral boundary condition *** !! +!! !! +!! namlbc lateral momentum boundary condition (default: NO selection) +!! namagrif agrif nested grid (read by child model only) ("key_agrif") +!! nam_tide Tidal forcing (default: OFF) +!! nambdy Unstructured open boundaries (default: OFF) +!! nambdy_dta Unstructured open boundaries - external data (see nambdy) +!! nambdy_tide tidal forcing at open boundaries (default: OFF) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namlbc ! lateral momentum boundary condition (default: NO selection) +!----------------------------------------------------------------------- + rn_shlat = 0. ! free slip +/ + +!!====================================================================== +!! *** Top/Bottom boundary condition *** !! +!! !! +!! namdrg top/bottom drag coefficient (default: NO selection) +!! namdrg_top top friction (ln_drg_OFF =F & ln_isfcav=T) +!! namdrg_bot bottom friction (ln_drg_OFF =F) +!! nambbc bottom temperature boundary condition (default: OFF) +!! nambbl bottom boundary layer scheme (default: OFF) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namtra_qsr ! penetrative solar radiation (ln_traqsr =T) +!----------------------------------------------------------------------- + ! ! type of penetration (default: NO selection) + ln_qsr_rgb = .true. ! RGB light penetration (Red-Green-Blue) + nn_chldta = 0 ! RGB : Chl data (=1) or cst value (=0) +/ +!----------------------------------------------------------------------- +&namdrg ! top/bottom drag coefficient (default: NO selection) +!----------------------------------------------------------------------- + ln_non_lin = .true. ! non-linear drag: Cd = Cd0 |U| +/ +!----------------------------------------------------------------------- +&nambbc ! bottom temperature boundary condition (default: OFF) +!----------------------------------------------------------------------- + ln_trabbc = .true. ! Apply a geothermal heating at the ocean bottom + nn_geoflx = 1 ! geothermal heat flux: = 1 constant flux +/ +!----------------------------------------------------------------------- +&nambbl ! bottom boundary layer scheme (default: OFF) +!----------------------------------------------------------------------- + ln_trabbl = .true. ! Bottom Boundary Layer parameterisation flag +/ + +!!====================================================================== +!! Tracer (T & S) namelists !! +!! !! +!! nameos equation of state (default: NO selection) +!! namtra_adv advection scheme (default: NO selection) +!! namtra_ldf lateral diffusion scheme (default: NO selection) +!! namtra_mle mixed layer eddy param. (Fox-Kemper param.) (default: OFF) +!! namtra_eiv eddy induced velocity param. (default: OFF) +!! namtra_dmp T & S newtonian damping (default: OFF) +!!====================================================================== +! +!----------------------------------------------------------------------- +&nameos ! ocean Equation Of Seawater (default: NO selection) +!----------------------------------------------------------------------- + ln_teos10 = .true. ! = Use TEOS-10 +/ +!----------------------------------------------------------------------- +&namtra_adv ! advection scheme for tracer (default: NO selection) +!----------------------------------------------------------------------- + ln_traadv_fct = .true. ! FCT scheme + nn_fct_h = 2 ! =2/4, horizontal 2nd / 4th order + nn_fct_v = 2 ! =2/4, vertical 2nd / COMPACT 4th order +/ +!----------------------------------------------------------------------- +&namtra_ldf ! lateral diffusion scheme for tracers (default: NO selection) +!----------------------------------------------------------------------- + ln_traldf_lap = .true. ! laplacian operator + ln_traldf_iso = .true. ! iso-neutral (standard operator) + ! + nn_aht_ijk_t = 20 ! space/time variation of eddy coefficient: +/ +!----------------------------------------------------------------------- +&namtra_mle ! mixed layer eddy parametrisation (Fox-Kemper) (default: OFF) +!----------------------------------------------------------------------- + ln_mle = .true. ! (T) use the Mixed Layer Eddy (MLE) parameterisation +/ +!----------------------------------------------------------------------- +&namtra_eiv ! eddy induced velocity param. (default: OFF) +!----------------------------------------------------------------------- + ln_ldfeiv = .true. ! use eddy induced velocity parameterization + ! + nn_aei_ijk_t = 20 ! space/time variation of eddy coefficient: +/ + +!!====================================================================== +!! *** Dynamics namelists *** !! +!! !! +!! nam_vvl vertical coordinate options (default: z-star) +!! namdyn_adv formulation of the momentum advection (default: NO selection) +!! namdyn_vor advection scheme (default: NO selection) +!! namdyn_hpg hydrostatic pressure gradient (default: NO selection) +!! namdyn_spg surface pressure gradient (default: NO selection) +!! namdyn_ldf lateral diffusion scheme (default: NO selection) +!! namdta_dyn offline TOP: dynamics read in files (OFF_SRC only) +!!====================================================================== +! +!----------------------------------------------------------------------- +&nam_vvl ! vertical coordinate options (default: z-star) +!----------------------------------------------------------------------- + ln_vvl_zstar = .true. ! z-star vertical coordinate + ln_vvl_dbg = .false. ! debug prints (T/F) +/ +!----------------------------------------------------------------------- +&namdyn_adv ! formulation of the momentum advection (default: NO selection) +!----------------------------------------------------------------------- + ln_dynadv_vec = .true. ! vector form (T) or flux form (F) + nn_dynkeg = 1 ! scheme for grad(KE): =0 C2 ; =1 Hollingsworth correction +/ +!----------------------------------------------------------------------- +&namdyn_vor ! Vorticity / Coriolis scheme (default: NO selection) +!----------------------------------------------------------------------- + ln_dynvor_een = .true. ! energy & enstrophy scheme +/ +!----------------------------------------------------------------------- +&namdyn_hpg ! Hydrostatic pressure gradient option (default: NO selection) +!----------------------------------------------------------------------- + ln_hpg_sco = .true. ! s-coordinate (standard jacobian formulation) +/ +!----------------------------------------------------------------------- +&namdyn_spg ! surface pressure gradient (default: NO selection) +!----------------------------------------------------------------------- + ln_dynspg_ts = .true. ! split-explicit free surface + ln_bt_auto = .false. ! Number of sub-step defined from: + nn_e = 30 ! =F : the number of sub-step in rn_Dt seconds +/ +!----------------------------------------------------------------------- +&namdyn_ldf ! lateral diffusion on momentum (default: NO selection) +!----------------------------------------------------------------------- + ln_dynldf_lap = .true. ! laplacian operator + ln_dynldf_hor = .true. ! horizontal (geopotential) + ! ! Coefficient + nn_ahm_ijk_t = 30 ! space/time variation of eddy coef +/ + +!!====================================================================== +!! vertical physics namelists !! +!! !! +!! namzdf vertical physics manager (default: NO selection) +!! namzdf_ric richardson number vertical mixing (ln_zdfric=T) +!! namzdf_tke TKE vertical mixing (ln_zdftke=T) +!! namzdf_gls GLS vertical mixing (ln_zdfgls=T) +!! namzdf_osm OSM vertical diffusion (ln_zdfosm=T) +!! namzdf_iwm tidal mixing parameterization (ln_zdfiwm=T) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namzdf ! vertical physics (default: NO selection) +!----------------------------------------------------------------------- + ln_zdftke = .true. ! Turbulent Kinetic Energy closure (T => fill namzdf_tke) + ln_zdfevd = .true. ! enhanced vertical diffusion + ln_zdfddm = .true. ! double diffusive mixing + ln_zdfiwm = .true. ! internal wave-induced mixing (T => fill namzdf_iwm) +/ diff --git a/examples/nemo/scripts/KGOs/namelist_cfg_orca2_like b/examples/nemo/scripts/KGOs/namelist_cfg_orca2_like new file mode 100644 index 0000000000..9b700a464d --- /dev/null +++ b/examples/nemo/scripts/KGOs/namelist_cfg_orca2_like @@ -0,0 +1,226 @@ +!!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +!! NEMO/OPA BENCH Configuration namelist : overwrite some defaults values defined in SHARED/namelist_ref +!!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +! +!----------------------------------------------------------------------- +&namrun ! parameters of the run +!----------------------------------------------------------------------- + cn_exp = 'BENCH' ! experience name + nn_it000 = 1 ! first time step + nn_itend = 10 ! last time step + nn_stock = -1 ! frequency of creation of a restart file (modulo referenced to 1) + nn_write = -1 ! frequency of write in the output file (modulo referenced to nn_it000) +/ +!----------------------------------------------------------------------- +&namusr_def ! User defined : BENCH configuration: Flat bottom, beta-plane +!----------------------------------------------------------------------- + nn_isize = 180 ! number of point in i-direction of global(local) domain if >0 (<0) + nn_jsize = 166 ! number of point in j-direction of global(local) domain if >0 (<0) + nn_ksize = 40 ! total number of point in k-direction + ln_Iperio = .true. ! i-periodicity + ln_Jperio = .false. ! j-periodicity + ln_NFold = .true. ! North pole folding + cn_NFtype = 'F' ! Folding type: T or F +/ +!----------------------------------------------------------------------- +&nammpp ! Massively Parallel Processing +!----------------------------------------------------------------------- + ln_nnogather= .true. ! activate code to avoid mpi_allgather use at the northfold + jpni = 0 ! jpni number of processors following i (set automatically if < 1) + jpnj = 0 ! jpnj number of processors following j (set automatically if < 1) +/ +!----------------------------------------------------------------------- +&namctl ! Control prints (default: OFF) +!----------------------------------------------------------------------- + ln_timing = .true. ! timing by routine write out in timing.output file + sn_cfctl%l_runstat = .true. ! Output run.stat +/ +!----------------------------------------------------------------------- +&namdom ! time and space domain +!----------------------------------------------------------------------- + rn_Dt = 360. ! time step for the dynamics (and tracer if nn_acc=0) + rn_atfp = 0.05 ! asselin time filter parameter + ln_meshmask = .false. ! =T create a mesh file +/ + +!!====================================================================== +!! *** Surface Boundary Condition namelists *** !! +!! !! +!! namsbc surface boundary condition manager (default: NO selection) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namsbc ! Surface Boundary Condition (surface module) +!----------------------------------------------------------------------- + ln_usr = .true. ! user defined formulation (T => check usrdef_sbc) + nn_ice = 2 ! =0 no ice boundary condition + ! ! =1 use observed ice-cover ( => fill namsbc_iif ) + ! ! =2 or 3 for SI3 and CICE, respectively + ln_traqsr = .true. ! Light penetration in the ocean (T => fill namtra_qsr) +/ + +! +!!====================================================================== +!! *** Lateral boundary condition *** !! +!! !! +!! namlbc lateral momentum boundary condition (default: NO selection) +!! namagrif agrif nested grid (read by child model only) ("key_agrif") +!! nam_tide Tidal forcing (default: OFF) +!! nambdy Unstructured open boundaries (default: OFF) +!! nambdy_dta Unstructured open boundaries - external data (see nambdy) +!! nambdy_tide tidal forcing at open boundaries (default: OFF) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namlbc ! lateral momentum boundary condition (default: NO selection) +!----------------------------------------------------------------------- + rn_shlat = 0. ! free slip +/ + +!!====================================================================== +!! *** Top/Bottom boundary condition *** !! +!! !! +!! namdrg top/bottom drag coefficient (default: NO selection) +!! namdrg_top top friction (ln_drg_OFF =F & ln_isfcav=T) +!! namdrg_bot bottom friction (ln_drg_OFF =F) +!! nambbc bottom temperature boundary condition (default: OFF) +!! nambbl bottom boundary layer scheme (default: OFF) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namtra_qsr ! penetrative solar radiation (ln_traqsr =T) +!----------------------------------------------------------------------- + ! ! type of penetration (default: NO selection) + ln_qsr_rgb = .true. ! RGB light penetration (Red-Green-Blue) + nn_chldta = 0 ! RGB : Chl data (=1) or cst value (=0) +/ +!----------------------------------------------------------------------- +&namdrg ! top/bottom drag coefficient (default: NO selection) +!----------------------------------------------------------------------- + ln_non_lin = .true. ! non-linear drag: Cd = Cd0 |U| +/ +!----------------------------------------------------------------------- +&nambbc ! bottom temperature boundary condition (default: OFF) +!----------------------------------------------------------------------- + ln_trabbc = .true. ! Apply a geothermal heating at the ocean bottom + nn_geoflx = 1 ! geothermal heat flux: = 1 constant flux +/ +!----------------------------------------------------------------------- +&nambbl ! bottom boundary layer scheme (default: OFF) +!----------------------------------------------------------------------- + ln_trabbl = .true. ! Bottom Boundary Layer parameterisation flag +/ + +!!====================================================================== +!! Tracer (T & S) namelists !! +!! !! +!! nameos equation of state (default: NO selection) +!! namtra_adv advection scheme (default: NO selection) +!! namtra_ldf lateral diffusion scheme (default: NO selection) +!! namtra_mle mixed layer eddy param. (Fox-Kemper param.) (default: OFF) +!! namtra_eiv eddy induced velocity param. (default: OFF) +!! namtra_dmp T & S newtonian damping (default: OFF) +!!====================================================================== +! +!----------------------------------------------------------------------- +&nameos ! ocean Equation Of Seawater (default: NO selection) +!----------------------------------------------------------------------- + ln_teos10 = .true. ! = Use TEOS-10 +/ +!----------------------------------------------------------------------- +&namtra_adv ! advection scheme for tracer (default: NO selection) +!----------------------------------------------------------------------- + ln_traadv_fct = .true. ! FCT scheme + nn_fct_h = 2 ! =2/4, horizontal 2nd / 4th order + nn_fct_v = 2 ! =2/4, vertical 2nd / COMPACT 4th order +/ +!----------------------------------------------------------------------- +&namtra_ldf ! lateral diffusion scheme for tracers (default: NO selection) +!----------------------------------------------------------------------- + ln_traldf_lap = .true. ! laplacian operator + ln_traldf_iso = .true. ! iso-neutral (standard operator) + ! + nn_aht_ijk_t = 20 ! space/time variation of eddy coefficient: +/ +!----------------------------------------------------------------------- +&namtra_mle ! mixed layer eddy parametrisation (Fox-Kemper) (default: OFF) +!----------------------------------------------------------------------- + ln_mle = .true. ! (T) use the Mixed Layer Eddy (MLE) parameterisation +/ +!----------------------------------------------------------------------- +&namtra_eiv ! eddy induced velocity param. (default: OFF) +!----------------------------------------------------------------------- + ln_ldfeiv = .true. ! use eddy induced velocity parameterization + ! + nn_aei_ijk_t = 20 ! space/time variation of eddy coefficient: +/ + +!!====================================================================== +!! *** Dynamics namelists *** !! +!! !! +!! nam_vvl vertical coordinate options (default: z-star) +!! namdyn_adv formulation of the momentum advection (default: NO selection) +!! namdyn_vor advection scheme (default: NO selection) +!! namdyn_hpg hydrostatic pressure gradient (default: NO selection) +!! namdyn_spg surface pressure gradient (default: NO selection) +!! namdyn_ldf lateral diffusion scheme (default: NO selection) +!! namdta_dyn offline TOP: dynamics read in files (OFF_SRC only) +!!====================================================================== +! +!----------------------------------------------------------------------- +&nam_vvl ! vertical coordinate options (default: z-star) +!----------------------------------------------------------------------- + ln_vvl_zstar = .true. ! z-star vertical coordinate + ln_vvl_dbg = .false. ! debug prints (T/F) +/ +!----------------------------------------------------------------------- +&namdyn_adv ! formulation of the momentum advection (default: NO selection) +!----------------------------------------------------------------------- + ln_dynadv_vec = .true. ! vector form (T) or flux form (F) + nn_dynkeg = 1 ! scheme for grad(KE): =0 C2 ; =1 Hollingsworth correction +/ +!----------------------------------------------------------------------- +&namdyn_vor ! Vorticity / Coriolis scheme (default: NO selection) +!----------------------------------------------------------------------- + ln_dynvor_een = .true. ! energy & enstrophy scheme +/ +!----------------------------------------------------------------------- +&namdyn_hpg ! Hydrostatic pressure gradient option (default: NO selection) +!----------------------------------------------------------------------- + ln_hpg_sco = .true. ! s-coordinate (standard jacobian formulation) +/ +!----------------------------------------------------------------------- +&namdyn_spg ! surface pressure gradient (default: NO selection) +!----------------------------------------------------------------------- + ln_dynspg_ts = .true. ! split-explicit free surface + ln_bt_auto = .false. ! Number of sub-step defined from: + nn_e = 30 ! =F : the number of sub-step in rn_Dt seconds +/ +!----------------------------------------------------------------------- +&namdyn_ldf ! lateral diffusion on momentum (default: NO selection) +!----------------------------------------------------------------------- + ln_dynldf_lap = .true. ! laplacian operator + ln_dynldf_hor = .true. ! horizontal (geopotential) + ! ! Coefficient + nn_ahm_ijk_t = 30 ! space/time variation of eddy coef +/ + +!!====================================================================== +!! vertical physics namelists !! +!! !! +!! namzdf vertical physics manager (default: NO selection) +!! namzdf_ric richardson number vertical mixing (ln_zdfric=T) +!! namzdf_tke TKE vertical mixing (ln_zdftke=T) +!! namzdf_gls GLS vertical mixing (ln_zdfgls=T) +!! namzdf_osm OSM vertical diffusion (ln_zdfosm=T) +!! namzdf_iwm tidal mixing parameterization (ln_zdfiwm=T) +!!====================================================================== +! +!----------------------------------------------------------------------- +&namzdf ! vertical physics (default: NO selection) +!----------------------------------------------------------------------- + ln_zdftke = .true. ! Turbulent Kinetic Energy closure (T => fill namzdf_tke) + ln_zdfevd = .true. ! enhanced vertical diffusion + ln_zdfddm = .true. ! double diffusive mixing + ln_zdfiwm = .true. ! internal wave-induced mixing (T => fill namzdf_iwm) +/ diff --git a/examples/nemo/scripts/omp_cpu_trans.py b/examples/nemo/scripts/omp_cpu_trans.py index 102a2c185a..b59069c68a 100755 --- a/examples/nemo/scripts/omp_cpu_trans.py +++ b/examples/nemo/scripts/omp_cpu_trans.py @@ -46,7 +46,25 @@ PROFILING_ENABLED = False # List of all files that psyclone will skip processing -FILES_TO_SKIP = NOT_PERFORMANT + NOT_WORKING +FILES_TO_SKIP = NOT_PERFORMANT + NOT_WORKING + [ + "lbclnk.f90", # TODO #2685: effective shape bug + "asminc.f90", + "trosk.f90", + "vremap.f90", + # "prtctl.f90", + # "fldread.f90", + # "dynspg_ts.f90", + # "lib_mpp.f90", + # "diaobs.f90", + # "traqsr.f90", + # "icethd_pnd.f90", + # "lbclnk.f90", + # "iom_nf90.f90", + # "ldfslp.f90", + # "icethd.f90", + # "icestp.f90", + # "sbcblk.f90" +] def trans(psyir): @@ -77,7 +95,8 @@ def trans(psyir): if subroutine.name in ("eos_rprof", "load_nml", "prt_ctl_write_sum", "sbc_blk", "lbc_lnk_pt2pt_sp", - "lbc_lnk_neicoll_sp"): + "lbc_lnk_neicoll_sp", "lbc_lnk_iprobe_sp", + "lbc_lnk_waitany_sp"): # TODO #1959: 'eos_rprof' make the ECMWF compilation fail # because it moves a statement function outside of the # specification part. From 3813045ccf1778bd307e70db3eb386b21de52233 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 28 Aug 2024 15:36:48 +0100 Subject: [PATCH 08/16] #2692 Revert some integration test changes that didn't work --- .github/workflows/lfric_test.yml | 8 +- .github/workflows/nemo_tests.yml | 25 +++++- .github/workflows/nemo_v5_tests.yml | 104 ++++++++++++------------- examples/nemo/scripts/omp_cpu_trans.py | 15 +--- 4 files changed, 81 insertions(+), 71 deletions(-) diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml index cf353dc121..49cff9c194 100644 --- a/.github/workflows/lfric_test.yml +++ b/.github/workflows/lfric_test.yml @@ -79,8 +79,8 @@ jobs: - name: LFRic passthrough (with DistributedMemory) run: | # Set up environment - source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack load lfric-build-environment%gcc@14 + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack load lfric-buildenv%gcc source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg @@ -119,8 +119,8 @@ jobs: - name: LFRic with all transformations run: | # Set up environment - source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack load lfric-build-environment%gcc@14 + source /apps/spack/spack-upstream/share/spack/setup-env.sh + spack load lfric-buildenv%gcc source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml index bbcd2bf468..39455828c0 100644 --- a/.github/workflows/nemo_tests.yml +++ b/.github/workflows/nemo_tests.yml @@ -36,7 +36,7 @@ # This workflow will use a self-hosted runner to perform the more expensive # integrations tests that are not run on GHA systems. -name: NEMOv4 Integration Tests +name: NEMO Integration Tests on: push @@ -82,6 +82,29 @@ jobs: cd lib/profiling/nvidia/ F90=nvfortran make + # PSyclone passthrough for 5.0-beta of NEMO. + - name: NEMO 5.0 beta passthrough without optimisation + run: | + . .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + # PSYCLONE_HOME has `/bin` appended to it by the build system. + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 + cd $NEMO_DIR + module load nvidia-hpcsdk/${NVFORTRAN_VERSION} + module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION} + module load perl/${PERL_VERSION} + # We compile at -O1 to permit comparison of the results. (N.B. this test + # passes at -O3 with the Intel ifx compiler.) + ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH clean -y + ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ + del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH/EXP00 + mpirun -np 4 ./nemo + diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME + # PSyclone passthrough for MetOffice NEMO - name: NEMO MetOffice Passthrough run: | diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index 7eea66e430..f085e76a03 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -100,35 +100,35 @@ jobs: export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME - - name: NEMO 5.0 nvidia passthrough - run: | - # Set up environment - source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack unload && spack load nemo-build-environment%nvhpc@24.5 - source .runner_venv/bin/activate - export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts - export PSYCLONE_HOME=${PWD}/.runner_venv - export NEMO_DIR=${HOME}/NEMOv5 + # - name: NEMO 5.0 nvidia passthrough + # run: | + # # Set up environment + # source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + # spack unload && spack load nemo-build-environment%nvhpc@24.5 + # source .runner_venv/bin/activate + # export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + # export PSYCLONE_HOME=${PWD}/.runner_venv + # export NEMO_DIR=${HOME}/NEMOv5 - # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # We compile at -O1 to permit comparison of the results. - cd $NEMO_DIR - cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" + # # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # # We compile at -O1 to permit comparison of the results. + # cd $NEMO_DIR + # cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + # export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" - # Clean up and compile - ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y - ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ - del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + # # Clean up and compile + # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y + # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ + # del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca2 - cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - mpirun -np 4 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - cat run.stat - export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - echo "Time-stepping duration = " $VAR_TIME + # # Run Orca2 + # cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + # cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + # mpirun -np 4 ./nemo + # # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + # cat run.stat + # export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + # echo "Time-stepping duration = " $VAR_TIME - name: NEMO 5.0 Intel passthrough run: | @@ -190,32 +190,32 @@ jobs: export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME - - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory) - run: | - # Set up environment - source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack unload && spack load nemo-build-environment%nvhpc@24.5 - source .runner_venv/bin/activate - export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts - export PSYCLONE_HOME=${PWD}/.runner_venv - export NEMO_DIR=${HOME}/NEMOv5 + # - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory) + # run: | + # # Set up environment + # source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + # spack unload && spack load nemo-build-environment%nvhpc@24.5 + # source .runner_venv/bin/activate + # export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + # export PSYCLONE_HOME=${PWD}/.runner_venv + # export NEMO_DIR=${HOME}/NEMOv5 - # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # We compile at -O1 to permit comparison of the results. - cd $NEMO_DIR - cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed" + # # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # # We compile at -O1 to permit comparison of the results. + # cd $NEMO_DIR + # cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + # export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed" - # Clean up and compile - ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y - ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ - del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + # # Clean up and compile + # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y + # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ + # del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca2 - cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - mpirun -np 1 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - cat run.stat - export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - echo "Time-stepping duration = " $VAR_TIME + # # Run Orca2 + # cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + # cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + # mpirun -np 1 ./nemo + # # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + # cat run.stat + # export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + # echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/nemo/scripts/omp_cpu_trans.py b/examples/nemo/scripts/omp_cpu_trans.py index b59069c68a..904b44d110 100755 --- a/examples/nemo/scripts/omp_cpu_trans.py +++ b/examples/nemo/scripts/omp_cpu_trans.py @@ -47,23 +47,10 @@ # List of all files that psyclone will skip processing FILES_TO_SKIP = NOT_PERFORMANT + NOT_WORKING + [ - "lbclnk.f90", # TODO #2685: effective shape bug + "lbclnk.f90", # TODO #2685: effective shape bug "asminc.f90", "trosk.f90", "vremap.f90", - # "prtctl.f90", - # "fldread.f90", - # "dynspg_ts.f90", - # "lib_mpp.f90", - # "diaobs.f90", - # "traqsr.f90", - # "icethd_pnd.f90", - # "lbclnk.f90", - # "iom_nf90.f90", - # "ldfslp.f90", - # "icethd.f90", - # "icestp.f90", - # "sbcblk.f90" ] From 3f6dd777f77339205a61300778b9eaa9ec4c6bf9 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 28 Aug 2024 16:36:35 +0100 Subject: [PATCH 09/16] #2692 Reintroduce LFRic and NEMOv5 nvhpc tests --- .github/workflows/lfric_test.yml | 14 +-- .github/workflows/nemo_tests.yml | 2 +- .github/workflows/nemo_v5_tests.yml | 114 +++++++++--------- .../KGOs/lfric_gunho_configuration_4its.nml | 21 ++-- 4 files changed, 74 insertions(+), 77 deletions(-) diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml index 49cff9c194..8f769d87b4 100644 --- a/.github/workflows/lfric_test.yml +++ b/.github/workflows/lfric_test.yml @@ -46,7 +46,7 @@ jobs: if: ${{ github.repository == 'stfc/PSyclone-mirror' }} runs-on: self-hosted env: - LFRIC_APPS_REV: 1192 + LFRIC_APPS_REV: 3269 PYTHON_VERSION: 3.12.5 steps: @@ -79,12 +79,12 @@ jobs: - name: LFRic passthrough (with DistributedMemory) run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh - spack load lfric-buildenv%gcc + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + spack load lfric-build-environment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg - export LFRIC_DIR=${HOME}/LFRic/lfric_apps + export LFRIC_DIR=${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV} cd ${LFRIC_DIR} # Clean previous version and compile again rm -rf applications/gungho_model/working @@ -119,12 +119,12 @@ jobs: - name: LFRic with all transformations run: | # Set up environment - source /apps/spack/spack-upstream/share/spack/setup-env.sh - spack load lfric-buildenv%gcc + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + spack load lfric-build-environment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg - export LFRIC_DIR=${HOME}/LFRic/lfric_apps + export LFRIC_DIR=${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV} export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test cd ${LFRIC_DIR} # Psyclone scripts must now be under 'optimisation' and be called 'global.py' diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml index 39455828c0..ba55787adc 100644 --- a/.github/workflows/nemo_tests.yml +++ b/.github/workflows/nemo_tests.yml @@ -36,7 +36,7 @@ # This workflow will use a self-hosted runner to perform the more expensive # integrations tests that are not run on GHA systems. -name: NEMO Integration Tests +name: NEMOv4 Integration Tests on: push diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index f085e76a03..55f7c77efa 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -100,48 +100,47 @@ jobs: export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME - # - name: NEMO 5.0 nvidia passthrough - # run: | - # # Set up environment - # source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - # spack unload && spack load nemo-build-environment%nvhpc@24.5 - # source .runner_venv/bin/activate - # export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts - # export PSYCLONE_HOME=${PWD}/.runner_venv - # export NEMO_DIR=${HOME}/NEMOv5 + - name: NEMO 5.0 nvidia passthrough + run: | + # Set up environment + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%nvhpc@24.5 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 - # # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # # We compile at -O1 to permit comparison of the results. - # cd $NEMO_DIR - # cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - # export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # We compile at -O1 to permit comparison of the results. + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" - # # Clean up and compile - # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y - # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ - # del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ + del_key "key_xios key_iomput key_top" add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 - # # Run Orca2 - # cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - # cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - # mpirun -np 4 ./nemo - # # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - # cat run.stat - # export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - # echo "Time-stepping duration = " $VAR_TIME + # Run Orca2 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + mpirun -np 4 ./nemo + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME - name: NEMO 5.0 Intel passthrough run: | # Set up environment source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack unload && spack load nemo-build-environment%oneapi@2024.04 + spack unload && spack load nemo-build-environment%oneapi source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts export PSYCLONE_HOME=${PWD}/.runner_venv export NEMO_DIR=${HOME}/NEMOv5 # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # We compile at -O1 to permit comparison of the results. cd $NEMO_DIR cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g" @@ -164,21 +163,20 @@ jobs: run: | # Set up environment source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack unload && spack load nemo-build-environment%oneapi@2024.04 + spack unload && spack load nemo-build-environment%oneapi source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts export PSYCLONE_HOME=${PWD}/.runner_venv export NEMO_DIR=${HOME}/NEMOv5 # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # We compile at -O1 to permit comparison of the results. cd $NEMO_DIR cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g -qopenmp" # Clean up and compile - ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y - ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ + ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_ONEAPI clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_ONEAPI -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 # Run Orca2 @@ -190,32 +188,32 @@ jobs: export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME - # - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory) - # run: | - # # Set up environment - # source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - # spack unload && spack load nemo-build-environment%nvhpc@24.5 - # source .runner_venv/bin/activate - # export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts - # export PSYCLONE_HOME=${PWD}/.runner_venv - # export NEMO_DIR=${HOME}/NEMOv5 + - name: NEMO 5.0 nvidia OpenMP for GPUs (managed memory) + run: | + # Set up environment + source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh + spack unload && spack load nemo-build-environment%nvhpc@24.5 + source .runner_venv/bin/activate + export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts + export PSYCLONE_HOME=${PWD}/.runner_venv + export NEMO_DIR=${HOME}/NEMOv5 - # # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # # We compile at -O1 to permit comparison of the results. - # cd $NEMO_DIR - # cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - # export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed" + # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS + # We compile at -O1 to permit comparison of the results. + cd $NEMO_DIR + cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm + export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed" - # # Clean up and compile - # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y - # ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ - # del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + # Clean up and compile + ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ + del_key "key_xios key_iomput key_top" add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 - # # Run Orca2 - # cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - # cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - # mpirun -np 1 ./nemo - # # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - # cat run.stat - # export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - # echo "Time-stepping duration = " $VAR_TIME + # Run Orca2 + cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + mpirun -np 1 ./nemo + # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + cat run.stat + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml index c47047c2d1..21aaa236ca 100644 --- a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml +++ b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its.nml @@ -92,6 +92,7 @@ stretching_method='linear', ancil_option = 'none' lbc_option = 'none' coarse_aerosol_ancil=.false. +zero_w2v_wind=.false., / &files @@ -196,14 +197,6 @@ stretching_method='linear', jacobi_relaxation = 0.5 / -&subgrid - dep_pt_stencil_extent = 3 - inner_order = 2 - outer_order = 2 - vertical_order = 2 - calculate_detj = 'upwind' -/ - ×tepping method = 'semi_implicit' dt = 3600.0 @@ -214,9 +207,7 @@ stretching_method='linear', outer_iterations = 2 inner_iterations = 2 runge_kutta_method = 'forward_euler' - spinup_period = 1 spinup_alpha = .false. - spinup_winds = .false. / &departure_points @@ -231,8 +222,11 @@ stretching_method='linear', &transport adjust_theta=.false. adjust_theta_above=0.0 + adjust_vhv_wind=.false. broken_w2_projection = .false. + calculate_detj = 'upwind' cap_density_predictor = 0.01 + dep_pt_stencil_extent = 3 profile_size = 3 field_names = 'density', 'potential_temperature', 'wind' equation_form = 1, 2, 2 @@ -248,7 +242,11 @@ stretching_method='linear', max_vert_cfl_calc='dep_point' min_val_method='iterative' min_value=0.0,0.0,-99999999.0,0.0 - consistent_ffsl_splitting=3*1 + ffsl_inner_order = 2 + ffsl_outer_order = 2 + ffsl_splitting=3*1 + ffsl_unity_3d=.false. + ffsl_vertical_order=3*2 cfl_mol_1d_stab = 1.0 cfl_mol_2d_stab = 1.0 cfl_mol_3d_stab = 1.0 @@ -261,6 +259,7 @@ stretching_method='linear', operators = 'fv' vertical_sl_order='cubic' slice_order='cubic' + si_outer_transport='none' reversible = .true., .true., .false. runge_kutta_method = 'ssp5' min_val_abs_tol=-1.0e-12 From 8b6e15a2d5b65d490240f8cd900d84b3e83b9b81 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 29 Aug 2024 09:24:16 +0100 Subject: [PATCH 10/16] #2692 Update LFRic CI to validate the checksums --- .github/workflows/lfric_test.yml | 10 +- ...ric_gunho_configuration_4its_checksums.txt | 3 + .../lfric_gunho_configuration_4its_output.txt | 668 ------------------ examples/lfric/scripts/compare_ouput.py | 15 +- 4 files changed, 13 insertions(+), 683 deletions(-) create mode 100644 examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_checksums.txt delete mode 100644 examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_output.txt diff --git a/.github/workflows/lfric_test.yml b/.github/workflows/lfric_test.yml index 8f769d87b4..2065cf7308 100644 --- a/.github/workflows/lfric_test.yml +++ b/.github/workflows/lfric_test.yml @@ -92,8 +92,8 @@ jobs: # Run cd applications/gungho_model/example cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml - mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt - python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_output.txt output.txt + mpirun -n 6 ../bin/gungho_model configuration.nml |& tee output.txt + python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_checksums.txt gungho_model-checksums.txt cat timer.txt export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5) export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5) @@ -105,7 +105,7 @@ jobs: --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'", github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'", ci_test: "LFRic Passthrough with DM", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos", - compiler:"spack-gfortran-11", date: new Date(), elapsed_time: '"$VAR_TIME"', + compiler:"spack-gfortran-14", date: new Date(), elapsed_time: '"$VAR_TIME"', num_of_halo_exchanges: '"$VAR_HALOS"'})' - name: Upload LFRic passthrough results @@ -138,7 +138,7 @@ jobs: cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its.nml configuration.nml export OMP_NUM_THREADS=6 mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt - python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_output.txt output.txt + python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gunho_configuration_4its_checksums.txt gungho_model-checksums.txt cat timer.txt export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5) export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5) @@ -150,7 +150,7 @@ jobs: --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'", github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'", ci_test: "LFRic all transformations", lfric_version: '"$LFRIC_APPS_REV"', omp_threads: '"$OMP_NUM_THREADS"', - system: "GlaDos", compiler:"spack-gfortran-11", date: new Date(), elapsed_time: '"$VAR_TIME"', + system: "GlaDos", compiler:"spack-gfortran-14", date: new Date(), elapsed_time: '"$VAR_TIME"', num_of_halo_exchanges: '"$VAR_HALOS"'})' - name: Upload LFRic optimised results diff --git a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_checksums.txt b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_checksums.txt new file mode 100644 index 0000000000..3e1916d8d9 --- /dev/null +++ b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_checksums.txt @@ -0,0 +1,3 @@ +Inner product checksum rho = 40D0CE5535FE8C13 +Inner product checksum theta = 41FCE95F543C2FD6 +Inner product checksum u = 45022E02BDAB8BD1 diff --git a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_output.txt b/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_output.txt deleted file mode 100644 index 35eb1d68bb..0000000000 --- a/examples/lfric/scripts/KGOs/lfric_gunho_configuration_4its_output.txt +++ /dev/null @@ -1,668 +0,0 @@ -20240514181114.070+0100:INFO : Runtime message logging severity set to log level: INFO -20240514181114.070+0100:INFO : Application built with 64-bit real numbers -20240514181114.070+0100:INFO : Setting up partition mesh(es) -20240514181114.070+0100:INFO : Using serial cubed sphere partitioner -20240514181114.070+0100:INFO : Reading global mesh: "C16" -20240514181114.073+0100:INFO : Reading global mesh: "C8" -20240514181114.075+0100:INFO : Reading global mesh: "C4" -20240514181114.077+0100:INFO : Reading global mesh: "C2" -20240514181114.109+0100:INFO : ... "C16"(id:1) based on mesh "C16" -20240514181114.111+0100:INFO : ... "C8"(id:2) based on mesh "C8" -20240514181114.111+0100:INFO : ... "C4"(id:3) based on mesh "C4" -20240514181114.111+0100:INFO : ... "C2"(id:4) based on mesh "C2" -20240514181114.111+0100:INFO : Adding intergrid map "C16"<-->"C8" -20240514181114.111+0100:INFO : Adding intergrid map "C8"<-->"C16" -20240514181114.112+0100:INFO : Adding intergrid map "C8"<-->"C4" -20240514181114.112+0100:INFO : Adding intergrid map "C4"<-->"C8" -20240514181114.112+0100:INFO : Adding intergrid map "C4"<-->"C2" -20240514181114.112+0100:INFO : Adding intergrid map "C2"<-->"C4" -20240514181114.113+0100:INFO : ... "C16_2d"(id:5) based on mesh "C16" -20240514181114.113+0100:INFO : ... "C8_2d"(id:6) based on mesh "C8" -20240514181114.113+0100:INFO : ... "C4_2d"(id:7) based on mesh "C4" -20240514181114.113+0100:INFO : ... "C2_2d"(id:8) based on mesh "C2" -20240514181114.113+0100:INFO : Adding intergrid map "C16_2d"<-->"C8_2d" -20240514181114.113+0100:INFO : Adding intergrid map "C8_2d"<-->"C16_2d" -20240514181114.113+0100:INFO : Adding intergrid map "C8_2d"<-->"C4_2d" -20240514181114.113+0100:INFO : Adding intergrid map "C4_2d"<-->"C8_2d" -20240514181114.113+0100:INFO : Adding intergrid map "C4_2d"<-->"C2_2d" -20240514181114.113+0100:INFO : Adding intergrid map "C2_2d"<-->"C4_2d" -20240514181114.113+0100:INFO : FEM specifics: creating function spaces... -20240514181114.115+0100:INFO : FEM specifics: Computing Wchi coordinate fields -20240514181114.362+0100:INFO : FEM specifics: Computing Wchi coordinate fields -20240514181114.422+0100:INFO : FEM specifics: Computing Wchi coordinate fields -20240514181114.436+0100:INFO : FEM specifics: Computing Wchi coordinate fields -20240514181114.440+0100:INFO : FEM specifics created -20240514181114.440+0100:INFO : FEM specifics: creating function space chains... -20240514181114.440+0100:INFO : Initialising MultiGrid 4-level function space chain. -20240514181114.446+0100:INFO : Function space chains created -20240514181114.446+0100:INFO : Initialising I/O context -20240514181114.501+0100:INFO : XIOS domain ['node'] is already registered. -20240514181114.521+0100:INFO : XIOS domain ['edge'] is already registered. -20240514181114.523+0100:INFO : XIOS domain ['face'] is already registered. -20240514181114.706+0100:INFO : Initialising XIOS context: gungho_atm -20240514181114.706+0100:INFO : Initialise surface altitude: Create surface altitude. -20240514181114.706+0100:INFO : Initialise surface altitude: No initialisation option set for orography. -20240514181114.706+0100:INFO : assign_orography_field: Flat surface requested. -20240514181114.706+0100:INFO : assign_orography_field: Flat surface requested. -20240514181114.706+0100:INFO : assign_orography_field: Flat surface requested. -20240514181114.707+0100:INFO : assign_orography_field: Flat surface requested. -20240514181114.707+0100:INFO : Gungho: creating runtime_constants -20240514181114.718+0100:INFO : Gungho: creating geometric_constants -20240514181114.878+0100:INFO : Gungho: created geometric_constants -20240514181114.878+0100:INFO : Gungho: creating fem_constants -20240514181114.878+0100:INFO : Creating FEM operators on 8 meshes -20240514181114.883+0100:INFO : Creating DeRham operators on mesh 1 -20240514181128.188+0100:INFO : Gungho: created fem_constants -20240514181128.188+0100:INFO : Gungho: creating physical_op_constants -20240514181134.816+0100:INFO : Gungho: created physical_op_constants -20240514181134.816+0100:INFO : Gungho: creating intermesh_constants -20240514181134.816+0100:INFO : Gungho: created intermesh_constants -20240514181134.816+0100:INFO : Gungho: created runtime_constants -20240514181134.816+0100:INFO : GungHo: Creating prognostics... -20240514181134.816+0100:INFO : Gungho: Initialising prognostic fields -20240514181134.816+0100:INFO : Initialise u with projection -20240514181136.458+0100:INFO : Mass matrix solve for W2 space -20240514181136.458+0100:INFO : Constructing mass matrix operator... -20240514181136.458+0100:INFO : done -20240514181136.459+0100:INFO : Constructing diagonal preconditioner... -20240514181136.459+0100:INFO : done -20240514181136.460+0100:INFO : Gungho: mass matrix solve: -20240514181136.528+0100:INFO : chebyshev[ 13], redidual = 0.13720448E-05 -20240514181138.422+0100:INFO : Gungho: Initialised prognostic fields -20240514181138.422+0100:INFO : Gungho: writing diagnostic output -20240514181151.341+0100:INFO : Mass matrix solve for W1 space -20240514181151.341+0100:INFO : Constructing mass matrix operator... -20240514181151.341+0100:INFO : done -20240514181151.341+0100:INFO : Constructing diagonal preconditioner... -20240514181151.341+0100:INFO : done -20240514181151.341+0100:INFO : Gungho: mass matrix solve: -20240514181151.478+0100:INFO : chebyshev[ 13], redidual = 0.60413254E-03 -20240514181151.488+0100:INFO : Computing Galerkin projection... -20240514181151.488+0100:INFO : vector field ... -20240514181152.304+0100:INFO : Mass matrix solve for W1 space -20240514181152.304+0100:INFO : Constructing mass matrix operator... -20240514181152.304+0100:INFO : done -20240514181152.304+0100:INFO : Constructing diagonal preconditioner... -20240514181152.305+0100:INFO : done -20240514181152.305+0100:INFO : Gungho: mass matrix solve: -20240514181152.442+0100:INFO : chebyshev[ 13], redidual = 0.64813382E-03 -20240514181153.484+0100:INFO : L2 of divergence = 0.27818194E+03 -20240514181153.487+0100:INFO : Gungho: creating si_operators -20240514181153.488+0100:INFO : si_ops[1]:creating ops -20240514181153.488+0100:INFO : si_ops[2]:creating ops -20240514181153.488+0100:INFO : si_ops[3]:creating ops -20240514181153.488+0100:INFO : si_ops[4]:creating ops -20240514181153.488+0100:INFO : Transport scheme built with 64-bit real numbers -20240514181153.488+0100:INFO : Setting up density transport metadata -20240514181153.488+0100:INFO : Setting up potential_temperature transport metadata -20240514181153.488+0100:INFO : Setting up wind transport metadata -20240514181153.490+0100:INFO : Gungho: computing horizontal flux coeffs -20240514181153.536+0100:INFO : Gungho: computing vertical flux coeffs -20240514181153.602+0100:INFO : Gungho: computing horizontal advective coeffs -20240514181153.690+0100:INFO : Gungho: computing vertical advective coeffs -20240514181153.759+0100:INFO : SI solver built with 32-bit real numbers -20240514181153.759+0100:INFO : Constructing pressure operator... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : semi_implicit_solver_alg_init: allocate and construct pressure/Helmholtz preconditioner -20240514181153.759+0100:INFO : Constructing pressure preconditioner... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : Multigrid_preconditioner_constructor[4]: setting smoothing parameters:0.8:2:2:4 -20240514181153.759+0100:INFO : construct_multigrid_preconditioner:make fields and operator on 4 levels -20240514181153.759+0100:INFO : Constructing pressure operator... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : Constructing pressure preconditioner... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : Constructing pressure operator... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : Constructing pressure preconditioner... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : Constructing pressure operator... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : Constructing pressure preconditioner... -20240514181153.759+0100:INFO : done -20240514181153.759+0100:INFO : semi_implicit_solver_alg_init: Setting up pressure solver -20240514181153.759+0100:INFO : Precondition only: No diagnostic norm output -20240514181153.759+0100:INFO : semi_implicit_solver_alg_init: ... Helmholtz done -20240514181153.759+0100:INFO : semi_implicit_solver_alg_init: allocate and construct mixed preconditioner -20240514181153.760+0100:INFO : Constructing approximate Schur mixed preconditioner... -20240514181153.760+0100:INFO : done -20240514181153.760+0100:INFO : semi_implicit_solver_alg_init: allocate and construct mixed solver -20240514181153.760+0100:INFO : semi_implicit_solver_alg_init: Initialised semi-implicit solver -20240514181153.760+0100:INFO : semi_implicit_timestep: initialised timestepping algorithm -20240514181156.266+0100:INFO : Conservation: dry mass 0.518558484586876416000000E+19 -20240514181156.266+0100:INFO : Conservation: total energy 0.128819341208001432046797E+25 -20240514181156.266+0100:INFO : Conservation: horizontal kinetic energy 0.395635108516052795392000E+21 -20240514181156.266+0100:INFO : Conservation: vertical kinetic energy 0.244949921513040488629102E-07 -20240514181156.266+0100:INFO : Conservation: potential energy 0.361311043441033289400320E+24 -20240514181156.266+0100:INFO : Conservation: dry internal energy 0.926486733530465052917760E+24 -20240514181156.266+0100:INFO : Conservation: moist internal energy 0.000000000000000000000000E+00 -20240514181156.266+0100:INFO : Conservation: dry aam 0.104846332118830507912478E+29 -20240514181156.266+0100:INFO : Conservation: dry physical entropy 0.197190441214010245120000E+20 -20240514181156.266+0100:INFO : Conservation: dry dynamic entropy 0.164108988309941302329344E+24 -20240514181156.266+0100:INFO : Configuration is not coupled to ocean -20240514181156.266+0100:S1:INFO : Start of timestep 1 -20240514181156.326+0100:S1:INFO : Mass matrix solve for W2 space -20240514181156.326+0100:S1:INFO : Constructing mass matrix operator... -20240514181156.326+0100:S1:INFO : done -20240514181156.326+0100:S1:INFO : Constructing diagonal preconditioner... -20240514181156.327+0100:S1:INFO : done -20240514181156.327+0100:S1:INFO : Gungho: mass matrix solve: -20240514181156.394+0100:S1:INFO : chebyshev[ 13], redidual = 0.13720263E-05 -20240514181156.396+0100:S1:INFO : Setting up transport_runtime on mesh: 1 -20240514181156.401+0100:S1:INFO : Min/max lipschitz_vertical = -0.16526623E-10 0.42183520E-10 -20240514181156.404+0100:S1:INFO : Transport: maximum horizontal CFL 0.25799002E+00 -20240514181156.405+0100:S1:INFO : Transport: maximum vertical CFL 0.42183520E-10 -20240514181156.405+0100:S1:INFO : Transport: maximum total CFL 0.25799002E+00 -20240514181156.405+0100:S1:INFO : Transporting density... -20240514181156.440+0100:S1:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181156.440+0100:S1:INFO : Transporting wind... -20240514181156.549+0100:S1:INFO : Transporting potential temperature... -20240514181157.240+0100:S1:INFO : loop indices (o, i): 1 1 -20240514181157.240+0100:S1:INFO : Gungho: computing si_operators -20240514181157.241+0100:S1:INFO : si_ops: mg level=1 -20240514181158.969+0100:S1:INFO : si_ops:MG restrict L1 to L2 -20240514181158.970+0100:S1:INFO : si_ops: mg level=2 -20240514181159.142+0100:S1:INFO : si_ops:MG restrict L2 to L3 -20240514181159.142+0100:S1:INFO : si_ops: mg level=3 -20240514181159.190+0100:S1:INFO : si_ops:MG restrict L3 to L4 -20240514181159.190+0100:S1:INFO : si_ops: mg level=4 -20240514181159.214+0100:S1:INFO : Gungho: mixed solve: -20240514181159.232+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181159.232+0100:S1:INFO : Precondition only starting -20240514181159.239+0100:S1:INFO : Precondition only: ... finished -20240514181159.258+0100:S1:INFO : BLOCK_GCR starting ... ||b|| = 0.28499729E+04: 0.28612875E+04 -20240514181159.262+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181159.262+0100:S1:INFO : Precondition only starting -20240514181159.270+0100:S1:INFO : Precondition only: ... finished -20240514181159.295+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181159.295+0100:S1:INFO : Precondition only starting -20240514181159.304+0100:S1:INFO : Precondition only: ... finished -20240514181159.327+0100:S1:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.2861E+04 final= 0.78064025E-01 abs= 0.11610027E+01 -20240514181159.327+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.11285603E+02 0.82191502E+00 0.72828630E-01 -20240514181159.327+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.51332286E-01 0.26264201E-03 0.51165072E-02 -20240514181159.327+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.28499505E+04 0.33882501E+00 0.11888803E-03 -20240514181159.936+0100:S1:INFO : loop indices (o, i): 1 2 -20240514181200.594+0100:S1:INFO : Gungho: mixed solve: -20240514181200.598+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181200.598+0100:S1:INFO : Precondition only starting -20240514181200.607+0100:S1:INFO : Precondition only: ... finished -20240514181200.627+0100:S1:INFO : BLOCK_GCR starting ... ||b|| = 0.26070021E+03: 0.26454480E+03 -20240514181200.630+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181200.630+0100:S1:INFO : Precondition only starting -20240514181200.639+0100:S1:INFO : Precondition only: ... finished -20240514181200.663+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181200.663+0100:S1:INFO : Precondition only starting -20240514181200.672+0100:S1:INFO : Precondition only: ... finished -20240514181200.695+0100:S1:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.2645E+03 final= 0.67319579E-01 abs= 0.32098644E+00 -20240514181200.695+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.38555418E+01 0.24652611E+00 0.63940718E-01 -20240514181200.695+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17558532E-01 0.54315938E-04 0.30934214E-02 -20240514181200.695+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.26067170E+03 0.74406019E-01 0.28543958E-03 -20240514181201.305+0100:S1:INFO : Setting up transport_runtime on mesh: 1 -20240514181201.309+0100:S1:INFO : Min/max lipschitz_vertical = -0.16526623E-10 0.42183520E-10 -20240514181201.313+0100:S1:INFO : Transport: maximum horizontal CFL 0.25964169E+00 -20240514181201.313+0100:S1:INFO : Transport: maximum vertical CFL 0.42183520E-10 -20240514181201.313+0100:S1:INFO : Transport: maximum total CFL 0.25964169E+00 -20240514181201.313+0100:S1:INFO : Transporting density... -20240514181201.341+0100:S1:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181201.341+0100:S1:INFO : Transporting wind... -20240514181201.447+0100:S1:INFO : Transporting potential temperature... -20240514181202.140+0100:S1:INFO : loop indices (o, i): 2 1 -20240514181202.140+0100:S1:INFO : Gungho: computing si_operators -20240514181202.141+0100:S1:INFO : si_ops: mg level=1 -20240514181203.838+0100:S1:INFO : si_ops:MG restrict L1 to L2 -20240514181203.839+0100:S1:INFO : si_ops: mg level=2 -20240514181204.007+0100:S1:INFO : si_ops:MG restrict L2 to L3 -20240514181204.008+0100:S1:INFO : si_ops: mg level=3 -20240514181204.052+0100:S1:INFO : si_ops:MG restrict L3 to L4 -20240514181204.052+0100:S1:INFO : si_ops: mg level=4 -20240514181204.076+0100:S1:INFO : Gungho: mixed solve: -20240514181204.080+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181204.080+0100:S1:INFO : Precondition only starting -20240514181204.088+0100:S1:INFO : Precondition only: ... finished -20240514181204.107+0100:S1:INFO : BLOCK_GCR starting ... ||b|| = 0.36712291E+03: 0.36866581E+03 -20240514181204.110+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181204.110+0100:S1:INFO : Precondition only starting -20240514181204.119+0100:S1:INFO : Precondition only: ... finished -20240514181204.143+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181204.143+0100:S1:INFO : Precondition only starting -20240514181204.152+0100:S1:INFO : Precondition only: ... finished -20240514181204.175+0100:S1:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.3687E+03 final= 0.89726618E-01 abs= 0.17517893E+00 -20240514181204.175+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.15157350E+01 0.13478089E+00 0.88921143E-01 -20240514181204.175+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.30297831E-01 0.21071898E-04 0.69549197E-03 -20240514181204.175+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.36711977E+03 0.40376970E-01 0.10998310E-03 -20240514181204.785+0100:S1:INFO : loop indices (o, i): 2 2 -20240514181205.442+0100:S1:INFO : Gungho: mixed solve: -20240514181205.446+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181205.446+0100:S1:INFO : Precondition only starting -20240514181205.455+0100:S1:INFO : Precondition only: ... finished -20240514181205.475+0100:S1:INFO : BLOCK_GCR starting ... ||b|| = 0.14787879E+03: 0.14846533E+03 -20240514181205.478+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181205.478+0100:S1:INFO : Precondition only starting -20240514181205.487+0100:S1:INFO : Precondition only: ... finished -20240514181205.511+0100:S1:INFO : Schur preconditioner pressure solve: -20240514181205.511+0100:S1:INFO : Precondition only starting -20240514181205.520+0100:S1:INFO : Precondition only: ... finished -20240514181205.543+0100:S1:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.1485E+03 final= 0.72391313E-01 abs= 0.52666324E-01 -20240514181205.543+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.57007801E+00 0.40499551E-01 0.71042120E-01 -20240514181205.543+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17563066E-01 0.22253591E-04 0.12670676E-02 -20240514181205.543+0100:S1:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.14787769E+03 0.12144520E-01 0.82125434E-04 -20240514181208.577+0100:S1:INFO : Conservation: dry mass 0.518558484586894131200000E+19 -20240514181208.577+0100:S1:INFO : Conservation: total energy 0.128820891640870834117018E+25 -20240514181208.577+0100:S1:INFO : Conservation: horizontal kinetic energy 0.397778444629730459648000E+21 -20240514181208.577+0100:S1:INFO : Conservation: vertical kinetic energy 0.450323375566936425781250E+13 -20240514181208.577+0100:S1:INFO : Conservation: potential energy 0.361216081024085055766528E+24 -20240514181208.577+0100:S1:INFO : Conservation: dry internal energy 0.926595056935490323218432E+24 -20240514181208.577+0100:S1:INFO : Conservation: moist internal energy 0.000000000000000000000000E+00 -20240514181208.577+0100:S1:INFO : Conservation: dry aam 0.104851787599623664622518E+29 -20240514181208.577+0100:S1:INFO : Conservation: dry physical entropy 0.197253253244961546240000E+20 -20240514181208.577+0100:S1:INFO : Conservation: dry dynamic entropy 0.164157673730163454509056E+24 -20240514181208.578+0100:S1:INFO : Min/max u = -0.19464412E+11 0.19485458E+11 -20240514181208.578+0100:S1:INFO : Min/max theta = 0.24004498E+03 0.66386877E+03 -20240514181208.578+0100:S1:INFO : End of timestep 1 -20240514181208.578+0100:S1:INFO : \****************************************************************************/ -20240514181208.579+0100:S2:INFO : Start of timestep 2 -20240514181208.644+0100:S2:INFO : Mass matrix solve for W2 space -20240514181208.644+0100:S2:INFO : Constructing mass matrix operator... -20240514181208.644+0100:S2:INFO : done -20240514181208.644+0100:S2:INFO : Constructing diagonal preconditioner... -20240514181208.644+0100:S2:INFO : done -20240514181208.644+0100:S2:INFO : Gungho: mass matrix solve: -20240514181208.711+0100:S2:INFO : chebyshev[ 13], redidual = 0.13719815E-05 -20240514181208.714+0100:S2:INFO : Setting up transport_runtime on mesh: 1 -20240514181208.717+0100:S2:INFO : Min/max lipschitz_vertical = -0.26699855E-02 0.45799384E-02 -20240514181208.721+0100:S2:INFO : Transport: maximum horizontal CFL 0.26121834E+00 -20240514181208.721+0100:S2:INFO : Transport: maximum vertical CFL 0.16878959E-01 -20240514181208.721+0100:S2:INFO : Transport: maximum total CFL 0.26788483E+00 -20240514181208.721+0100:S2:INFO : Transporting density... -20240514181208.749+0100:S2:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181208.749+0100:S2:INFO : Transporting wind... -20240514181208.856+0100:S2:INFO : Transporting potential temperature... -20240514181209.549+0100:S2:INFO : loop indices (o, i): 1 1 -20240514181209.549+0100:S2:INFO : Gungho: computing si_operators -20240514181209.549+0100:S2:INFO : si_ops: mg level=1 -20240514181211.258+0100:S2:INFO : si_ops:MG restrict L1 to L2 -20240514181211.259+0100:S2:INFO : si_ops: mg level=2 -20240514181211.428+0100:S2:INFO : si_ops:MG restrict L2 to L3 -20240514181211.428+0100:S2:INFO : si_ops: mg level=3 -20240514181211.473+0100:S2:INFO : si_ops:MG restrict L3 to L4 -20240514181211.473+0100:S2:INFO : si_ops: mg level=4 -20240514181211.496+0100:S2:INFO : Gungho: mixed solve: -20240514181211.501+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181211.501+0100:S2:INFO : Precondition only starting -20240514181211.508+0100:S2:INFO : Precondition only: ... finished -20240514181211.527+0100:S2:INFO : BLOCK_GCR starting ... ||b|| = 0.28634335E+04: 0.28761613E+04 -20240514181211.530+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181211.530+0100:S2:INFO : Precondition only starting -20240514181211.539+0100:S2:INFO : Precondition only: ... finished -20240514181211.563+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181211.563+0100:S2:INFO : Precondition only starting -20240514181211.572+0100:S2:INFO : Precondition only: ... finished -20240514181211.596+0100:S2:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.2876E+04 final= 0.55387204E-01 abs= 0.89440245E+00 -20240514181211.596+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.12522504E+02 0.68074885E+00 0.54362037E-01 -20240514181211.596+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.23267591E+00 0.22118839E-03 0.95062866E-03 -20240514181211.596+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.28634061E+04 0.21343241E+00 0.74537947E-04 -20240514181212.205+0100:S2:INFO : loop indices (o, i): 1 2 -20240514181212.863+0100:S2:INFO : Gungho: mixed solve: -20240514181212.867+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181212.867+0100:S2:INFO : Precondition only starting -20240514181212.876+0100:S2:INFO : Precondition only: ... finished -20240514181212.895+0100:S2:INFO : BLOCK_GCR starting ... ||b|| = 0.19792638E+03: 0.20112035E+03 -20240514181212.898+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181212.898+0100:S2:INFO : Precondition only starting -20240514181212.907+0100:S2:INFO : Precondition only: ... finished -20240514181212.931+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181212.931+0100:S2:INFO : Precondition only starting -20240514181212.940+0100:S2:INFO : Precondition only: ... finished -20240514181212.963+0100:S2:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.2011E+03 final= 0.64743575E-01 abs= 0.25760209E+00 -20240514181212.963+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.32022864E+01 0.20197259E+00 0.63071369E-01 -20240514181212.963+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17588780E-01 0.24470069E-04 0.13912318E-02 -20240514181212.963+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.19790047E+03 0.55605031E-01 0.28097473E-03 -20240514181213.572+0100:S2:INFO : Setting up transport_runtime on mesh: 1 -20240514181213.576+0100:S2:INFO : Min/max lipschitz_vertical = -0.26699855E-02 0.45799384E-02 -20240514181213.580+0100:S2:INFO : Transport: maximum horizontal CFL 0.26104169E+00 -20240514181213.580+0100:S2:INFO : Transport: maximum vertical CFL 0.16878959E-01 -20240514181213.580+0100:S2:INFO : Transport: maximum total CFL 0.26757376E+00 -20240514181213.580+0100:S2:INFO : Transporting density... -20240514181213.608+0100:S2:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181213.608+0100:S2:INFO : Transporting wind... -20240514181213.715+0100:S2:INFO : Transporting potential temperature... -20240514181214.408+0100:S2:INFO : loop indices (o, i): 2 1 -20240514181214.408+0100:S2:INFO : Gungho: computing si_operators -20240514181214.408+0100:S2:INFO : si_ops: mg level=1 -20240514181216.118+0100:S2:INFO : si_ops:MG restrict L1 to L2 -20240514181216.119+0100:S2:INFO : si_ops: mg level=2 -20240514181216.287+0100:S2:INFO : si_ops:MG restrict L2 to L3 -20240514181216.288+0100:S2:INFO : si_ops: mg level=3 -20240514181216.332+0100:S2:INFO : si_ops:MG restrict L3 to L4 -20240514181216.332+0100:S2:INFO : si_ops: mg level=4 -20240514181216.355+0100:S2:INFO : Gungho: mixed solve: -20240514181216.359+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181216.360+0100:S2:INFO : Precondition only starting -20240514181216.367+0100:S2:INFO : Precondition only: ... finished -20240514181216.386+0100:S2:INFO : BLOCK_GCR starting ... ||b|| = 0.65992104E+03: 0.66175548E+03 -20240514181216.389+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181216.390+0100:S2:INFO : Precondition only starting -20240514181216.398+0100:S2:INFO : Precondition only: ... finished -20240514181216.422+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181216.422+0100:S2:INFO : Precondition only starting -20240514181216.431+0100:S2:INFO : Precondition only: ... finished -20240514181216.457+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181216.457+0100:S2:INFO : Precondition only starting -20240514181216.466+0100:S2:INFO : Precondition only: ... finished -20240514181216.491+0100:S2:INFO : BLOCK_GCR solver_algorithm: [ 1, 3], iters, init= 0.6618E+03 final= 0.58931029E-01 abs= 0.14032065E+00 -20240514181216.491+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.18060716E+01 0.10585187E+00 0.58608902E-01 -20240514181216.491+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.30845119E-01 0.83253158E-05 0.26990707E-03 -20240514181216.491+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.65991857E+03 0.34460445E-01 0.52219239E-04 -20240514181217.101+0100:S2:INFO : loop indices (o, i): 2 2 -20240514181217.759+0100:S2:INFO : Gungho: mixed solve: -20240514181217.763+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181217.763+0100:S2:INFO : Precondition only starting -20240514181217.772+0100:S2:INFO : Precondition only: ... finished -20240514181217.791+0100:S2:INFO : BLOCK_GCR starting ... ||b|| = 0.14617643E+03: 0.14714560E+03 -20240514181217.794+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181217.794+0100:S2:INFO : Precondition only starting -20240514181217.803+0100:S2:INFO : Precondition only: ... finished -20240514181217.827+0100:S2:INFO : Schur preconditioner pressure solve: -20240514181217.827+0100:S2:INFO : Precondition only starting -20240514181217.836+0100:S2:INFO : Precondition only: ... finished -20240514181217.860+0100:S2:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.1471E+03 final= 0.52421746E-01 abs= 0.64412724E-01 -20240514181217.860+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.95471702E+00 0.49599024E-01 0.51951545E-01 -20240514181217.860+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17576454E-01 0.64839964E-05 0.36890242E-03 -20240514181217.860+0100:S2:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.14617331E+03 0.14807216E-01 0.10129904E-03 -20240514181220.894+0100:S2:INFO : Conservation: dry mass 0.518558484586865459200000E+19 -20240514181220.894+0100:S2:INFO : Conservation: total energy 0.128819330333784322919629E+25 -20240514181220.894+0100:S2:INFO : Conservation: horizontal kinetic energy 0.396762972576729792512000E+21 -20240514181220.894+0100:S2:INFO : Conservation: vertical kinetic energy 0.156304612719413007812500E+14 -20240514181220.894+0100:S2:INFO : Conservation: potential energy 0.361303870831214725169152E+24 -20240514181220.894+0100:S2:INFO : Conservation: dry internal energy 0.926492669518421382135808E+24 -20240514181220.894+0100:S2:INFO : Conservation: moist internal energy 0.000000000000000000000000E+00 -20240514181220.894+0100:S2:INFO : Conservation: dry aam 0.104846745082680597247500E+29 -20240514181220.894+0100:S2:INFO : Conservation: dry physical entropy 0.197194456334854389760000E+20 -20240514181220.894+0100:S2:INFO : Conservation: dry dynamic entropy 0.164110401390432250494976E+24 -20240514181220.895+0100:S2:INFO : Min/max u = -0.19475632E+11 0.19492437E+11 -20240514181220.895+0100:S2:INFO : Min/max theta = 0.24004499E+03 0.66386848E+03 -20240514181220.895+0100:S2:INFO : End of timestep 2 -20240514181220.895+0100:S2:INFO : \****************************************************************************/ -20240514181220.895+0100:S3:INFO : Start of timestep 3 -20240514181220.960+0100:S3:INFO : Mass matrix solve for W2 space -20240514181220.960+0100:S3:INFO : Constructing mass matrix operator... -20240514181220.960+0100:S3:INFO : done -20240514181220.960+0100:S3:INFO : Constructing diagonal preconditioner... -20240514181220.960+0100:S3:INFO : done -20240514181220.961+0100:S3:INFO : Gungho: mass matrix solve: -20240514181221.028+0100:S3:INFO : chebyshev[ 13], redidual = 0.13717897E-05 -20240514181221.031+0100:S3:INFO : Setting up transport_runtime on mesh: 1 -20240514181221.035+0100:S3:INFO : Min/max lipschitz_vertical = -0.49123912E-02 0.50615125E-02 -20240514181221.038+0100:S3:INFO : Transport: maximum horizontal CFL 0.26099408E+00 -20240514181221.038+0100:S3:INFO : Transport: maximum vertical CFL 0.21230683E-01 -20240514181221.038+0100:S3:INFO : Transport: maximum total CFL 0.27480678E+00 -20240514181221.039+0100:S3:INFO : Transporting density... -20240514181221.066+0100:S3:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181221.067+0100:S3:INFO : Transporting wind... -20240514181221.173+0100:S3:INFO : Transporting potential temperature... -20240514181221.866+0100:S3:INFO : loop indices (o, i): 1 1 -20240514181221.867+0100:S3:INFO : Gungho: computing si_operators -20240514181221.867+0100:S3:INFO : si_ops: mg level=1 -20240514181223.576+0100:S3:INFO : si_ops:MG restrict L1 to L2 -20240514181223.577+0100:S3:INFO : si_ops: mg level=2 -20240514181223.746+0100:S3:INFO : si_ops:MG restrict L2 to L3 -20240514181223.746+0100:S3:INFO : si_ops: mg level=3 -20240514181223.790+0100:S3:INFO : si_ops:MG restrict L3 to L4 -20240514181223.790+0100:S3:INFO : si_ops: mg level=4 -20240514181223.814+0100:S3:INFO : Gungho: mixed solve: -20240514181223.818+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181223.818+0100:S3:INFO : Precondition only starting -20240514181223.825+0100:S3:INFO : Precondition only: ... finished -20240514181223.845+0100:S3:INFO : BLOCK_GCR starting ... ||b|| = 0.34315373E+04: 0.34462618E+04 -20240514181223.848+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181223.848+0100:S3:INFO : Precondition only starting -20240514181223.857+0100:S3:INFO : Precondition only: ... finished -20240514181223.881+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181223.881+0100:S3:INFO : Precondition only starting -20240514181223.890+0100:S3:INFO : Precondition only: ... finished -20240514181223.913+0100:S3:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.3446E+04 final= 0.55468391E-01 abs= 0.10285451E+01 -20240514181223.913+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.14366185E+02 0.78804266E+00 0.54853996E-01 -20240514181223.913+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.38844546E+00 0.21145822E-03 0.54437043E-03 -20240514181223.913+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.34315072E+04 0.24029098E+00 0.70024910E-04 -20240514181224.522+0100:S3:INFO : loop indices (o, i): 1 2 -20240514181225.180+0100:S3:INFO : Gungho: mixed solve: -20240514181225.184+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181225.184+0100:S3:INFO : Precondition only starting -20240514181225.193+0100:S3:INFO : Precondition only: ... finished -20240514181225.212+0100:S3:INFO : BLOCK_GCR starting ... ||b|| = 0.34661703E+03: 0.35069719E+03 -20240514181225.215+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181225.215+0100:S3:INFO : Precondition only starting -20240514181225.224+0100:S3:INFO : Precondition only: ... finished -20240514181225.248+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181225.248+0100:S3:INFO : Precondition only starting -20240514181225.257+0100:S3:INFO : Precondition only: ... finished -20240514181225.280+0100:S3:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.3507E+03 final= 0.55170318E-01 abs= 0.28019221E+00 -20240514181225.280+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.40867120E+01 0.21317609E+00 0.52163227E-01 -20240514181225.280+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17540521E-01 0.49356853E-04 0.28138760E-02 -20240514181225.281+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.34659294E+03 0.66966765E-01 0.19321445E-03 -20240514181225.890+0100:S3:INFO : Setting up transport_runtime on mesh: 1 -20240514181225.894+0100:S3:INFO : Min/max lipschitz_vertical = -0.49123912E-02 0.50615125E-02 -20240514181225.897+0100:S3:INFO : Transport: maximum horizontal CFL 0.26162458E+00 -20240514181225.897+0100:S3:INFO : Transport: maximum vertical CFL 0.21230683E-01 -20240514181225.897+0100:S3:INFO : Transport: maximum total CFL 0.27540812E+00 -20240514181225.897+0100:S3:INFO : Transporting density... -20240514181225.925+0100:S3:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181225.925+0100:S3:INFO : Transporting wind... -20240514181226.032+0100:S3:INFO : Transporting potential temperature... -20240514181226.725+0100:S3:INFO : loop indices (o, i): 2 1 -20240514181226.725+0100:S3:INFO : Gungho: computing si_operators -20240514181226.725+0100:S3:INFO : si_ops: mg level=1 -20240514181228.436+0100:S3:INFO : si_ops:MG restrict L1 to L2 -20240514181228.436+0100:S3:INFO : si_ops: mg level=2 -20240514181228.605+0100:S3:INFO : si_ops:MG restrict L2 to L3 -20240514181228.605+0100:S3:INFO : si_ops: mg level=3 -20240514181228.650+0100:S3:INFO : si_ops:MG restrict L3 to L4 -20240514181228.650+0100:S3:INFO : si_ops: mg level=4 -20240514181228.673+0100:S3:INFO : Gungho: mixed solve: -20240514181228.677+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181228.677+0100:S3:INFO : Precondition only starting -20240514181228.685+0100:S3:INFO : Precondition only: ... finished -20240514181228.704+0100:S3:INFO : BLOCK_GCR starting ... ||b|| = 0.12615170E+04: 0.12644590E+04 -20240514181228.707+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181228.707+0100:S3:INFO : Precondition only starting -20240514181228.716+0100:S3:INFO : Precondition only: ... finished -20240514181228.740+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181228.740+0100:S3:INFO : Precondition only starting -20240514181228.749+0100:S3:INFO : Precondition only: ... finished -20240514181228.772+0100:S3:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.1264E+04 final= 0.92057296E-01 abs= 0.33844783E+00 -20240514181228.773+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.28641274E+01 0.26251398E+00 0.91655833E-01 -20240514181228.773+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.81124046E-01 0.27687038E-04 0.34129261E-03 -20240514181228.773+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.12615138E+04 0.75906161E-01 0.60170695E-04 -20240514181229.382+0100:S3:INFO : loop indices (o, i): 2 2 -20240514181230.039+0100:S3:INFO : Gungho: mixed solve: -20240514181230.044+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181230.044+0100:S3:INFO : Precondition only starting -20240514181230.052+0100:S3:INFO : Precondition only: ... finished -20240514181230.072+0100:S3:INFO : BLOCK_GCR starting ... ||b|| = 0.14830954E+03: 0.14967788E+03 -20240514181230.075+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181230.075+0100:S3:INFO : Precondition only starting -20240514181230.084+0100:S3:INFO : Precondition only: ... finished -20240514181230.108+0100:S3:INFO : Schur preconditioner pressure solve: -20240514181230.108+0100:S3:INFO : Precondition only starting -20240514181230.117+0100:S3:INFO : Precondition only: ... finished -20240514181230.140+0100:S3:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.1497E+03 final= 0.54465233E-01 abs= 0.94202118E-01 -20240514181230.140+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.13569812E+01 0.73143691E-01 0.53901772E-01 -20240514181230.140+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17559926E-01 0.74017793E-05 0.42151540E-03 -20240514181230.140+0100:S3:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.14830333E+03 0.21051026E-01 0.14194574E-03 -20240514181233.175+0100:S3:INFO : Conservation: dry mass 0.518558484587003904000000E+19 -20240514181233.175+0100:S3:INFO : Conservation: total energy 0.128821839020541126429901E+25 -20240514181233.175+0100:S3:INFO : Conservation: horizontal kinetic energy 0.399564851923196116992000E+21 -20240514181233.175+0100:S3:INFO : Conservation: vertical kinetic energy 0.346429817008122421875000E+14 -20240514181233.175+0100:S3:INFO : Conservation: potential energy 0.361217105183989473214464E+24 -20240514181233.175+0100:S3:INFO : Conservation: dry internal energy 0.926601720134855685046272E+24 -20240514181233.175+0100:S3:INFO : Conservation: moist internal energy 0.000000000000000000000000E+00 -20240514181233.175+0100:S3:INFO : Conservation: dry aam 0.104852317943115891393104E+29 -20240514181233.175+0100:S3:INFO : Conservation: dry physical entropy 0.197253472434171535360000E+20 -20240514181233.175+0100:S3:INFO : Conservation: dry dynamic entropy 0.164158099471694766800896E+24 -20240514181233.176+0100:S3:INFO : Min/max u = -0.19561411E+11 0.19564367E+11 -20240514181233.176+0100:S3:INFO : Min/max theta = 0.24004504E+03 0.66386881E+03 -20240514181233.176+0100:S3:INFO : End of timestep 3 -20240514181233.176+0100:S3:INFO : \****************************************************************************/ -20240514181233.176+0100:S4:INFO : Start of timestep 4 -20240514181233.241+0100:S4:INFO : Mass matrix solve for W2 space -20240514181233.241+0100:S4:INFO : Constructing mass matrix operator... -20240514181233.241+0100:S4:INFO : done -20240514181233.241+0100:S4:INFO : Constructing diagonal preconditioner... -20240514181233.241+0100:S4:INFO : done -20240514181233.242+0100:S4:INFO : Gungho: mass matrix solve: -20240514181233.309+0100:S4:INFO : chebyshev[ 13], redidual = 0.13717968E-05 -20240514181233.311+0100:S4:INFO : Setting up transport_runtime on mesh: 1 -20240514181233.315+0100:S4:INFO : Min/max lipschitz_vertical = -0.42094510E-02 0.64327047E-02 -20240514181233.318+0100:S4:INFO : Transport: maximum horizontal CFL 0.26215079E+00 -20240514181233.318+0100:S4:INFO : Transport: maximum vertical CFL 0.29018009E-01 -20240514181233.318+0100:S4:INFO : Transport: maximum total CFL 0.27765320E+00 -20240514181233.319+0100:S4:INFO : Transporting density... -20240514181233.347+0100:S4:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181233.347+0100:S4:INFO : Transporting wind... -20240514181233.453+0100:S4:INFO : Transporting potential temperature... -20240514181234.147+0100:S4:INFO : loop indices (o, i): 1 1 -20240514181234.147+0100:S4:INFO : Gungho: computing si_operators -20240514181234.147+0100:S4:INFO : si_ops: mg level=1 -20240514181235.859+0100:S4:INFO : si_ops:MG restrict L1 to L2 -20240514181235.860+0100:S4:INFO : si_ops: mg level=2 -20240514181236.029+0100:S4:INFO : si_ops:MG restrict L2 to L3 -20240514181236.029+0100:S4:INFO : si_ops: mg level=3 -20240514181236.073+0100:S4:INFO : si_ops:MG restrict L3 to L4 -20240514181236.073+0100:S4:INFO : si_ops: mg level=4 -20240514181236.097+0100:S4:INFO : Gungho: mixed solve: -20240514181236.101+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181236.101+0100:S4:INFO : Precondition only starting -20240514181236.108+0100:S4:INFO : Precondition only: ... finished -20240514181236.128+0100:S4:INFO : BLOCK_GCR starting ... ||b|| = 0.35620513E+04: 0.35773372E+04 -20240514181236.131+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181236.131+0100:S4:INFO : Precondition only starting -20240514181236.140+0100:S4:INFO : Precondition only: ... finished -20240514181236.164+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181236.164+0100:S4:INFO : Precondition only starting -20240514181236.173+0100:S4:INFO : Precondition only: ... finished -20240514181236.196+0100:S4:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.3577E+04 final= 0.58704728E-01 abs= 0.11424760E+01 -20240514181236.196+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.14711049E+02 0.85893303E+00 0.58386932E-01 -20240514181236.196+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.60520753E+00 0.14418091E-03 0.23823384E-03 -20240514181236.196+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.35620209E+04 0.28339884E+00 0.79561251E-04 -20240514181236.805+0100:S4:INFO : loop indices (o, i): 1 2 -20240514181237.462+0100:S4:INFO : Gungho: mixed solve: -20240514181237.467+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181237.467+0100:S4:INFO : Precondition only starting -20240514181237.475+0100:S4:INFO : Precondition only: ... finished -20240514181237.495+0100:S4:INFO : BLOCK_GCR starting ... ||b|| = 0.33698572E+03: 0.34108707E+03 -20240514181237.498+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181237.498+0100:S4:INFO : Precondition only starting -20240514181237.507+0100:S4:INFO : Precondition only: ... finished -20240514181237.531+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181237.531+0100:S4:INFO : Precondition only starting -20240514181237.540+0100:S4:INFO : Precondition only: ... finished -20240514181237.563+0100:S4:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.3411E+03 final= 0.56976035E-01 abs= 0.29298480E+00 -20240514181237.563+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.41087970E+01 0.22339604E+00 0.54370182E-01 -20240514181237.563+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17603714E-01 0.42239398E-04 0.23994595E-02 -20240514181237.563+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.33696067E+03 0.69546517E-01 0.20639358E-03 -20240514181238.172+0100:S4:INFO : Setting up transport_runtime on mesh: 1 -20240514181238.176+0100:S4:INFO : Min/max lipschitz_vertical = -0.42094510E-02 0.64327047E-02 -20240514181238.180+0100:S4:INFO : Transport: maximum horizontal CFL 0.26217586E+00 -20240514181238.180+0100:S4:INFO : Transport: maximum vertical CFL 0.29018009E-01 -20240514181238.180+0100:S4:INFO : Transport: maximum total CFL 0.27818231E+00 -20240514181238.180+0100:S4:INFO : Transporting density... -20240514181238.208+0100:S4:INFO : Transport: number of MoL substeps in 3D direction = 1 -20240514181238.208+0100:S4:INFO : Transporting wind... -20240514181238.315+0100:S4:INFO : Transporting potential temperature... -20240514181239.008+0100:S4:INFO : loop indices (o, i): 2 1 -20240514181239.008+0100:S4:INFO : Gungho: computing si_operators -20240514181239.008+0100:S4:INFO : si_ops: mg level=1 -20240514181240.719+0100:S4:INFO : si_ops:MG restrict L1 to L2 -20240514181240.720+0100:S4:INFO : si_ops: mg level=2 -20240514181240.888+0100:S4:INFO : si_ops:MG restrict L2 to L3 -20240514181240.889+0100:S4:INFO : si_ops: mg level=3 -20240514181240.933+0100:S4:INFO : si_ops:MG restrict L3 to L4 -20240514181240.933+0100:S4:INFO : si_ops: mg level=4 -20240514181240.957+0100:S4:INFO : Gungho: mixed solve: -20240514181240.961+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181240.961+0100:S4:INFO : Precondition only starting -20240514181240.968+0100:S4:INFO : Precondition only: ... finished -20240514181240.987+0100:S4:INFO : BLOCK_GCR starting ... ||b|| = 0.15096316E+04: 0.15133473E+04 -20240514181240.991+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181240.991+0100:S4:INFO : Precondition only starting -20240514181240.999+0100:S4:INFO : Precondition only: ... finished -20240514181241.023+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181241.023+0100:S4:INFO : Precondition only starting -20240514181241.032+0100:S4:INFO : Precondition only: ... finished -20240514181241.055+0100:S4:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.1513E+04 final= 0.93474816E-01 abs= 0.43441886E+00 -20240514181241.056+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.36412133E+01 0.33832623E+00 0.92915795E-01 -20240514181241.056+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.78899817E-01 0.39086462E-04 0.49539357E-03 -20240514181241.056+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.15096272E+04 0.96053547E-01 0.63627328E-04 -20240514181241.663+0100:S4:INFO : loop indices (o, i): 2 2 -20240514181242.320+0100:S4:INFO : Gungho: mixed solve: -20240514181242.324+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181242.324+0100:S4:INFO : Precondition only starting -20240514181242.333+0100:S4:INFO : Precondition only: ... finished -20240514181242.352+0100:S4:INFO : BLOCK_GCR starting ... ||b|| = 0.14519468E+03: 0.14713316E+03 -20240514181242.356+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181242.356+0100:S4:INFO : Precondition only starting -20240514181242.364+0100:S4:INFO : Precondition only: ... finished -20240514181242.388+0100:S4:INFO : Schur preconditioner pressure solve: -20240514181242.389+0100:S4:INFO : Precondition only starting -20240514181242.397+0100:S4:INFO : Precondition only: ... finished -20240514181242.421+0100:S4:INFO : BLOCK_GCR solver_algorithm: [ 1, 2], iters, init= 0.1471E+03 final= 0.55943974E-01 abs= 0.13608150E+00 -20240514181242.421+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 1 0.19337835E+01 0.10640783E+00 0.55025719E-01 -20240514181242.421+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 2 0.17578572E-01 0.12550256E-04 0.71395196E-03 -20240514181242.421+0100:S4:INFO : BLOCK_GCR field errors (Init/Final/Rel): 3 0.14518180E+03 0.29661126E-01 0.20430333E-03 -20240514181245.455+0100:S4:INFO : Conservation: dry mass 0.518558484586940928000000E+19 -20240514181245.455+0100:S4:INFO : Conservation: total energy 0.128818482139284821875098E+25 -20240514181245.455+0100:S4:INFO : Conservation: horizontal kinetic energy 0.398650362891281432576000E+21 -20240514181245.455+0100:S4:INFO : Conservation: vertical kinetic energy 0.584765380067394062500000E+14 -20240514181245.455+0100:S4:INFO : Conservation: potential energy 0.361301974871262378328064E+24 -20240514181245.455+0100:S4:INFO : Conservation: dry internal energy 0.926484196100217965117440E+24 -20240514181245.455+0100:S4:INFO : Conservation: moist internal energy 0.000000000000000000000000E+00 -20240514181245.455+0100:S4:INFO : Conservation: dry aam 0.104847140131188872894784E+29 -20240514181245.455+0100:S4:INFO : Conservation: dry physical entropy 0.197195418962301665280000E+20 -20240514181245.455+0100:S4:INFO : Conservation: dry dynamic entropy 0.164108906326561879228416E+24 -20240514181245.456+0100:S4:INFO : Min/max u = -0.19558914E+11 0.19566959E+11 -20240514181245.456+0100:S4:INFO : Min/max theta = 0.24004507E+03 0.66386832E+03 -20240514181245.456+0100:S4:INFO : End of timestep 4 -20240514181245.456+0100:S4:INFO : \****************************************************************************/ -20240514181245.457+0100:INFO : finalise_model_data: all fields have been cleared -20240514181248.465+0100:INFO : ||= Routine =|| = time(s) =||= No. calls =||= time per call(s) =|| -20240514181248.465+0100:INFO : || gungho_model|| 94.40|| 1|| 94.3956|| -20240514181248.465+0100:INFO : || runtime_constants_alg|| 20.11|| 1|| 20.1091|| -20240514181248.465+0100:INFO : || geometric_constants_alg|| 0.16|| 1|| 0.1605|| -20240514181248.465+0100:INFO : || fem_constants_alg|| 13.31|| 1|| 13.3099|| -20240514181248.465+0100:INFO : || physical_op_constants_alg|| 6.63|| 1|| 6.6272|| -20240514181248.465+0100:INFO : || intermesh_constants_alg|| 0.00|| 1|| 0.0000|| -20240514181248.465+0100:INFO : || mass_matrix_solver_alg|| 13.28|| 42|| 0.3161|| -20240514181248.465+0100:INFO : || xios_update_calendar|| 0.00|| 5|| 0.0001|| -20240514181248.465+0100:INFO : || gungho_diagnostics_driver|| 15.06|| 1|| 15.0628|| -20240514181248.465+0100:INFO : || si_operators_alg:create|| 0.00|| 1|| 0.0006|| -20240514181248.465+0100:INFO : || conservation_alg|| 12.17|| 5|| 2.4333|| -20240514181248.465+0100:INFO : || semi_implicit_timestep_alg|| 39.49|| 4|| 9.8715|| -20240514181248.465+0100:INFO : || rhs_alg|| 10.74|| 20|| 0.5372|| -20240514181248.465+0100:INFO : || gungho_transport_control|| 1.41|| 8|| 0.1766|| -20240514181248.465+0100:INFO : || transport.runtime_init|| 0.06|| 8|| 0.0076|| -20240514181248.465+0100:INFO : || transport.runtime_allocate|| 0.00|| 2|| 0.0000|| -20240514181248.465+0100:INFO : || transport.field_initialise|| 0.00|| 8|| 0.0001|| -20240514181248.465+0100:INFO : || transport.vert_dep_pts|| 0.01|| 8|| 0.0013|| -20240514181248.465+0100:INFO : || transport.cfl_calculation|| 0.03|| 8|| 0.0036|| -20240514181248.465+0100:INFO : || transport.mol_conservative|| 0.23|| 8|| 0.0285|| -20240514181248.465+0100:INFO : || transport.w3_hori_recon|| 0.30|| 160|| 0.0019|| -20240514181248.465+0100:INFO : || transport.w3_vert_recon|| 0.15|| 160|| 0.0009|| -20240514181248.465+0100:INFO : || transport.end_of_transport|| 0.02|| 40|| 0.0004|| -20240514181248.465+0100:INFO : || transport.build_up_flux|| 0.01|| 8|| 0.0007|| -20240514181248.465+0100:INFO : || transport.wind|| 0.86|| 8|| 0.1070|| -20240514181248.465+0100:INFO : || transport.wind_to_comps|| 0.21|| 8|| 0.0262|| -20240514181248.465+0100:INFO : || transport.mol_advective|| 0.84|| 32|| 0.0263|| -20240514181248.465+0100:INFO : || transport.wind_from_comps|| 0.05|| 8|| 0.0066|| -20240514181248.465+0100:INFO : || transport.wt_hori_recon|| 0.14|| 40|| 0.0035|| -20240514181248.465+0100:INFO : || transport.wt_vert_recon|| 0.07|| 40|| 0.0017|| -20240514181248.465+0100:INFO : || si_operators_alg:compute|| 15.52|| 8|| 1.9399|| -20240514181248.465+0100:INFO : || semi_implicit_solver_alg|| 1.84|| 16|| 0.1151|| -20240514181248.465+0100:INFO : || mixed_solver|| 1.66|| 16|| 0.1036|| -20240514181248.465+0100:INFO : || mixed_schur_preconditioner_alg|| 0.68|| 49|| 0.0138|| -20240514181248.465+0100:INFO : || mixed_schur rhs|| 0.14|| 49|| 0.0028|| -20240514181248.466+0100:INFO : || mixed_schur solve|| 0.43|| 49|| 0.0087|| -20240514181248.466+0100:INFO : || helmholtz lhs|| 0.17|| 735|| 0.0002|| -20240514181248.466+0100:INFO : || schur back substitute|| 0.11|| 49|| 0.0023|| -20240514181248.466+0100:INFO : || mixed_operator|| 0.75|| 49|| 0.0152|| -20240514181248.466+0100:INFO : || exner_from_eos|| 9.62|| 16|| 0.6015|| -20240514181248.466+0100:INFO : gungho_model completed. diff --git a/examples/lfric/scripts/compare_ouput.py b/examples/lfric/scripts/compare_ouput.py index a951205ebf..c5eafe95de 100644 --- a/examples/lfric/scripts/compare_ouput.py +++ b/examples/lfric/scripts/compare_ouput.py @@ -38,9 +38,8 @@ import sys import os -import math -TOLERANCE = 1e-04 +TOLERANCE_DIGITS = 10 def main(): @@ -64,14 +63,10 @@ def main(): line_f1 = file1.readline() line_f2 = file2.readline() while line_f1 and line_f2: - if ( - "Conservation" in line_f1 - or "Residual" in line_f1 - ): - value_f1 = float(line_f1.split()[-1]) - value_f2 = float(line_f2.split()[-1]) - if not math.isclose(value_f1, value_f2, rel_tol=TOLERANCE): - sys.exit(f"The values are not equal:\n{line_f1}{line_f2}") + value1 = line_f1.split('=')[-1][:TOLERANCE_DIGITS] + value2 = line_f2.split('=')[-1][:TOLERANCE_DIGITS] + if value1 != value2: + sys.exit(f"The values are not equal:\n{line_f1}{line_f2}") # Get next lines line_f1 = file1.readline() From 4af63ffd9025d717fe488172015afbb3d2788d46 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 29 Aug 2024 09:57:01 +0100 Subject: [PATCH 11/16] #2692 Add NEMOv5 output comparisons with a tolerance --- .github/workflows/nemo_v5_tests.yml | 37 ++++++------ examples/nemo/scripts/compare_ouput.py | 84 ++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 examples/nemo/scripts/compare_ouput.py diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index 55f7c77efa..634c55ba9a 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -71,7 +71,7 @@ jobs: pip install . # PSyclone passthrough for 5.0-beta of NEMO. - - name: NEMO 5.0 gcc passthrough + - name: NEMO 5.0 gfortran passthrough run: | # Set up environment source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh @@ -95,7 +95,7 @@ jobs: cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00 cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg mpirun -np 4 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -117,6 +117,7 @@ jobs: export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" # Clean up and compile + # Without key_mpi_off it fails to compile (even without psyclone) ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ del_key "key_xios key_iomput key_top" add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 @@ -125,7 +126,7 @@ jobs: cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg mpirun -np 4 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -153,17 +154,18 @@ jobs: # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - mpirun -np 4 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - cat run.stat - export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - echo "Time-stepping duration = " $VAR_TIME + # It currently segfaults + # mpirun -np 4 ./nemo + # python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat + # cat run.stat + # export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + # echo "Time-stepping duration = " $VAR_TIME - - name: NEMO 5.0 Intel OpenMP for CPUs + - name: NEMO 5.0 gfortran OpenMP for CPUs run: | # Set up environment source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh - spack unload && spack load nemo-build-environment%oneapi + spack unload && spack load nemo-build-environment%gcc@14 source .runner_venv/bin/activate export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts export PSYCLONE_HOME=${PWD}/.runner_venv @@ -172,18 +174,18 @@ jobs: # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS cd $NEMO_DIR cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - export FCFLAGS="-i4 -r8 -O2 -fp-model precise -fno-alias -g -qopenmp" + export FCFLAGS="-fdefault-real-8 -O2 -fcray-pointer -ffree-line-length-none -g -fopenmp" # Clean up and compile - ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_ONEAPI clean -y - ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_ONEAPI -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ + ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_GCC clean -y + ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_GCC -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 # Run Orca2 - cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 + cd $NEMO_DIR/tests/BENCH_OMP_THREADING_GCC/EXP00 cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg OMP_NUM_THREADS=4 mpirun -np 1 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -205,15 +207,16 @@ jobs: export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -g -mp=gpu -gpu=managed" # Clean up and compile + # Without key_mpi_off it fails to compile (even without psyclone) ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ del_key "key_xios key_iomput key_top" add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 # Run Orca2 - cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 + cd $NEMO_DIR/tests/BENCH_OMP_OFFLOAD_NVHPC/EXP00 cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg mpirun -np 1 ./nemo - # diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat cat run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/nemo/scripts/compare_ouput.py b/examples/nemo/scripts/compare_ouput.py new file mode 100644 index 0000000000..8760adcb33 --- /dev/null +++ b/examples/nemo/scripts/compare_ouput.py @@ -0,0 +1,84 @@ +# ----------------------------------------------------------------------------- +# BSD 3-Clause License +# +# Copyright (c) 2023-2024, Science and Technology Facilities Council +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# ----------------------------------------------------------------------------- +# Author: S. Siso, STFC Daresbury Lab +# ----------------------------------------------------------------------------- + +''' Script to compare LFRic output files. ''' + +import sys +import os + +TOLERANCE_DIGITS = 8 + + +def main(): + ''' Compare the two provided LFRic output files, it checks that the + Conservation and Residual values are equivalent (within a tolerance). ''' + + # Parse input arguments + if len(sys.argv) - 1 != 2: + sys.exit(f"This script expects exactly 2 arguments with the two files " + f"to compare, but got: {sys.argv}") + filename1 = sys.argv[1] + filename2 = sys.argv[2] + if not os.path.isfile(filename1): + sys.exit(f"The first argument '{filename1}' must point to a file.") + if not os.path.isfile(filename2): + sys.exit(f"The second argument '{filename2}' must point to a file.") + + # Compare filename1 with filename2 + with open(filename1, 'r', encoding="utf-8") as file1, \ + open(filename2, 'r', encoding="utf-8") as file2: + line_f1 = file1.readline() + line_f2 = file2.readline() + while line_f1 and line_f2: + # Get all numbers (rhs of each :) + values1 = line_f1.split(':')[::2] + values2 = line_f2.split(':')[::2] + for value1, value2 in zip(values1, values2): + if value1[:TOLERANCE_DIGITS] != value2[:TOLERANCE_DIGITS]: + sys.exit(f"The values are not equal:\n{line_f1}{line_f2}") + + # Get next lines + line_f1 = file1.readline() + line_f2 = file2.readline() + + if file1.readline() or file2.readline(): + sys.exit("The files have a different number of lines") + + sys.exit(0) # Successful comparison + + +if __name__ == "__main__": + main() From 72206e2252a99bd047f4d0c14434790df26e484e Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Thu, 29 Aug 2024 10:23:55 +0100 Subject: [PATCH 12/16] #2692 Fix output comparison scripts --- examples/lfric/scripts/compare_ouput.py | 2 +- examples/nemo/scripts/compare_ouput.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/examples/lfric/scripts/compare_ouput.py b/examples/lfric/scripts/compare_ouput.py index c5eafe95de..3edc37c078 100644 --- a/examples/lfric/scripts/compare_ouput.py +++ b/examples/lfric/scripts/compare_ouput.py @@ -44,7 +44,7 @@ def main(): ''' Compare the two provided LFRic output files, it checks that the - Conservation and Residual values are equivalent (within a tolerance). ''' + checksum values are equivalent (within a tolerance). ''' # Parse input arguments if len(sys.argv) - 1 != 2: diff --git a/examples/nemo/scripts/compare_ouput.py b/examples/nemo/scripts/compare_ouput.py index 8760adcb33..29ee8a83cb 100644 --- a/examples/nemo/scripts/compare_ouput.py +++ b/examples/nemo/scripts/compare_ouput.py @@ -34,17 +34,27 @@ # Author: S. Siso, STFC Daresbury Lab # ----------------------------------------------------------------------------- -''' Script to compare LFRic output files. ''' +''' Script to compare NEMO run.stat output files. ''' import sys import os +import math -TOLERANCE_DIGITS = 8 +TOLERANCE = 1e-05 + + +def is_float(x): + ''' Check if the given value is a float. ''' + try: + _ = float(x) + return True + except ValueError: + return False def main(): - ''' Compare the two provided LFRic output files, it checks that the - Conservation and Residual values are equivalent (within a tolerance). ''' + ''' Compare the two provided NEMO run.stat output files, it checks that the + values are equivalent (within a tolerance). ''' # Parse input arguments if len(sys.argv) - 1 != 2: @@ -64,10 +74,10 @@ def main(): line_f2 = file2.readline() while line_f1 and line_f2: # Get all numbers (rhs of each :) - values1 = line_f1.split(':')[::2] - values2 = line_f2.split(':')[::2] + values1 = [float(x) for x in line_f1.split(' ') if is_float(x)] + values2 = [float(x) for x in line_f2.split(' ') if is_float(x)] for value1, value2 in zip(values1, values2): - if value1[:TOLERANCE_DIGITS] != value2[:TOLERANCE_DIGITS]: + if not math.isclose(value1, value2, rel_tol=TOLERANCE): sys.exit(f"The values are not equal:\n{line_f1}{line_f2}") # Get next lines From b95a2c24d15a7b86441d2aa43398f3e22dc0ad25 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Tue, 3 Sep 2024 15:21:56 +0100 Subject: [PATCH 13/16] Improve aspects related with NEMOv5 integration test --- .github/workflows/nemo_tests.yml | 23 -- .github/workflows/nemo_v5_tests.yml | 47 ++-- .../nemo/scripts/KGOs/arch-linux_spack.fcm | 7 + .../nemo/scripts/KGOs/namelist_cfg_orca2_like | 226 ------------------ ...elist_cfg_orca1_like => namelist_cfg_test} | 12 +- .../KGOs/run.stat.bench.orca1.1mpi.10steps | 10 - .../KGOs/run.stat.bench.orca2.4mpi.10steps | 10 - .../scripts/KGOs/run.stat.bench.test.100steps | 100 ++++++++ examples/nemo/scripts/compare_ouput.py | 2 +- examples/nemo/scripts/passthrough.py | 3 +- .../psyir/symbols/generic_interface_symbol.py | 2 +- .../transformations/parallel_loop_trans.py | 3 + .../tests/psyir/backend/fortran_test.py | 2 +- .../parallel_loop_trans_test.py | 14 +- 14 files changed, 155 insertions(+), 306 deletions(-) delete mode 100644 examples/nemo/scripts/KGOs/namelist_cfg_orca2_like rename examples/nemo/scripts/KGOs/{namelist_cfg_orca1_like => namelist_cfg_test} (96%) delete mode 100644 examples/nemo/scripts/KGOs/run.stat.bench.orca1.1mpi.10steps delete mode 100644 examples/nemo/scripts/KGOs/run.stat.bench.orca2.4mpi.10steps create mode 100644 examples/nemo/scripts/KGOs/run.stat.bench.test.100steps diff --git a/.github/workflows/nemo_tests.yml b/.github/workflows/nemo_tests.yml index ba55787adc..bbcd2bf468 100644 --- a/.github/workflows/nemo_tests.yml +++ b/.github/workflows/nemo_tests.yml @@ -82,29 +82,6 @@ jobs: cd lib/profiling/nvidia/ F90=nvfortran make - # PSyclone passthrough for 5.0-beta of NEMO. - - name: NEMO 5.0 beta passthrough without optimisation - run: | - . .runner_venv/bin/activate - export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts - # PSYCLONE_HOME has `/bin` appended to it by the build system. - export PSYCLONE_HOME=${PWD}/.runner_venv - export NEMO_DIR=${HOME}/NEMOv5 - cd $NEMO_DIR - module load nvidia-hpcsdk/${NVFORTRAN_VERSION} - module load hdf5/${HDF5_VERSION} netcdf_c/${NETCDF_C_VERSION} netcdf_fortran/${NETCDF_FORTRAN_VERSION} - module load perl/${PERL_VERSION} - # We compile at -O1 to permit comparison of the results. (N.B. this test - # passes at -O3 with the Intel ifx compiler.) - ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH clean -y - ./makenemo -r BENCH -m linux_nvidia_O1 -n BENCH_PASSTHROUGH -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ - del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 - cd $NEMO_DIR/tests/BENCH_PASSTHROUGH/EXP00 - mpirun -np 4 ./nemo - diff run.stat ${PSYCLONE_NEMO_DIR}/KGOs/run.stat.bench.orca2.4mpi.10steps - export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - echo "Time-stepping duration = " $VAR_TIME - # PSyclone passthrough for MetOffice NEMO - name: NEMO MetOffice Passthrough run: | diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index 634c55ba9a..b4ce53049f 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -34,7 +34,7 @@ # Author S. Siso, STFC Daresbury Lab # This workflow will use a self-hosted runner to perform the more expensive -# integrations tests that are not run on GHA systems. +# NEMOv5 integrations tests that are not run on GHA systems. name: NEMOv5 Integration Tests @@ -93,10 +93,11 @@ jobs: # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg mpirun -np 4 ./nemo - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat - cat run.stat + tail run.stat + # This was produced with gfortran, so we can do an exact diff + diff $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -111,10 +112,9 @@ jobs: export NEMO_DIR=${HOME}/NEMOv5 # Set up FCM: PATHs are loaded from SPACK, we only need to set the FCFLAGS - # We compile at -O1 to permit comparison of the results. cd $NEMO_DIR cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm - export FCFLAGS="-i4 -Mr8 -O1 -Minline -Mcray=pointer -Mpre -g" + export FCFLAGS="-i4 -Mr8 -O2 -Minline -Mcray=pointer -Mpre -g" # Clean up and compile # Without key_mpi_off it fails to compile (even without psyclone) @@ -124,10 +124,10 @@ jobs: # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - mpirun -np 4 ./nemo - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat - cat run.stat + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + ./nemo + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + tail run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -153,13 +153,12 @@ jobs: # Run Orca2 cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - # It currently segfaults - # mpirun -np 4 ./nemo - # python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat - # cat run.stat - # export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') - # echo "Time-stepping duration = " $VAR_TIME + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + mpirun -np 6 ./nemo + tail run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') + echo "Time-stepping duration = " $VAR_TIME - name: NEMO 5.0 gfortran OpenMP for CPUs run: | @@ -183,10 +182,10 @@ jobs: # Run Orca2 cd $NEMO_DIR/tests/BENCH_OMP_THREADING_GCC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg OMP_NUM_THREADS=4 mpirun -np 1 ./nemo - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat - cat run.stat + tail run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -214,9 +213,9 @@ jobs: # Run Orca2 cd $NEMO_DIR/tests/BENCH_OMP_OFFLOAD_NVHPC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_orca2_like namelist_cfg - mpirun -np 1 ./nemo - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.orca2.4mpi.10steps run.stat - cat run.stat + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + ./nemo + tail run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm index db066ca63a..94943258f3 100644 --- a/examples/nemo/scripts/KGOs/arch-linux_spack.fcm +++ b/examples/nemo/scripts/KGOs/arch-linux_spack.fcm @@ -1,3 +1,10 @@ +# This fcm file is intended to be used with the psyclone-spack nemo-build-environment recipe +# which will populate all environment variables but PSYCLONE_HOME and FCFLAGS, which should +# be populated manually for the desired target. For example, using: +# $ spack load nemo-build-environment%nvhpc +# $ export PSYCLONE_HOME=${PWD}/.venv +# $ export FCFLAGS="-i4 -Mr8 -O3 -Minline -Mcray=pointer -Mpre -mp" + %PSYCLONE_HOME ${PSYCLONE_HOME} %NCDF_INC -I${NCDF_F_HOME}/include -I${NCDF_C_HOME}/include -I${HDF5_HOME}/include %NCDF_LIB -L${NCDF_F_HOME}/lib -lnetcdff -L${NCDF_C_HOME}/lib -lnetcdf diff --git a/examples/nemo/scripts/KGOs/namelist_cfg_orca2_like b/examples/nemo/scripts/KGOs/namelist_cfg_orca2_like deleted file mode 100644 index 9b700a464d..0000000000 --- a/examples/nemo/scripts/KGOs/namelist_cfg_orca2_like +++ /dev/null @@ -1,226 +0,0 @@ -!!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -!! NEMO/OPA BENCH Configuration namelist : overwrite some defaults values defined in SHARED/namelist_ref -!!>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -! -!----------------------------------------------------------------------- -&namrun ! parameters of the run -!----------------------------------------------------------------------- - cn_exp = 'BENCH' ! experience name - nn_it000 = 1 ! first time step - nn_itend = 10 ! last time step - nn_stock = -1 ! frequency of creation of a restart file (modulo referenced to 1) - nn_write = -1 ! frequency of write in the output file (modulo referenced to nn_it000) -/ -!----------------------------------------------------------------------- -&namusr_def ! User defined : BENCH configuration: Flat bottom, beta-plane -!----------------------------------------------------------------------- - nn_isize = 180 ! number of point in i-direction of global(local) domain if >0 (<0) - nn_jsize = 166 ! number of point in j-direction of global(local) domain if >0 (<0) - nn_ksize = 40 ! total number of point in k-direction - ln_Iperio = .true. ! i-periodicity - ln_Jperio = .false. ! j-periodicity - ln_NFold = .true. ! North pole folding - cn_NFtype = 'F' ! Folding type: T or F -/ -!----------------------------------------------------------------------- -&nammpp ! Massively Parallel Processing -!----------------------------------------------------------------------- - ln_nnogather= .true. ! activate code to avoid mpi_allgather use at the northfold - jpni = 0 ! jpni number of processors following i (set automatically if < 1) - jpnj = 0 ! jpnj number of processors following j (set automatically if < 1) -/ -!----------------------------------------------------------------------- -&namctl ! Control prints (default: OFF) -!----------------------------------------------------------------------- - ln_timing = .true. ! timing by routine write out in timing.output file - sn_cfctl%l_runstat = .true. ! Output run.stat -/ -!----------------------------------------------------------------------- -&namdom ! time and space domain -!----------------------------------------------------------------------- - rn_Dt = 360. ! time step for the dynamics (and tracer if nn_acc=0) - rn_atfp = 0.05 ! asselin time filter parameter - ln_meshmask = .false. ! =T create a mesh file -/ - -!!====================================================================== -!! *** Surface Boundary Condition namelists *** !! -!! !! -!! namsbc surface boundary condition manager (default: NO selection) -!!====================================================================== -! -!----------------------------------------------------------------------- -&namsbc ! Surface Boundary Condition (surface module) -!----------------------------------------------------------------------- - ln_usr = .true. ! user defined formulation (T => check usrdef_sbc) - nn_ice = 2 ! =0 no ice boundary condition - ! ! =1 use observed ice-cover ( => fill namsbc_iif ) - ! ! =2 or 3 for SI3 and CICE, respectively - ln_traqsr = .true. ! Light penetration in the ocean (T => fill namtra_qsr) -/ - -! -!!====================================================================== -!! *** Lateral boundary condition *** !! -!! !! -!! namlbc lateral momentum boundary condition (default: NO selection) -!! namagrif agrif nested grid (read by child model only) ("key_agrif") -!! nam_tide Tidal forcing (default: OFF) -!! nambdy Unstructured open boundaries (default: OFF) -!! nambdy_dta Unstructured open boundaries - external data (see nambdy) -!! nambdy_tide tidal forcing at open boundaries (default: OFF) -!!====================================================================== -! -!----------------------------------------------------------------------- -&namlbc ! lateral momentum boundary condition (default: NO selection) -!----------------------------------------------------------------------- - rn_shlat = 0. ! free slip -/ - -!!====================================================================== -!! *** Top/Bottom boundary condition *** !! -!! !! -!! namdrg top/bottom drag coefficient (default: NO selection) -!! namdrg_top top friction (ln_drg_OFF =F & ln_isfcav=T) -!! namdrg_bot bottom friction (ln_drg_OFF =F) -!! nambbc bottom temperature boundary condition (default: OFF) -!! nambbl bottom boundary layer scheme (default: OFF) -!!====================================================================== -! -!----------------------------------------------------------------------- -&namtra_qsr ! penetrative solar radiation (ln_traqsr =T) -!----------------------------------------------------------------------- - ! ! type of penetration (default: NO selection) - ln_qsr_rgb = .true. ! RGB light penetration (Red-Green-Blue) - nn_chldta = 0 ! RGB : Chl data (=1) or cst value (=0) -/ -!----------------------------------------------------------------------- -&namdrg ! top/bottom drag coefficient (default: NO selection) -!----------------------------------------------------------------------- - ln_non_lin = .true. ! non-linear drag: Cd = Cd0 |U| -/ -!----------------------------------------------------------------------- -&nambbc ! bottom temperature boundary condition (default: OFF) -!----------------------------------------------------------------------- - ln_trabbc = .true. ! Apply a geothermal heating at the ocean bottom - nn_geoflx = 1 ! geothermal heat flux: = 1 constant flux -/ -!----------------------------------------------------------------------- -&nambbl ! bottom boundary layer scheme (default: OFF) -!----------------------------------------------------------------------- - ln_trabbl = .true. ! Bottom Boundary Layer parameterisation flag -/ - -!!====================================================================== -!! Tracer (T & S) namelists !! -!! !! -!! nameos equation of state (default: NO selection) -!! namtra_adv advection scheme (default: NO selection) -!! namtra_ldf lateral diffusion scheme (default: NO selection) -!! namtra_mle mixed layer eddy param. (Fox-Kemper param.) (default: OFF) -!! namtra_eiv eddy induced velocity param. (default: OFF) -!! namtra_dmp T & S newtonian damping (default: OFF) -!!====================================================================== -! -!----------------------------------------------------------------------- -&nameos ! ocean Equation Of Seawater (default: NO selection) -!----------------------------------------------------------------------- - ln_teos10 = .true. ! = Use TEOS-10 -/ -!----------------------------------------------------------------------- -&namtra_adv ! advection scheme for tracer (default: NO selection) -!----------------------------------------------------------------------- - ln_traadv_fct = .true. ! FCT scheme - nn_fct_h = 2 ! =2/4, horizontal 2nd / 4th order - nn_fct_v = 2 ! =2/4, vertical 2nd / COMPACT 4th order -/ -!----------------------------------------------------------------------- -&namtra_ldf ! lateral diffusion scheme for tracers (default: NO selection) -!----------------------------------------------------------------------- - ln_traldf_lap = .true. ! laplacian operator - ln_traldf_iso = .true. ! iso-neutral (standard operator) - ! - nn_aht_ijk_t = 20 ! space/time variation of eddy coefficient: -/ -!----------------------------------------------------------------------- -&namtra_mle ! mixed layer eddy parametrisation (Fox-Kemper) (default: OFF) -!----------------------------------------------------------------------- - ln_mle = .true. ! (T) use the Mixed Layer Eddy (MLE) parameterisation -/ -!----------------------------------------------------------------------- -&namtra_eiv ! eddy induced velocity param. (default: OFF) -!----------------------------------------------------------------------- - ln_ldfeiv = .true. ! use eddy induced velocity parameterization - ! - nn_aei_ijk_t = 20 ! space/time variation of eddy coefficient: -/ - -!!====================================================================== -!! *** Dynamics namelists *** !! -!! !! -!! nam_vvl vertical coordinate options (default: z-star) -!! namdyn_adv formulation of the momentum advection (default: NO selection) -!! namdyn_vor advection scheme (default: NO selection) -!! namdyn_hpg hydrostatic pressure gradient (default: NO selection) -!! namdyn_spg surface pressure gradient (default: NO selection) -!! namdyn_ldf lateral diffusion scheme (default: NO selection) -!! namdta_dyn offline TOP: dynamics read in files (OFF_SRC only) -!!====================================================================== -! -!----------------------------------------------------------------------- -&nam_vvl ! vertical coordinate options (default: z-star) -!----------------------------------------------------------------------- - ln_vvl_zstar = .true. ! z-star vertical coordinate - ln_vvl_dbg = .false. ! debug prints (T/F) -/ -!----------------------------------------------------------------------- -&namdyn_adv ! formulation of the momentum advection (default: NO selection) -!----------------------------------------------------------------------- - ln_dynadv_vec = .true. ! vector form (T) or flux form (F) - nn_dynkeg = 1 ! scheme for grad(KE): =0 C2 ; =1 Hollingsworth correction -/ -!----------------------------------------------------------------------- -&namdyn_vor ! Vorticity / Coriolis scheme (default: NO selection) -!----------------------------------------------------------------------- - ln_dynvor_een = .true. ! energy & enstrophy scheme -/ -!----------------------------------------------------------------------- -&namdyn_hpg ! Hydrostatic pressure gradient option (default: NO selection) -!----------------------------------------------------------------------- - ln_hpg_sco = .true. ! s-coordinate (standard jacobian formulation) -/ -!----------------------------------------------------------------------- -&namdyn_spg ! surface pressure gradient (default: NO selection) -!----------------------------------------------------------------------- - ln_dynspg_ts = .true. ! split-explicit free surface - ln_bt_auto = .false. ! Number of sub-step defined from: - nn_e = 30 ! =F : the number of sub-step in rn_Dt seconds -/ -!----------------------------------------------------------------------- -&namdyn_ldf ! lateral diffusion on momentum (default: NO selection) -!----------------------------------------------------------------------- - ln_dynldf_lap = .true. ! laplacian operator - ln_dynldf_hor = .true. ! horizontal (geopotential) - ! ! Coefficient - nn_ahm_ijk_t = 30 ! space/time variation of eddy coef -/ - -!!====================================================================== -!! vertical physics namelists !! -!! !! -!! namzdf vertical physics manager (default: NO selection) -!! namzdf_ric richardson number vertical mixing (ln_zdfric=T) -!! namzdf_tke TKE vertical mixing (ln_zdftke=T) -!! namzdf_gls GLS vertical mixing (ln_zdfgls=T) -!! namzdf_osm OSM vertical diffusion (ln_zdfosm=T) -!! namzdf_iwm tidal mixing parameterization (ln_zdfiwm=T) -!!====================================================================== -! -!----------------------------------------------------------------------- -&namzdf ! vertical physics (default: NO selection) -!----------------------------------------------------------------------- - ln_zdftke = .true. ! Turbulent Kinetic Energy closure (T => fill namzdf_tke) - ln_zdfevd = .true. ! enhanced vertical diffusion - ln_zdfddm = .true. ! double diffusive mixing - ln_zdfiwm = .true. ! internal wave-induced mixing (T => fill namzdf_iwm) -/ diff --git a/examples/nemo/scripts/KGOs/namelist_cfg_orca1_like b/examples/nemo/scripts/KGOs/namelist_cfg_test similarity index 96% rename from examples/nemo/scripts/KGOs/namelist_cfg_orca1_like rename to examples/nemo/scripts/KGOs/namelist_cfg_test index dabc93656c..f926a1fae1 100644 --- a/examples/nemo/scripts/KGOs/namelist_cfg_orca1_like +++ b/examples/nemo/scripts/KGOs/namelist_cfg_test @@ -7,16 +7,16 @@ !----------------------------------------------------------------------- cn_exp = 'BENCH' ! experience name nn_it000 = 1 ! first time step - nn_itend = 10 ! last time step + nn_itend = 100 ! last time step nn_stock = -1 ! frequency of creation of a restart file (modulo referenced to 1) nn_write = -1 ! frequency of write in the output file (modulo referenced to nn_it000) / !----------------------------------------------------------------------- &namusr_def ! User defined : BENCH configuration: Flat bottom, beta-plane !----------------------------------------------------------------------- - nn_isize = 360 ! number of point in i-direction of global(local) domain if >0 (<0) - nn_jsize = 331 ! number of point in j-direction of global(local) domain if >0 (<0) - nn_ksize = 75 ! total number of point in k-direction + nn_isize = 64 ! number of point in i-direction of global(local) domain if >0 (<0) + nn_jsize = 64 ! number of point in j-direction of global(local) domain if >0 (<0) + nn_ksize = 30 ! total number of point in k-direction ln_Iperio = .true. ! i-periodicity ln_Jperio = .false. ! j-periodicity ln_NFold = .true. ! North pole folding @@ -32,14 +32,12 @@ !----------------------------------------------------------------------- &namctl ! Control prints (default: OFF) !----------------------------------------------------------------------- - ln_timing = .true. ! timing by routine write out in timing.output file - sn_cfctl%l_runstat = .true. ! Output run.stat + ln_timing = .false. ! timing by routine write out in timing.output file / !----------------------------------------------------------------------- &namdom ! time and space domain !----------------------------------------------------------------------- rn_Dt = 360. ! time step for the dynamics (and tracer if nn_acc=0) - rn_atfp = 0.05 ! asselin time filter parameter ln_meshmask = .false. ! =T create a mesh file / diff --git a/examples/nemo/scripts/KGOs/run.stat.bench.orca1.1mpi.10steps b/examples/nemo/scripts/KGOs/run.stat.bench.orca1.1mpi.10steps deleted file mode 100644 index 7045f1a205..0000000000 --- a/examples/nemo/scripts/KGOs/run.stat.bench.orca1.1mpi.10steps +++ /dev/null @@ -1,10 +0,0 @@ - it : 1 |ssh|_max: 0.1780546181233501D+01 |U|_max: 0.5387517646959111D-02 |V|_max: 0.3195000880447698D-03 S_min: 0.2996363302884726D+02 S_max: 0.3103618351360060D+02 - it : 2 |ssh|_max: 0.2375555622510219D+01 |U|_max: 0.5672848887562452D-02 |V|_max: 0.9513563140855660D-03 S_min: 0.2996923238430155D+02 S_max: 0.3103618349472616D+02 - it : 3 |ssh|_max: 0.2658346680662056D+01 |U|_max: 0.5630211204767677D-02 |V|_max: 0.2162910025457961D-02 S_min: 0.2996933670445712D+02 S_max: 0.3103618347584601D+02 - it : 4 |ssh|_max: 0.2934239216210058D+01 |U|_max: 0.5516025332200458D-02 |V|_max: 0.4239854427692400D-02 S_min: 0.2996944175392938D+02 S_max: 0.3103618345697813D+02 - it : 5 |ssh|_max: 0.3253769296534487D+01 |U|_max: 0.5465090550333113D-02 |V|_max: 0.5423381451649731D-02 S_min: 0.2996954616861731D+02 S_max: 0.3103618343811188D+02 - it : 6 |ssh|_max: 0.3571200856885356D+01 |U|_max: 0.5444890565595887D-02 |V|_max: 0.6882023017927434D-02 S_min: 0.2996965174924546D+02 S_max: 0.3103618341925338D+02 - it : 7 |ssh|_max: 0.3886813180848246D+01 |U|_max: 0.5388735309699873D-02 |V|_max: 0.9026603176296201D-02 S_min: 0.2996975724236724D+02 S_max: 0.3103618340039123D+02 - it : 8 |ssh|_max: 0.4196428552962857D+01 |U|_max: 0.5390563044559263D-02 |V|_max: 0.1096647624858080D-01 S_min: 0.2996986354950105D+02 S_max: 0.3103618338153352D+02 - it : 9 |ssh|_max: 0.4480769787036634D+01 |U|_max: 0.5329239329828882D-02 |V|_max: 0.1304683048921278D-01 S_min: 0.2996997010394977D+02 S_max: 0.3103618336267388D+02 - it : 10 |ssh|_max: 0.4762099422403111D+01 |U|_max: 0.5328304219510681D-02 |V|_max: 0.1549797648725442D-01 S_min: 0.2997007725053140D+02 S_max: 0.3103618334381900D+02 diff --git a/examples/nemo/scripts/KGOs/run.stat.bench.orca2.4mpi.10steps b/examples/nemo/scripts/KGOs/run.stat.bench.orca2.4mpi.10steps deleted file mode 100644 index c7b931126c..0000000000 --- a/examples/nemo/scripts/KGOs/run.stat.bench.orca2.4mpi.10steps +++ /dev/null @@ -1,10 +0,0 @@ - it : 1 |ssh|_max: 0.2324250795589936D+01 |U|_max: 0.5772501368032043D-02 |V|_max: 0.6196911175364361D-03 S_min: 0.2996910761041623D+02 S_max: 0.3102375296601515D+02 - it : 2 |ssh|_max: 0.3434809921730785D+01 |U|_max: 0.6632906923459561D-02 |V|_max: 0.2712336557360906D-02 S_min: 0.2996915264910223D+02 S_max: 0.3102375275240546D+02 - it : 3 |ssh|_max: 0.3918328912037654D+01 |U|_max: 0.7557907171659503D-02 |V|_max: 0.7630767639115136D-02 S_min: 0.2996920837574815D+02 S_max: 0.3102375253646906D+02 - it : 4 |ssh|_max: 0.4387913879430794D+01 |U|_max: 0.8831612368402865D-02 |V|_max: 0.1460045954743054D-01 S_min: 0.2996926422967360D+02 S_max: 0.3102375235579429D+02 - it : 5 |ssh|_max: 0.4825681433254806D+01 |U|_max: 0.7017251584865124D-02 |V|_max: 0.1862177770284584D-01 S_min: 0.2996932004710497D+02 S_max: 0.3102375217312339D+02 - it : 6 |ssh|_max: 0.5247678728139970D+01 |U|_max: 0.7809549291043862D-02 |V|_max: 0.2344253183953412D-01 S_min: 0.2996937612623072D+02 S_max: 0.3102375202261755D+02 - it : 7 |ssh|_max: 0.5620203458083161D+01 |U|_max: 0.6528224608683247D-02 |V|_max: 0.2890298565516065D-01 S_min: 0.2996943240632039D+02 S_max: 0.3102375187005024D+02 - it : 8 |ssh|_max: 0.5953675959141655D+01 |U|_max: 0.6536953163286566D-02 |V|_max: 0.3667525808892204D-01 S_min: 0.2996948879434852D+02 S_max: 0.3102375174693533D+02 - it : 9 |ssh|_max: 0.6228450145828211D+01 |U|_max: 0.5635293477865413D-02 |V|_max: 0.4329379927128271D-01 S_min: 0.2996954553070009D+02 S_max: 0.3102375162178034D+02 - it : 10 |ssh|_max: 0.6483453724882898D+01 |U|_max: 0.5701938594625860D-02 |V|_max: 0.4920016849737715D-01 S_min: 0.2996960233984670D+02 S_max: 0.3102375152368392D+02 diff --git a/examples/nemo/scripts/KGOs/run.stat.bench.test.100steps b/examples/nemo/scripts/KGOs/run.stat.bench.test.100steps new file mode 100644 index 0000000000..95b5c35d01 --- /dev/null +++ b/examples/nemo/scripts/KGOs/run.stat.bench.test.100steps @@ -0,0 +1,100 @@ + it : 1 |ssh|_max: 0.2336851764570087D+01 |U|_max: 0.7052149477800684D-02 |V|_max: 0.2308260467200877D-02 S_min: 0.2996908781150693D+02 S_max: 0.3101392942716721D+02 + it : 2 |ssh|_max: 0.3739164083094878D+01 |U|_max: 0.1029616821992987D-01 |V|_max: 0.9486960009862211D-02 S_min: 0.2996910922616945D+02 S_max: 0.3101392859195436D+02 + it : 3 |ssh|_max: 0.4179101131274851D+01 |U|_max: 0.1301524988138879D-01 |V|_max: 0.2226585559898513D-01 S_min: 0.2996913484029493D+02 S_max: 0.3101392781894970D+02 + it : 4 |ssh|_max: 0.4569875511150748D+01 |U|_max: 0.1401053780649201D-01 |V|_max: 0.3894932715115156D-01 S_min: 0.2996916048603851D+02 S_max: 0.3101392710567817D+02 + it : 5 |ssh|_max: 0.4796169575324639D+01 |U|_max: 0.1103849688785435D-01 |V|_max: 0.5247761995640531D-01 S_min: 0.2996918638309708D+02 S_max: 0.3101392644923621D+02 + it : 6 |ssh|_max: 0.4986687389489863D+01 |U|_max: 0.1061909566753376D-01 |V|_max: 0.6682435160313002D-01 S_min: 0.2996921212409725D+02 S_max: 0.3101392584662024D+02 + it : 7 |ssh|_max: 0.5137377532600958D+01 |U|_max: 0.7776206738812892D-02 |V|_max: 0.8230434351326746D-01 S_min: 0.2996923806059345D+02 S_max: 0.3101392529517780D+02 + it : 8 |ssh|_max: 0.5245041711882124D+01 |U|_max: 0.1114333312848225D-01 |V|_max: 0.9661364041991180D-01 S_min: 0.2996926384223953D+02 S_max: 0.3101392479253744D+02 + it : 9 |ssh|_max: 0.5162398664673749D+01 |U|_max: 0.1465335344080455D-01 |V|_max: 0.1139851405821525D+00 S_min: 0.2996929000744533D+02 S_max: 0.3101392433638144D+02 + it : 10 |ssh|_max: 0.5005955481222619D+01 |U|_max: 0.2023455937023107D-01 |V|_max: 0.1294762239325535D+00 S_min: 0.2996931617422945D+02 S_max: 0.3101392392447119D+02 + it : 11 |ssh|_max: 0.4855135547363155D+01 |U|_max: 0.2414868247481750D-01 |V|_max: 0.1381240863345169D+00 S_min: 0.2996934315861018D+02 S_max: 0.3101392355475714D+02 + it : 12 |ssh|_max: 0.4692755224833184D+01 |U|_max: 0.3065956316452254D-01 |V|_max: 0.1451002472720360D+00 S_min: 0.2996937014494064D+02 S_max: 0.3101392322533556D+02 + it : 13 |ssh|_max: 0.4536098100542291D+01 |U|_max: 0.3442772287760921D-01 |V|_max: 0.1495389977889906D+00 S_min: 0.2996938036344907D+02 S_max: 0.3101392293436327D+02 + it : 14 |ssh|_max: 0.4431563612482347D+01 |U|_max: 0.4081132586570636D-01 |V|_max: 0.1519115296349336D+00 S_min: 0.2996939059705999D+02 S_max: 0.3101392268007751D+02 + it : 15 |ssh|_max: 0.4427364591240526D+01 |U|_max: 0.4497061368218146D-01 |V|_max: 0.1527299906039677D+00 S_min: 0.2996941616404622D+02 S_max: 0.3101392246083734D+02 + it : 16 |ssh|_max: 0.4374572635856285D+01 |U|_max: 0.5019633620636200D-01 |V|_max: 0.1522475888777552D+00 S_min: 0.2996944173095707D+02 S_max: 0.3101392227509158D+02 + it : 17 |ssh|_max: 0.4341821484889612D+01 |U|_max: 0.5531555071540357D-01 |V|_max: 0.1517913760503613D+00 S_min: 0.2996946794573724D+02 S_max: 0.3101392212133833D+02 + it : 18 |ssh|_max: 0.4325752650484618D+01 |U|_max: 0.5984280097656534D-01 |V|_max: 0.1506951155150409D+00 S_min: 0.2996949416824957D+02 S_max: 0.3101392199813921D+02 + it : 19 |ssh|_max: 0.4301354130031908D+01 |U|_max: 0.6412034806212455D-01 |V|_max: 0.1492011360136977D+00 S_min: 0.2996952103824535D+02 S_max: 0.3101392189957653D+02 + it : 20 |ssh|_max: 0.4266486330341549D+01 |U|_max: 0.6846696076334181D-01 |V|_max: 0.1487061949929750D+00 S_min: 0.2996954790895234D+02 S_max: 0.3101392180101296D+02 + it : 21 |ssh|_max: 0.4228899683771108D+01 |U|_max: 0.7286106786441825D-01 |V|_max: 0.1482733050868238D+00 S_min: 0.2996957538213342D+02 S_max: 0.3101392170244917D+02 + it : 22 |ssh|_max: 0.4216167750032006D+01 |U|_max: 0.7942948840862415D-01 |V|_max: 0.1471684529331044D+00 S_min: 0.2996960281724479D+02 S_max: 0.3101392160388572D+02 + it : 23 |ssh|_max: 0.4194760562121319D+01 |U|_max: 0.8259229294750987D-01 |V|_max: 0.1458617371205514D+00 S_min: 0.2996963081714462D+02 S_max: 0.3101392150532316D+02 + it : 24 |ssh|_max: 0.4168713559873129D+01 |U|_max: 0.8868057756900001D-01 |V|_max: 0.1444473218199361D+00 S_min: 0.2996965875627752D+02 S_max: 0.3101392140676201D+02 + it : 25 |ssh|_max: 0.4134949051225507D+01 |U|_max: 0.9077057185821176D-01 |V|_max: 0.1439947172949023D+00 S_min: 0.2996968725371423D+02 S_max: 0.3101392130820279D+02 + it : 26 |ssh|_max: 0.4102570874661376D+01 |U|_max: 0.9385269294354093D-01 |V|_max: 0.1432368550039717D+00 S_min: 0.2996971569976720D+02 S_max: 0.3101392120964600D+02 + it : 27 |ssh|_max: 0.4087523300279630D+01 |U|_max: 0.9719584642584371D-01 |V|_max: 0.1422171247740371D+00 S_min: 0.2996974469969665D+02 S_max: 0.3101392111109208D+02 + it : 28 |ssh|_max: 0.4066711100039118D+01 |U|_max: 0.9981230002154087D-01 |V|_max: 0.1409251258433798D+00 S_min: 0.2996977366131380D+02 S_max: 0.3101392101254152D+02 + it : 29 |ssh|_max: 0.4041002208104996D+01 |U|_max: 0.1010000359259521D+00 |V|_max: 0.1393809648399729D+00 S_min: 0.2996980318329325D+02 S_max: 0.3101392091399473D+02 + it : 30 |ssh|_max: 0.4009285982003062D+01 |U|_max: 0.1032359453137443D+00 |V|_max: 0.1386229372478098D+00 S_min: 0.2996983266255877D+02 S_max: 0.3101392081545213D+02 + it : 31 |ssh|_max: 0.3982415222842285D+01 |U|_max: 0.1064909952799985D+00 |V|_max: 0.1376784018230090D+00 S_min: 0.2996986268498295D+02 S_max: 0.3101392071691412D+02 + it : 32 |ssh|_max: 0.3969982464349176D+01 |U|_max: 0.1090049443795551D+00 |V|_max: 0.1364874284015920D+00 S_min: 0.2996989265164330D+02 S_max: 0.3101392061838107D+02 + it : 33 |ssh|_max: 0.3951693237880511D+01 |U|_max: 0.1091781336121864D+00 |V|_max: 0.1350595740917304D+00 S_min: 0.2996992314600502D+02 S_max: 0.3101392051985331D+02 + it : 34 |ssh|_max: 0.3929024132864214D+01 |U|_max: 0.1108394851484137D+00 |V|_max: 0.1334223685706022D+00 S_min: 0.2996995357902092D+02 S_max: 0.3101392042133121D+02 + it : 35 |ssh|_max: 0.3902051104798720D+01 |U|_max: 0.1123215057057724D+00 |V|_max: 0.1324678559470795D+00 S_min: 0.2996998460938622D+02 S_max: 0.3101392032281504D+02 + it : 36 |ssh|_max: 0.3881823555864093D+01 |U|_max: 0.1135896712524620D+00 |V|_max: 0.1314603955531179D+00 S_min: 0.2997001560011671D+02 S_max: 0.3101392022430511D+02 + it : 37 |ssh|_max: 0.3867100883930083D+01 |U|_max: 0.1146836715283930D+00 |V|_max: 0.1303987239276822D+00 S_min: 0.2997004703037640D+02 S_max: 0.3101392012580167D+02 + it : 38 |ssh|_max: 0.3847949152864398D+01 |U|_max: 0.1158160520232005D+00 |V|_max: 0.1292381833357298D+00 S_min: 0.2997007844703269D+02 S_max: 0.3101392002730497D+02 + it : 39 |ssh|_max: 0.3825127641498751D+01 |U|_max: 0.1170177368085170D+00 |V|_max: 0.1279194616031653D+00 S_min: 0.2997010997053799D+02 S_max: 0.3101391992881522D+02 + it : 40 |ssh|_max: 0.3799528294348824D+01 |U|_max: 0.1186024141919507D+00 |V|_max: 0.1265811278328502D+00 S_min: 0.2997014149200218D+02 S_max: 0.3101391983033262D+02 + it : 41 |ssh|_max: 0.3772032775514555D+01 |U|_max: 0.1194298985387558D+00 |V|_max: 0.1257718604302258D+00 S_min: 0.2997017312118061D+02 S_max: 0.3101391973185734D+02 + it : 42 |ssh|_max: 0.3751883989161410D+01 |U|_max: 0.1202594108999530D+00 |V|_max: 0.1248426990485741D+00 S_min: 0.2997020485109384D+02 S_max: 0.3101391963338953D+02 + it : 43 |ssh|_max: 0.3733872363853277D+01 |U|_max: 0.1209428304251352D+00 |V|_max: 0.1238750010512143D+00 S_min: 0.2997023681279894D+02 S_max: 0.3101391953492931D+02 + it : 44 |ssh|_max: 0.3708689314338177D+01 |U|_max: 0.1216476445564216D+00 |V|_max: 0.1230482176287984D+00 S_min: 0.2997026888352095D+02 S_max: 0.3101391943647680D+02 + it : 45 |ssh|_max: 0.3675315817224827D+01 |U|_max: 0.1217187086074750D+00 |V|_max: 0.1223729447437161D+00 S_min: 0.2997030115697795D+02 S_max: 0.3101391933803208D+02 + it : 46 |ssh|_max: 0.3638628468354306D+01 |U|_max: 0.1220536369642546D+00 |V|_max: 0.1216146579240754D+00 S_min: 0.2997033350191469D+02 S_max: 0.3101391923959522D+02 + it : 47 |ssh|_max: 0.3603078747818889D+01 |U|_max: 0.1228390657960220D+00 |V|_max: 0.1209725239865341D+00 S_min: 0.2997036595291029D+02 S_max: 0.3101391914116627D+02 + it : 48 |ssh|_max: 0.3578024562485435D+01 |U|_max: 0.1233180522853033D+00 |V|_max: 0.1204603678388503D+00 S_min: 0.2997039840296358D+02 S_max: 0.3101391904274524D+02 + it : 49 |ssh|_max: 0.3552088235953275D+01 |U|_max: 0.1237554573374173D+00 |V|_max: 0.1198699312416498D+00 S_min: 0.2997043094517514D+02 S_max: 0.3101391894433213D+02 + it : 50 |ssh|_max: 0.3524980372696726D+01 |U|_max: 0.1252151996325486D+00 |V|_max: 0.1191536339279734D+00 S_min: 0.2997046348723887D+02 S_max: 0.3101391884592693D+02 + it : 51 |ssh|_max: 0.3496694919192748D+01 |U|_max: 0.1244822857127000D+00 |V|_max: 0.1183524274010442D+00 S_min: 0.2997049610934201D+02 S_max: 0.3101391874752956D+02 + it : 52 |ssh|_max: 0.3467275233288353D+01 |U|_max: 0.1243785561579632D+00 |V|_max: 0.1174596101044341D+00 S_min: 0.2997052873156215D+02 S_max: 0.3101391864913996D+02 + it : 53 |ssh|_max: 0.3437074898196145D+01 |U|_max: 0.1249843254911183D+00 |V|_max: 0.1165990177600824D+00 S_min: 0.2997056142115479D+02 S_max: 0.3101391855075804D+02 + it : 54 |ssh|_max: 0.3412808776859360D+01 |U|_max: 0.1262160905869304D+00 |V|_max: 0.1160025742906133D+00 S_min: 0.2997059411101461D+02 S_max: 0.3101391845238375D+02 + it : 55 |ssh|_max: 0.3391450350315669D+01 |U|_max: 0.1255493311068343D+00 |V|_max: 0.1153094788274209D+00 S_min: 0.2997062685752950D+02 S_max: 0.3101391835401706D+02 + it : 56 |ssh|_max: 0.3368770895797094D+01 |U|_max: 0.1257336544090955D+00 |V|_max: 0.1145337415699852D+00 S_min: 0.2997065960424352D+02 S_max: 0.3101391825565806D+02 + it : 57 |ssh|_max: 0.3353200034887146D+01 |U|_max: 0.1269656754817013D+00 |V|_max: 0.1136697423684477D+00 S_min: 0.2997069239915498D+02 S_max: 0.3101391815730700D+02 + it : 58 |ssh|_max: 0.3347600402412007D+01 |U|_max: 0.1272877934478984D+00 |V|_max: 0.1127229210566140D+00 S_min: 0.2997072519408028D+02 S_max: 0.3101391805896441D+02 + it : 59 |ssh|_max: 0.3437288728770171D+01 |U|_max: 0.1262930293680335D+00 |V|_max: 0.1116946626352186D+00 S_min: 0.2997071218253398D+02 S_max: 0.3101391796063123D+02 + it : 60 |ssh|_max: 0.3436080618231017D+01 |U|_max: 0.1273247312425994D+00 |V|_max: 0.1110159073289365D+00 S_min: 0.2997069923992022D+02 S_max: 0.3101391786230895D+02 + it : 61 |ssh|_max: 0.3498032084302758D+01 |U|_max: 0.1274936440026021D+00 |V|_max: 0.1102702437296978D+00 S_min: 0.2997073066598240D+02 S_max: 0.3101391776399960D+02 + it : 62 |ssh|_max: 0.3548875977055387D+01 |U|_max: 0.1287231634754152D+00 |V|_max: 0.1094421295742103D+00 S_min: 0.2997076213406375D+02 S_max: 0.3101391766570563D+02 + it : 63 |ssh|_max: 0.3611010538472270D+01 |U|_max: 0.1289986514796674D+00 |V|_max: 0.1085344229984168D+00 S_min: 0.2997079424665560D+02 S_max: 0.3101391756742955D+02 + it : 64 |ssh|_max: 0.3616484493253657D+01 |U|_max: 0.1293211256377026D+00 |V|_max: 0.1075492677366715D+00 S_min: 0.2997082645847949D+02 S_max: 0.3101391746917325D+02 + it : 65 |ssh|_max: 0.3700460333102999D+01 |U|_max: 0.1295006357484061D+00 |V|_max: 0.1064875051227157D+00 S_min: 0.2997085933317098D+02 S_max: 0.3101391737093726D+02 + it : 66 |ssh|_max: 0.3697987297630913D+01 |U|_max: 0.1291667138787790D+00 |V|_max: 0.1057037291354291D+00 S_min: 0.2997089223691964D+02 S_max: 0.3101391727272005D+02 + it : 67 |ssh|_max: 0.3717067730483875D+01 |U|_max: 0.1291720826499722D+00 |V|_max: 0.1049385987028414D+00 S_min: 0.2997092514369806D+02 S_max: 0.3101391717451805D+02 + it : 68 |ssh|_max: 0.3725017796413917D+01 |U|_max: 0.1287456717129999D+00 |V|_max: 0.1040924503724060D+00 S_min: 0.2997095805078223D+02 S_max: 0.3101391707632598D+02 + it : 69 |ssh|_max: 0.3783102705892638D+01 |U|_max: 0.1284629705168528D+00 |V|_max: 0.1031564825708730D+00 S_min: 0.2997099095103568D+02 S_max: 0.3101391697813832D+02 + it : 70 |ssh|_max: 0.3818433367959464D+01 |U|_max: 0.1282813749451166D+00 |V|_max: 0.1021781423168341D+00 S_min: 0.2997102385175010D+02 S_max: 0.3101391687995094D+02 + it : 71 |ssh|_max: 0.3851131306793331D+01 |U|_max: 0.1277861814682851D+00 |V|_max: 0.1009683893629114D+00 S_min: 0.2997105673789912D+02 S_max: 0.3101391678176254D+02 + it : 72 |ssh|_max: 0.3878317161616489D+01 |U|_max: 0.1270678368284661D+00 |V|_max: 0.9974557638529602D-01 S_min: 0.2997108962446472D+02 S_max: 0.3101391668357471D+02 + it : 73 |ssh|_max: 0.3899703715688805D+01 |U|_max: 0.1263371285216474D+00 |V|_max: 0.9756453661860603D-01 S_min: 0.2997112288329826D+02 S_max: 0.3101391658539023D+02 + it : 74 |ssh|_max: 0.3915016760780269D+01 |U|_max: 0.1269321248943728D+00 |V|_max: 0.9409623040708052D-01 S_min: 0.2997115614495198D+02 S_max: 0.3101391648721021D+02 + it : 75 |ssh|_max: 0.3942061473499172D+01 |U|_max: 0.1261161951122663D+00 |V|_max: 0.8894907704785138D-01 S_min: 0.2997118999289991D+02 S_max: 0.3101391638903134D+02 + it : 76 |ssh|_max: 0.3960769464011461D+01 |U|_max: 0.1264125511157165D+00 |V|_max: 0.8198090241845961D-01 S_min: 0.2997122384368348D+02 S_max: 0.3101391629084492D+02 + it : 77 |ssh|_max: 0.3972823012982603D+01 |U|_max: 0.1262003190368890D+00 |V|_max: 0.7274702218054171D-01 S_min: 0.2997125832933568D+02 S_max: 0.3101391619263892D+02 + it : 78 |ssh|_max: 0.4182767258105401D+01 |U|_max: 0.1254429168184836D+00 |V|_max: 0.6174855656923273D-01 S_min: 0.2997129280599455D+02 S_max: 0.3101391609440241D+02 + it : 79 |ssh|_max: 0.4464320492211273D+01 |U|_max: 0.1250107534504160D+00 |V|_max: 0.4938998002230323D-01 S_min: 0.2997132725935006D+02 S_max: 0.3101391599613026D+02 + it : 80 |ssh|_max: 0.4691659593199160D+01 |U|_max: 0.1249465581843477D+00 |V|_max: 0.4406556323374541D-01 S_min: 0.2997136172406285D+02 S_max: 0.3101391589782565D+02 + it : 81 |ssh|_max: 0.4847534239764846D+01 |U|_max: 0.1246216569432278D+00 |V|_max: 0.4490728485955739D-01 S_min: 0.2997139616869343D+02 S_max: 0.3101391579949843D+02 + it : 82 |ssh|_max: 0.4921368678320993D+01 |U|_max: 0.1250401706237382D+00 |V|_max: 0.4570676250134878D-01 S_min: 0.2997143063996485D+02 S_max: 0.3101391570116008D+02 + it : 83 |ssh|_max: 0.4909323017975378D+01 |U|_max: 0.1247182478822446D+00 |V|_max: 0.4646968203897572D-01 S_min: 0.2997146508443484D+02 S_max: 0.3101391560281827D+02 + it : 84 |ssh|_max: 0.4812117528475492D+01 |U|_max: 0.1246568340051059D+00 |V|_max: 0.5348530102148100D-01 S_min: 0.2997149953533280D+02 S_max: 0.3101391550447450D+02 + it : 85 |ssh|_max: 0.4634376427481938D+01 |U|_max: 0.1248504619935375D+00 |V|_max: 0.6513834069350445D-01 S_min: 0.2997149559614360D+02 S_max: 0.3101391540612583D+02 + it : 86 |ssh|_max: 0.4382675694681749D+01 |U|_max: 0.1252693521629411D+00 |V|_max: 0.7858657185884789D-01 S_min: 0.2997149162749577D+02 S_max: 0.3101391530777037D+02 + it : 87 |ssh|_max: 0.4208799088831883D+01 |U|_max: 0.1254101195033613D+00 |V|_max: 0.9208886037021863D-01 S_min: 0.2997152422294450D+02 S_max: 0.3101391520941244D+02 + it : 88 |ssh|_max: 0.4220457453582305D+01 |U|_max: 0.1254274875887708D+00 |V|_max: 0.1049689960927698D+00 S_min: 0.2997155681853749D+02 S_max: 0.3101391511106351D+02 + it : 89 |ssh|_max: 0.4224806935877820D+01 |U|_max: 0.1255654852969129D+00 |V|_max: 0.1157302555662952D+00 S_min: 0.2997153227260946D+02 S_max: 0.3101391501273761D+02 + it : 90 |ssh|_max: 0.4237697503344373D+01 |U|_max: 0.1260641626704890D+00 |V|_max: 0.1238266661856865D+00 S_min: 0.2997150772419434D+02 S_max: 0.3101391491444352D+02 + it : 91 |ssh|_max: 0.4269285515231942D+01 |U|_max: 0.1262749984156360D+00 |V|_max: 0.1290184649847683D+00 S_min: 0.2997153801826848D+02 S_max: 0.3101391480701454D+02 + it : 92 |ssh|_max: 0.4282296969944682D+01 |U|_max: 0.1268305620620326D+00 |V|_max: 0.1320767028793483D+00 S_min: 0.2997156828269546D+02 S_max: 0.3101391469044970D+02 + it : 93 |ssh|_max: 0.4293648162388765D+01 |U|_max: 0.1271524042492304D+00 |V|_max: 0.1327537509901237D+00 S_min: 0.2997159899144967D+02 S_max: 0.3101391456587404D+02 + it : 94 |ssh|_max: 0.4308927978726551D+01 |U|_max: 0.1268164306612873D+00 |V|_max: 0.1334577207469428D+00 S_min: 0.2997162964590055D+02 S_max: 0.3101391443314500D+02 + it : 95 |ssh|_max: 0.4315702895557181D+01 |U|_max: 0.1264933205160509D+00 |V|_max: 0.1340208119283705D+00 S_min: 0.2997166075040118D+02 S_max: 0.3101391429169621D+02 + it : 96 |ssh|_max: 0.4326461128621080D+01 |U|_max: 0.1263418695560262D+00 |V|_max: 0.1341896431904529D+00 S_min: 0.2997169183629738D+02 S_max: 0.3101391414146741D+02 + it : 97 |ssh|_max: 0.4339387118318213D+01 |U|_max: 0.1263028910790446D+00 |V|_max: 0.1340172539470082D+00 S_min: 0.2997172340478246D+02 S_max: 0.3101391398341884D+02 + it : 98 |ssh|_max: 0.4348288854361329D+01 |U|_max: 0.1263825890978068D+00 |V|_max: 0.1332936318973822D+00 S_min: 0.2997175497128216D+02 S_max: 0.3101391381939025D+02 + it : 99 |ssh|_max: 0.4350958876474419D+01 |U|_max: 0.1262222066500761D+00 |V|_max: 0.1323665655498839D+00 S_min: 0.2997178702549860D+02 S_max: 0.3101391365143150D+02 + it : 100 |ssh|_max: 0.4359024235826554D+01 |U|_max: 0.1261633139791182D+00 |V|_max: 0.1310216302821914D+00 S_min: 0.2997181908528601D+02 S_max: 0.3101391348098758D+02 diff --git a/examples/nemo/scripts/compare_ouput.py b/examples/nemo/scripts/compare_ouput.py index 29ee8a83cb..fcdd20bb51 100644 --- a/examples/nemo/scripts/compare_ouput.py +++ b/examples/nemo/scripts/compare_ouput.py @@ -40,7 +40,7 @@ import os import math -TOLERANCE = 1e-05 +TOLERANCE = 1e-07 def is_float(x): diff --git a/examples/nemo/scripts/passthrough.py b/examples/nemo/scripts/passthrough.py index 13ad293442..8eb7d9ca91 100755 --- a/examples/nemo/scripts/passthrough.py +++ b/examples/nemo/scripts/passthrough.py @@ -37,10 +37,9 @@ ''' Process Nemo code with PSyclone but don't do any changes. This file is only needed to provide a FILES_TO_SKIP list. ''' -from utils import NOT_WORKING # List of all files that psyclone will skip processing -FILES_TO_SKIP = NOT_WORKING +FILES_TO_SKIP = [] def trans(_): diff --git a/src/psyclone/psyir/symbols/generic_interface_symbol.py b/src/psyclone/psyir/symbols/generic_interface_symbol.py index c0427918b8..3ab39d354f 100644 --- a/src/psyclone/psyir/symbols/generic_interface_symbol.py +++ b/src/psyclone/psyir/symbols/generic_interface_symbol.py @@ -46,7 +46,7 @@ class GenericInterfaceSymbol(RoutineSymbol): :param str name: name of the interface. :param routines: the routines that this interface provides access to, - nd whether or not each of them is a module procedure. + and whether or not each of them is a module procedure. :type routines: list[ tuple[:py:class:`psyclone.psyir.symbols.RoutineSymbol`, bool]] :param kwargs: additional keyword arguments provided by diff --git a/src/psyclone/psyir/transformations/parallel_loop_trans.py b/src/psyclone/psyir/transformations/parallel_loop_trans.py index 94f9b32ea5..4bb06aa45c 100644 --- a/src/psyclone/psyir/transformations/parallel_loop_trans.py +++ b/src/psyclone/psyir/transformations/parallel_loop_trans.py @@ -108,6 +108,8 @@ def validate(self, node, options=None): :raises TransformationError: if the given loops calls a procedure that is not guaranteed to be pure (and therefore could have dependencies beyond the specified by the arguments intent) + :raises TransformationError: if the given loop is inside a pure routine + as these do not allow parallel constructs. :raises TransformationError: if there is a data dependency that prevents the parallelisation of the loop and the provided options don't disregard them. @@ -143,6 +145,7 @@ def validate(self, node, options=None): routine = node.ancestor(Routine) if routine is not None and routine.parent is not None: try: + # TODO #2596: Replace with routine.symbol rsym = routine.parent.symbol_table.lookup(routine.name) if rsym.is_pure or rsym.is_elemental: raise TransformationError( diff --git a/src/psyclone/tests/psyir/backend/fortran_test.py b/src/psyclone/tests/psyir/backend/fortran_test.py index ccdb8bbef9..8137536122 100644 --- a/src/psyclone/tests/psyir/backend/fortran_test.py +++ b/src/psyclone/tests/psyir/backend/fortran_test.py @@ -853,7 +853,7 @@ def test_gen_access_stmts_avoids_internal(fortran_reader, fortran_writer): psyir = fortran_reader.psyir_from_source(test_module) # Check that the internal symbol exists but is not listed assert psyir.children[0].symbol_table.lookup("_PSYCLONE_INTERNAL_test") - assert "_PSYCLONE_INTERNAL_test" not in fortran_writer(psyir) + assert "_psyclone_internal_test" not in fortran_writer(psyir).lower() def test_fw_exception(fortran_writer): diff --git a/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py b/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py index 2a2f69ab07..f5e36ba3ec 100644 --- a/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py +++ b/src/psyclone/tests/psyir/transformations/parallel_loop_trans_test.py @@ -123,6 +123,13 @@ def test_paralooptrans_validate_loop_inside_pure(fortran_reader): a loop inside a pure (or elemental) routine ''' psyir = fortran_reader.psyir_from_source(''' + program test_prog + real, dimension(10) :: var = 1 + integer :: j + do j = LBOUND(var), UBOUND(var) + var(j) = var(j) + 1 + end do + end program module test contains elemental function my_func(a) @@ -148,7 +155,12 @@ def test_paralooptrans_validate_loop_inside_pure(fortran_reader): ''') trans = ParaTrans() - for loop in psyir.walk(Loop): + loops = psyir.walk(Loop) + + # The first one succeeds as it is not inside a pure function + trans.validate(loops[0], {"verbose": True}) + + for loop in loops[1:]: # Check that we reject parallelisng inside a pure routine with pytest.raises(TransformationError) as err: trans.validate(loop, {"verbose": True}) From 31cd75382bc496cd7fc8e32ac975647e0f570560 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Tue, 3 Sep 2024 17:23:21 +0100 Subject: [PATCH 14/16] #2692 Fix NEMO test issues --- examples/nemo/scripts/KGOs/namelist_cfg_test | 3 ++- examples/nemo/scripts/passthrough.py | 3 ++- examples/nemo/scripts/utils.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/nemo/scripts/KGOs/namelist_cfg_test b/examples/nemo/scripts/KGOs/namelist_cfg_test index f926a1fae1..834f83b538 100644 --- a/examples/nemo/scripts/KGOs/namelist_cfg_test +++ b/examples/nemo/scripts/KGOs/namelist_cfg_test @@ -32,7 +32,8 @@ !----------------------------------------------------------------------- &namctl ! Control prints (default: OFF) !----------------------------------------------------------------------- - ln_timing = .false. ! timing by routine write out in timing.output file + ln_timing = .true. ! timing by routine write out in timing.output file + sn_cfctl%l_runstat = .TRUE. / !----------------------------------------------------------------------- &namdom ! time and space domain diff --git a/examples/nemo/scripts/passthrough.py b/examples/nemo/scripts/passthrough.py index 8eb7d9ca91..13ad293442 100755 --- a/examples/nemo/scripts/passthrough.py +++ b/examples/nemo/scripts/passthrough.py @@ -37,9 +37,10 @@ ''' Process Nemo code with PSyclone but don't do any changes. This file is only needed to provide a FILES_TO_SKIP list. ''' +from utils import NOT_WORKING # List of all files that psyclone will skip processing -FILES_TO_SKIP = [] +FILES_TO_SKIP = NOT_WORKING def trans(_): diff --git a/examples/nemo/scripts/utils.py b/examples/nemo/scripts/utils.py index 051e3d1e8c..04ef313fd3 100755 --- a/examples/nemo/scripts/utils.py +++ b/examples/nemo/scripts/utils.py @@ -64,10 +64,9 @@ # Files that we won't touch at all, either because PSyclone actually fails # or because it produces incorrect Fortran. NOT_WORKING = [ + # NEMOv4 bugs: # TODO #717 - array accessed inside WHERE does not use array notation "diurnal_bulk.f90", - # TODO #1902: Excluded to avoid HoistLocalArraysTrans bug - "mpp_ini.f90", ] # If routine names contain these substrings then we do not profile them @@ -227,7 +226,8 @@ def normalise_loops( statements out of the loop nest. ''' - if hoist_local_arrays: + # TODO #1902: NEMO4 mpi_ini.f90 has a HoistLocalArraysTrans bug + if hoist_local_arrays and schedule.root.name != "mpp_ini.f90": # Apply the HoistLocalArraysTrans when possible try: HoistLocalArraysTrans().apply(schedule) From 333ed991b59cce8e69c1d9aeaccc02f3a3f354a8 Mon Sep 17 00:00:00 2001 From: Sergi Siso Date: Wed, 4 Sep 2024 19:15:42 +0100 Subject: [PATCH 15/16] #2629 Update NEMOv5 cfg and runstat name --- .github/workflows/nemo_v5_tests.yml | 40 +++++++++---------- ...list_cfg_test => namelist_cfg_bench_small} | 0 ...100steps => run.stat.bench.small.100steps} | 0 3 files changed, 20 insertions(+), 20 deletions(-) rename examples/nemo/scripts/KGOs/{namelist_cfg_test => namelist_cfg_bench_small} (100%) rename examples/nemo/scripts/KGOs/{run.stat.bench.test.100steps => run.stat.bench.small.100steps} (100%) diff --git a/.github/workflows/nemo_v5_tests.yml b/.github/workflows/nemo_v5_tests.yml index b4ce53049f..663fd44a43 100644 --- a/.github/workflows/nemo_v5_tests.yml +++ b/.github/workflows/nemo_v5_tests.yml @@ -89,15 +89,15 @@ jobs: # Clean up and compile ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_GCC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ - del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca2 + # Run test cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_GCC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg mpirun -np 4 ./nemo tail run.stat # This was produced with gfortran, so we can do an exact diff - diff $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + diff $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -120,13 +120,13 @@ jobs: # Without key_mpi_off it fails to compile (even without psyclone) ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_NVHPC -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ - del_key "key_xios key_iomput key_top" add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 + add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 - # Run Orca2 + # Run test cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_NVHPC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg ./nemo - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat tail run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -149,14 +149,14 @@ jobs: # Clean up and compile ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_PASSTHROUGH_ONEAPI -p ${PSYCLONE_NEMO_DIR}/passthrough.py \ - del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca2 + # Run test cd $NEMO_DIR/tests/BENCH_PASSTHROUGH_ONEAPI/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg mpirun -np 6 ./nemo tail run.stat - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -178,14 +178,14 @@ jobs: # Clean up and compile ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_GCC clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_THREADING_GCC -p ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py \ - del_key "key_xios key_iomput key_top" add_key "key_nosignedzero" -j 4 -v 1 + add_key "key_nosignedzero" -j 4 -v 1 - # Run Orca2 + # Run test cd $NEMO_DIR/tests/BENCH_OMP_THREADING_GCC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg OMP_NUM_THREADS=4 mpirun -np 1 ./nemo tail run.stat - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME @@ -209,13 +209,13 @@ jobs: # Without key_mpi_off it fails to compile (even without psyclone) ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC clean -y ./makenemo -r BENCH -m linux_spack -n BENCH_OMP_OFFLOAD_NVHPC -p ${PSYCLONE_NEMO_DIR}/omp_gpu_trans.py \ - del_key "key_xios key_iomput key_top" add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 + add_key "key_mpi_off key_nosignedzero" -j 4 -v 1 - # Run Orca2 + # Run test cd $NEMO_DIR/tests/BENCH_OMP_OFFLOAD_NVHPC/EXP00 - cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_test namelist_cfg + cp $PSYCLONE_NEMO_DIR/KGOs/namelist_cfg_bench_small namelist_cfg ./nemo tail run.stat - python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.test.100steps run.stat + python $PSYCLONE_NEMO_DIR/compare_ouput.py $PSYCLONE_NEMO_DIR/KGOs/run.stat.bench.small.100steps run.stat export VAR_TIME=$(awk '/ step /{print $3}' timing.output | head -n 1 | sed -e 's/s//') echo "Time-stepping duration = " $VAR_TIME diff --git a/examples/nemo/scripts/KGOs/namelist_cfg_test b/examples/nemo/scripts/KGOs/namelist_cfg_bench_small similarity index 100% rename from examples/nemo/scripts/KGOs/namelist_cfg_test rename to examples/nemo/scripts/KGOs/namelist_cfg_bench_small diff --git a/examples/nemo/scripts/KGOs/run.stat.bench.test.100steps b/examples/nemo/scripts/KGOs/run.stat.bench.small.100steps similarity index 100% rename from examples/nemo/scripts/KGOs/run.stat.bench.test.100steps rename to examples/nemo/scripts/KGOs/run.stat.bench.small.100steps From 07fec7438cda8bedd37a135af2496982b650a892 Mon Sep 17 00:00:00 2001 From: Andrew Porter Date: Thu, 5 Sep 2024 10:34:17 +0100 Subject: [PATCH 16/16] #2698 update changelog and UG --- changelog | 3 +++ psyclone.pdf | Bin 1281146 -> 1281226 bytes 2 files changed, 3 insertions(+) diff --git a/changelog b/changelog index cd0b538b3b..86bacb78fc 100644 --- a/changelog +++ b/changelog @@ -213,6 +213,9 @@ 73) PR #2700 for #2699. Fix docstring error for Config class. + 74) PR #2698 for #2629. Fix NEMOv5 issues and extend related + integration tests. + release 2.5.0 14th of February 2024 1) PR #2199 for #2189. Fix bugs with missing maps in enter data diff --git a/psyclone.pdf b/psyclone.pdf index b9d98b38909fd1456ae4206e4b343dfdba685cde..4d4112326eaa2993a7c7c87ba6896264d9b422e6 100644 GIT binary patch delta 14296 zcmc(_RajeJ&@T)GcXy{~aZ=nJin~j(;83JRgHznyT}yGN1Sk|INO9NV4lNY;(*ED~ z+uG)XTh(-s6F z5)l&c5fTWk5&%DH1VO$GDIYZUj4D$A23Wi3W3r^(d(bx1jtaAq)f#*mhT~9mlVFUK?E!Hq%k$BJ^{(mT!=DRnu|DWkLCEW#TRc*UI{g?O zvL-yPRq@K-Ws@X7`h$f6 zBp$4UA3n%RzPkHFqGga*f*ZN+FC^Ui%?xwrTr4&-d0lm)<>n*pR+g__Nw553Rm_S5 zisf5pv}NAIa72c2>6%+4Mm=A!@Oav3$0rnYv$>Rgd!2M!V(w{vPDCChiaK2%g7j5$%vnnYk#SN(_Dd>&e6>rV)Ki(K`P_^JR!1kL$+>uD`x z;s|5*R9ZY2@7pm5JteCWc?9WAwd9xJG*X3FXsG@}dg=(fQp(ZYj|?8X=lz3Bbee#N z$J>GmSL9OMe!Pr6aUKHtKG{r!5L(8tJrxM5R4gZCKdhHqtGFZZwhwp6v0YlZc;vC&#>rZ$(Uq(1wr?#1luY8j12 zAkPuZ!3WYqXz5xFZhqqZJiRnSZHC7KT!!6QB;9{&WqObGoAV>#R&ykn=oMU~ZY8B2 z_bw($rSunc(~vK7{8*w*JOj_4t(xEvkgk~JnFRCE&Nkbr?t4j0Fy=1PJL2pBA|#Yi zw1RMJ7u254a8l-XXe9>0Q8CoxKqm|2(SA(Xx5p13oHOUP9Hud)>s^=Ru5lfoG$rXQ zj^Dxa_@b&?DPA!f)C^vOJrh`pU!fzN7qiyC2cd6Mu44XRThMGD;fkkgJ2$Qjk<;)+ zt<1Z!!I$QwdxCpFJSLHBR&3(cy(RIFckkvw@Dr%-2vQ_J@v>D&Hh(FI9ycm_j4c0R|IG*F-EOBs=?BvfN zg%+)m0-H_SpcXF_Lfx8Y1zz^HI(PczUgqV~-#OYeL1Z1;yqjB`4)V$Z$+_6F_cI@Z z(8hmsF1`vL%HDK1?a8C3?P1Z}Oq1V%rX*WW>*Ymd%-%egQ z#@v8%6aby%3e2s~)$=-LG8ta#z47(BF{YR_5}=4n<4ECkW=Tz2fkrK~!nAhyoJDW9 zMECVc#fF-qJoS!u$W1b<3odW!G*s)(w_gLf#&>ra3uvI)Zv7n9nzt$Zxiy2J`NJFg zWtq^c$Oi|P+Ik73$1vHh8QYpJ0ne%Mu@nxIyANmEzrZ)lH2hD_^aCP!hxA#lb+TvU zp<$?DfZso<{530jg=sLe*^%lOwH>zUxZl5*B>>yp@K#v9t@~Y`wb5bK*dScZ-V&=< zcW93gV~p!tmi6@9fOYBqKA3D)?kVnU@|*E9S5}6Y-Fom&;HF6Ew#z~YQ>-`o2C!jm zLI}}N?FsDAqkQWFU{6QxNu+pXvl(=Xb%ub!yBHwlR3Y6<;DeS<8nI%*Z5e!F=aX+{ z)|m@%=*qR?*En#ag$vualCH$v>oG>bfQBUvos!a>)%C?TEI9s-cZRg46oZwEHCLvg>Z+g)92^I|9l- z`l%di;Bg2(r7-69Bnmyj4Jy388rMD9tP8=XxK>HmAl74IhQhD70j4j2RmyK)GNgC_ zgl$RI03lX*zKqLJcs8(+p(~{Te&E6v+Qba&;Ce%L;si<5P0Dq|MI?kVcyFtpQVQRR zdGm>F<%4d|uZiApjK{{G3B5a(%0D`zH`J}&v6{}07MT3VHi=8b7-IQd`O_9|>Zgb% zmL!#nv|*-i%7hG@Y*O;lyz5q=@k7+G%+eCkuXZ$PicH@}Q&juFG@srnLyGfrSv&Bp zhkq;kXpFFuu+Wz&V*12q<}KpJsHu-=1l`aDsb)&b;*K1*S+Il<>F}f)9NZev@d5Eb zxdcEwH==3}8;#=YnrzNv%Jq#A6a<3s%IuFIJjQn{&7U07QR}`r5c{iFafZ1rEd+lY zZ_ui@a}Prb3C*ejBVdkxFC?rQ^a_SVFxG6IU{wfX?3h-CxN-LprNtnLBzq#tWYc!K zKtx#K9V^q(QbIzk%g7P*@bx`O>f5AiSArPBtHqREXhWrLSa>V-Wc@(Uvk($)-FN`{En5){8b9&F)ArDaqvX1IAi?`H{*xJJM%5)m8XeCkD(Bj@J6R1HxH~ z^@!AbQRMjK*44<$N9{i((ejULf;YLcU~D9-1igT#N#_OrZG0`C)%pAWWGUv!-Gq?% z@gGY~09{X3>~ARDQu-4kkbGP!aTN2pk@s-AM+#w4T*4f&x}-;8lz2kAIk6V~Y#HQ< z09J6o{l?wZ(MjZ4iqF9X{ksLN%L;`|>xgOcfVV^Xv-Xio>uY=H-MkeF8{pnXCmTVB zz@N4mr?r#&YB}cF<~cN%L4_i^@SMc^RK;_=jc+lhtCR3D)9Y0T8zzx1@>X-Xxb3bB zY7WgOYIb%yrLT|Px&KhjN!h;_=snt4pdkjw9RL2H>ak=xetLVlEhEsfb`mf!v3un2 z&EMtx+?_Qq3;RYXF>IAF!XH)nD_jT>9e0z_RG@?&M%5wwk1sF+;tVE3UxmE z=;Fl?RVwRm`v!tPXlWE&w>U!mQHf&`DQso&!^G_KF5j&7Q_|$@zv+%dT+xW^ju=g4 z?LIF^L~W|rBUnwsg^n@rz7}2cc$XdqF%+jGtMLCGg_N^lyEdp4BbMQi1fT{}nc2;W z5<26O+s3XNMk0z1e`0rhGm1jiQIEx-Cm+}p7{>N$sdzP3gVD;A`jGc^&QteTt;j+( z;Pm7bA&oFvc%DZ#LUe`4GA~V5b|SU9yAI8ncCKAb+NY+b%81GuetON4z?H(}?&>>H zo{8SBw!A(ye=%3b;>M*R*Kywr}Mj2JloyD%^01IAg zQ>Rq{ec^f~MoGZAH?IY}Z0P6R@e8VV(RC$MWxDp)Nzu^A&rMft|f4QjBc} z+k-Zmsjd-x4bjWf`@=u`51l!d?W0WyLapJ-wT!5b6HppRexpI(kVX)Ezf_Ud<6&`+ zS)a1-)KnALP+$d4Wj$d+K2U6A6Jh~xrogS$kUYR`g<9z#j>E^Bq{{H?Q=_ivR~blM z#8xO}M|@KL>(oeydN4IIU&dN!)*5eSR=VuwvntJj{|nPi*p2Lo!dkDASkk?SB5f~z zFV}G-x}fA2lb($}L&hYV z(W&5YFr7ecNVpi!*p&fOaV8QoRHL>PC>uSvWNjxkWY=MKD?S#*MO;xamABWL_7(2Q z3gE>+weU5m%s>TlS>*PM-b|3aC7)6YPFgi_-u^xsmk;rq(GB$1Z|tp=(Fr3tOorP9 z7xF}KkoPjFn({Yu#gorgs_+7-7Mn8GewWmE#6^Vg>tP2Bao6xN;?kJG$Hiwhr^bvZ z2Jp<1n??L*?8Cnb`>2;QX9A#wP6kScWyo>rL6+Ht7F^D5^tw#pHb_;;Kd-&(n!`o0 zKYiwLxYkfFvCK?LO5jUyrv2QPeEF=@wO@}r3~M^eGWZ@ z^^3L6pCPVP%{ETOPb9_~ldN>tKanUVpR{zUvOy@^Ea#eunQ#NQsFk)1@SOcg({eq8 zgGt=oi+Te}PXT`3;!7nHxRe9kmuA0l&&l<<X{6~PT@Wo=INqoD6| zSuQW+nu&__u`c|=BTn>_LIn>=G9(skVNJN0slYAV;P%H-ewTz|^%L1v<*k{0X42N0 z>_F^S+*~)z5?MG+>b_4wGmqh>oKoH@-Y=5AW6zepV=nhM>7#ThhHzSgRoEkMx3qvE zv>XpHv%vbhd6M{^2^4^1++`JH*&-UrXNqc5&&kO+;9eEHtH&OB${#R+Yvl@5yBZQh z!9`$P+Vk}u2z<@Iyu`Mb^m;M(zJ7G^^`Z&?Ga$Z+KJex2+boID)XGoSXn;fWKbViy zPY1vV5k^QbLWU6vj8I{O1|xJBVZaCzMp!Vyh7k^oaAAbkrk{?l+sF2XhntU=m-{t0 z4>LD6Gb=ims;jlErKb%oi>xp&H!qM|i2Hv`+UE5Mm@&0^c%Tf;ej~#N`;-x<75)D;F-7N6v2n5U1kv(v^9TV`wgAjv##}#7V@IF;Tcc!W z!}(UL@0#1+I)tK5S9iX-JHak~CFi6FdgFihZfwh))ux3wSkBHeEN|%n*@Zv1D8uv2 zVl<~3hG$6|hI8U_RCvyj0m%!?R9+W(vk4PszMpPZsRDXV{=C1Kay<XfA25Osj!MzJP z1$}_l5!?k+fpDO8ctd?^&5}2{w;2NPzmS}%85q}_&%N5X5UY+ z`6~O$JL!T@Fs{5?6)zvY1SA7W3A5`HYSwH<>*sN!K4o7<>UVd8Kc&o;b{37|kMhpf z;PS+CMhB~Q&E#~4dE=c@7Kv}#0QXE>Wm5P;y{Vj2H3&pvyh&7Pq?EE0v;)-%*oqdQ z)Fp>IHKZ&JJCj@lWaGPlB+@J3!fL zY^^ZC2SgR7;|B{FIh~3wTn1Kb$}pZ@MmubT-HwZlHu)hGCg{mDfdtUnf^9#y*s&(= zV)dKVQ^g@DnK7RLM4&h13#dC5%*)ixNd2aTG^e!7qFXNYD9*Uf@osP@a#pu$llW(V zNAR>uu_>b{9Cq>aw~>Dd7ssS#Mkk6HUkFo-g~?cus*e24INPSSg-}oiYC`Z1h_3lx z&XLFEbt`^E&z09+FGrmU(gf}~Ctt7&MiR^A&{6^praPgWxbdMFAfUHa5r4jHNkhs3 z+O0In^qj?V+WIad@Xex-G&le<26Eu;!BU;X%)a7Lu8eY?{8&2rE`LrzLu-H9U_7M) zEe+}jVsBOo@FyC;0!~~jNfT*#=u^FBcTrN!i@xB`-rxK$^TfZDn+}3;RNn-qYQ!~# zl#}a^o`s@-9w-OGaQfJ?kCYDs5$B;g`UO*m(udytEr;YAog+Pe>tUSh+Af?W zqGutirXOlaAwisL`L0VpsT=f{t@0g#cd!c(yCJ0u6?RBX9|#3kmCs(CAgxTd77e03 zLa>2f!@s&Y>qf)z$^TuGtz>n*|dpdmbEEkC?Kwe+!Emb#+xB{H~x9zAdCC;$a>IR$@Up7JfRN4`_jah(^ z;_gpA6Re})Kr1C&<1Xk7 zo15eZXb$L)|AiiE4goQx1OFrB3liz3Kv-L>Z6T5>fp@~+!}?Qqk&Zf0NEEk0v~JqZ z4~-u>M?c5@ds>>9>(2cwpmb+&bAM53n<6Oyo4n@phnRW{&ys1+b=7|7LXK}{&u@k!gWmVk9SNIRk_BZLb zSl!6ubk}p}MFH~D1{&Q;{Y{&coQ0Qro2+v~DIka3>oDJ< z7?8iv$A9V7OxElX_7IdY{cGvq&>8gJ777#M@EzXS{nIe8uwH}MKZ4ior5SI?$uaUP z6281^4;~6R$a|$jwtBJ{th-ASU6=xiMb&5PwtFLowwgcuORyol@QTzm z*vaWZBhY9URd6V5UfAV9pnfx6WOzEhgEd!(5ofFJc_B75@#3(}7_-1oX=;G06yoE&PAn zasStSrsjX$SKq+!^1q25x|yIS_Qbp&(lkU0muCNf_m1p+-Tws^r=6pySej+Gt~1l3 z$+lH*O6+(YHi3o_P`z|Apkim2{bSrzo=+0Co{3ezFe#V-{%zG~N{9+saPO--PV1+>SY z>^!-#9(hs}2XA7I55aCEg-h3e@b)GN&X9qA`ACC#43u#H39wERbSH`t?U!}|2ZmVz zYei%PQDG{q*@V3LwkDvOVITVxqj_UVD|b3;T9Qp z^pNzv8h$>~0aWnHjT!Yj#0N;sa*yz@u5NF!{moC`ssGUL7we9LzHdqfRPdeW6dw2p zU?#6kaQrVp1-0B1(B3pOf&62-9#EA>MYE${fJOMeSgM0x{$*$Rwgjy0>)y`+@#6dnUlT&TK}Su?xlnHkD5BO%*>~BdeA0eZS4YGY2B&URL{|AAp;T<{L$do`LTM0{& zpYl6xiCr;n6hEbQa%R_*7BN~&stk6*0xVF-Up6~M!2i%Mu2QvmHV_zi-@bx&8*x|2 z3cuVVK5hmKkh#Hipua0Qq>zRs;zwN7*I@O?WYAp%v61Qlnq-qeGt?V5A9j$TG7|b#f`GpxW8VEug-foJ$Pz^+W$s|i zT!kIx=oRIErL<{O7fmHd3HUtbCKIN;Nly+I?)V3RaU}?C;oR_?CGI+#)i-rPWIztW z|1i-1l}?w31$7)l_&264g!Tw|G3Nh3@JMJ2Zym7>nt2fR4+AxKqL{-zKwxz==$lnP zqZ>|~%)vKg@D?vft!u!(w(gtoR=@>ZTsV7!{4d~OVxP?3#Qo0#EE^$Ie zPVBniE9wiV;2biTi4{dF?gij}mA{1+MVS(RgjNKHTHsP5z?u(k#E3prLG<|GolQhR zG_~N_O-xcG*h3dL@?6q`sjw}Wd-eNhAvn7(&Ln3F;Sz}DTA*^q(r6z|{ls$@JqEP~KXUQPpKB@%pN0&|+) zt<0c2oP-4?lI-+&OT74i;yiBoKE!M|>4|b_>VH^nOoybB3n$09UugbNhFd>;kL_$z zu6F{86lt@;=8mQPpH#JrxXeX`MP$K>$=(`g_sP=lLX6Z0CO+Li5DI)U@`vs{Lkh~! z)U>Wi9F4n4JPixM+*PWAj?@RKAN{+Ko^Jp%4{#OVc5=_-1>SB>eXWI8`}hj`TG1ul z_9@^Ng&caeXPbm1x1I_h{^e^{61qv6NVbQVh_pwTcxjCpYp_Wpa1+aCK{+hXi#3_E z^fLbwu^pW=%WEw$9Zo+;h0a5=H36v}(RRZ7sg*zdUdA>7oDPfu`Vc)f#0~w zj)LI)#lIs;KX{=x_72s}h5suyb%>vB=t1+rqeWRq!xY~-Tp)TkTp)Qj>ddk(zMB#M zbE?M;uj#Oj6q?i4=T9{ty@~iqSozNdTq#!O=Rmm?O%&rs`C$wmgxzEAmQ4q04pxR_EHNBM78nPRd7-9- zZtarPL1MKqjuU{fH5g+SLd;n2MWvF{^Q$C5PR?lr{J~@3#JR03=@-Xu7sZ|VfTj=m z(?RhZyKS^$#uT>ax<8!7kY81+81PHBj#}4@uev&CKMmo$A$ORsu24|%%d+R3mlgVOz0_lx&8$WKn2Nih z3#M;9I}Y-bUix)N;OjA0DdN%hhOqq$O*>4u;sv!{7bF}x4_nZEM^R!OqmeRBvv0YI zHf%9n8F@vA@-2HwfJ>Rd$3}?O!~|jVyH%5`7J+IAx)C_VnG4%;-0~{AdkI)r$lCO4-&9IoP{V>v^4j~iTrRUU_Mv4gRezk zK!HOJPLc}Zu-N5>U~=;!*2Bfd!in9JZhF29qq)r(J*+f8HoM7}Z{;H8R7cRGvCJQ0Ak|ql|H=jl+$Z=CR6NQAGlyc5P$XMb%l3RS>E?Xv& z$N#Bbe~VW@y+vQVIrKRlB36&72a%~qrw)EokIfI!zJP@E^p;S;5H?5|0#K@tO6Z5H znm}g%xQjbA{LWE2RVSB0|MCa@n)zPlXWJ{;Dbf4&x^;JHG+LGk+EgT7ELrz?Xd*0F;y}he^?8_@X2%5*cdKVJ^gAV`Rt?{ zso=3Q|81}}YYe+h5-e`2T)ObE7K-5!?SG-8rub){vWZWF8i$h)Pa=@ae&OKRA_myTx9K_jvdDKZmzI+Ggy@iRrNBAmEc_#B##DCn{kQLtt zkSyBvB?L&5w`elgco7E)V@T%ZE_?uxcdfhjsS`ExM15%zoin69vTjl9JdR|~eBpDJ zPCkCsp;mk}u^VvN>mWHv)AO3wRKlfi<`D?#056685r5UpsgS@fTEX$Fg~M5zXU=A; zi2jK8Eb*-KM7Xp|t^61kUQJ~x(JnG~VQ-Xf#Ounq*SZfJ?MPtgQTa*uwM?GMf*r@N zs}DfsJ6NnqHVx+4+hgMBKdFSgx;TY*Wxgs^^IYYuDcDUO}V)FMbJTJMgKt-KLkQQjiCu9TaqY14jQ5ra81N5UD z;_PZA#|F2XVCrxw3RFYT?{LDOiZ%1kl6~^U2{qL+&ZW zxiR?p{Y4(IB1L(EAi{prDPV8Gyod$93xco5xiS73*IS`_7Xq-rJ!#W;n0n)eO9>yz zp8C5K_>7{KyEV-%h#tXy^I8Nbj1V81vY8n!ZZHQyD^&L(&TWJ$^p?P%IuOPNUvDA^ z+7+n>nB~NV{xA5OAQKQjsGH9Zp>YH15d6)jtZsatqr<-neo+C$H)AJd=ZAPUdS!Lhb4G6vq>0y`W5|Crev8#F2|};Y zvead(Reqx@!rpx@2X%3R12)GQ_j+jgnOeS=Cn01q* zP^o0j=$5dsy;?^*ZBFmDs45pWg_!-;go1B`w_La`GaT4Uw=BE*faA}Rs{_GkEo?H@ zx+6x6-om>YEfd}78fACg0|*gDuTYt$52g)mZrg>(1`F1AW9f5APHBfq-?jd8SQJ8& zTmHiSYC=`7k^bKGI?bc%P6U=Zwn5#pf1w?!wS;t|%au)a7uf$KO&xaV7^F#8iw_c3 zkb|$-|2#@?;g~~~*ROhSSWtHtFaSbV*@L&gd!dyoe(yKr*#8Vlqz{1~&{oY}o6_2D z^$t5o#&MOYdL;i#Vw8qGfio88^dEw5kTokpLKxH%G2 zIy!dKe#RMAQqse?Qa?l3ZL42f4t)OFni7?)LSh}2y5at%rRTncoN*S(hXTiV#Z|7< zy_!d=7@e0d-eBcB71r>%q8M<2TTjG!{l7-cc zmezBBjTJG(U%?pGEs}cwhmm7<Zc^Ln=i-gPsM+ZN%nAhS-@v<9Us|8djqFGl0Y=RoBL`$@5fyAngOX%1b2Qta*oh>;LWH$==}JU2p005e1hcCg3?5K=*$p*|$o zuZ(?!?FjOpdtAY7>kJgT*W%6IExgGgz6tLnimy$-#tDO1TR6~msURzO)v?i~dGq5|NYE4xNAQ zy9Wh%R>&g+Fa8`8r5|CP$t(tG4a`4x$vG=*Yg-G;(Ur-=XQ?F}68^`O@|x~4jBbJW z1@FK|TE6bQdGxbYh!Bq4VtTEQVcOOba(CLVgygohF@oMHYGSg^&5z5Hr&)LR+U#^{<@sX{2rSD!_cf#1!v=$~&hng@?q?+MFH$B*iCh>K8{Fh;jx8L=DUZZXwetulzL1T{!hRRWjUDKOXi{L|Tijf_>%|?KDfcQUiggx>WjHpCuceNVABP4NNE# zexg4d2xCB*|5ovDDSz2+xAf$HELA7m_DkXu`e9w-89G_*xMAr_oUcC2{C~#(S-$F! z-qX2wr$6i({zva0OEo9szLetl7Swnps8a*kemN@*uLCycVCNAl|yb$2m*8Xi8>9 z`_+3WPl``9YwrgAivkUp4ra7hMggB{m_IV9ZR=wzFm5_kQ+l4QQ@9&_!^#oOgfgh@xH zTg=$1-(g<4)C)g#CF8Glc_r+q2>8eghpq)RuLa5%8RIff; z-Lj}P&;=}+v@GTeg}3)d7}f5iNfdFTx`n$N$~&<*QGIz(#cB?o+$VW5Hs9CG@apb( z3s`K8fc+5hVBb`@eUO~u8e2>Ha9p(%EQ_B$=zPg36$XF-=$gt zfKw+X(fQV1NIzsl$Wxc}3G1{e{`>{Sx#BJ**bOlket;P3Pm%myohbZnfhf{$cIuP+ zTyc^ssH@2Wa~L3mG{n)l!$F=0mqIM&VZKG=HeDR)N2CS@EYT21p4$u+R|G1T=yNQC zelQ4;lLU!Ic+CG^GbT^`?2b2F`VnO~yAsJYUO^DILyMNb*(vXK>Zd*IVib0`fF#;x(qw)aLQnO^EoW!O0CtGmma^6ac?~?vhN!kIck3 zi;B4Gm%++NJS5D9C1zY)lE=C-bUbKq|0S-qgr8UNiZ>FzC5fzgF;4=@(%Oy5m`K*T zc=UFrG8r?!3)3d4h@JPB72kLE*785LFkN^mxX(PJNQZYvn6SoLfznm*{mwAq1*Px{MFJ_;u&V^kMWSfz;8wk9^iY}7k0kT9yh!$AJJZ+WA-%jz<)X?oP4?+`LU`y=Za3tuE zusMBo5N8(iR3)w%ieL{ti3^Mg_)*%R0UY*`K_2fRuK8LC?*kn1EXmsaLX*KH!4%~> z?5-3@ab-Aapp;C}sGSTQ!i#xYJH7#elUJd=3gZExNQHd}P}(&ty*C9pgUC^#R5?Y{{eQ zinl@CZ_cutdQ0JF)@Wz3u2x{PBz6zUc*MWhIZLAfZ_>W3y^gCBBJcx7$F+l{k|}b2 zQH9c4Ox_;DQWF?tD!K2&+RSG4D*T!{0_;FEs_Wvvv zfGk9Q0z@QXP5PwcMv>-{jLQ`W?u7q$Kduo*CeuaAilE! zkvjB^#}&8~{%Md_g2Z+MaY2x=eGW+1OFelL76@b^qW<`9k|a=X3hV`s)X(KrQ7NpL zeLa{#sKqF!i!BhT$Zm9x^R*0qw0O}$ZLff+6@E0*xJ7WadTP5Mh!Tt zQ@VI>yEPF+8fOTuc7EF$Vp8LIH))RZvm_sB4nu&)z1=Hn;PYEb7lUoLNc@VA>$xw# z$CfjFw;~v^+gRP(Rig$blqg-)w%y+1M;a#!%H_#J|G0xY{6hkFS6&>t#;8d+)1 zq_*7-@FGPL1Xo?ZZ3Qx5e;{*jANw@WBbQseIzFdy87#EXUU+{6?F?Zn-LCdJ?I}@6 zs6D<4qyB8T@Zfg~TI&qFYw~TjG(Gq0r50AP$A;n7}9 z+33DW{dgK&?)9?Yb@Q{z*>R(0XENy4B>~ZCCFd>rgsvC{~g4FTv+b3Vb z#{7P}TNWp=jfq3enp^*uBI*`H*Pr(}kB*Pt1XN{A@4!D~? z{L%BA%GWKW<)ij}+)U30&nBuhQ-yn^**M5ghTe{Y8k3Zzz3Z2{gwxdnHYFOaiYc!| zaec){L7#xh%isPB54qWwhaLs_cZVJ{p_S?%9&f&?e}}QDK}qopo`-g5fXPqs1^r)( z@&5D1rUl1A{_!3DCne$;yuSrI1L`<$elJWNRHGM+A^CTAibt-AS9}V1uD|a{h8`g1 zPwB#oXTbmR?|3l5+x3`kPax4=<;+xZ7{=Z;~|964Qkd94#s)Ux*U+6Ag;8SB_b2Btz324y;1h~-|8RgXF(f>bT CbpLt) delta 14226 zcmch+WmKF`@Gc00yAK*51P{)j!8H&p!QBEGAi&_xi@UqKy9Ee@;2vB81eYMegC*Dw z`Q88S-cR?O{jeXZPrubw)z#fR^>hy-=_-HIRO*(+@ZGh$gDC!x8n27teMQi3j8Ba} zLL&-%6bpS6Lxc;cQZVNueTAkM zWXn%noS-4{aBV2^I-Z#QmC9ioub2bMX8ldw?GXnvRTi3{`zsH=+`IFTdWq*GQ_N|( zC`$PB$>?Sr1lnwArQHrUXM5I)4y{Al!8zklw%))wv&LCj(d=-p@zaTDnboh*vs%Y|oqbrPs++fLQ)K8u;#{&lW z;}p=*)>`RG>^7T{$DSyu#MQ&*s+X&_sYQbuTHo*reD=B<>lHMfNz+13l4+?wn3mCM z#>UW4mkS?VH@u>9XP>n;j^=xImReLS zpB0>8aW5d(ZB&aTmqx+|fyQP)_`0tJv z@Pf=~C0MY4IYt)EjK`Wm-)5f$GbK9y9aR%PI<+eykd=ps+emyz_o{s)!s&Zl>tG4! z*=O_*hdxg>Z7X1WbRkAQSE(i8A3OmYa|3-*Ri{yl0KkU zoonf3^VK$tD8x`qn&eZT@*<*QEN@O0x=Gl_UW2COWTH+no^{dLuQ!K$9`Gg=0U}_+ zU6>%hK~^p5m(cSNtas(#i)*^HtCXd4XIOeSl23jjAY;turt@{5VSf(Xx>|megFRDo zYx%~W8+~X1qc7M4?+pm#{X$3fS#2Tf$5b-FHYssH;(d*4i7%}8V{M+Rd~hzGyx(N4 zA@-+j)ulWlO3(ZbIqZTgXCQ7{6{yn+-QCD34ohGLe~B&W8YyH6bA!#i*KB56?jv0@ z-C1>0Ly640;Lp37PSwS(55mT{Ao$7Y2Tdo@Y;G0vM|9~6WZ-Ht*Ho#m$3|sCjnx0Z zHIWN+Gz}VL?74|Wl*q&}uL%X4;Oo;-_d4|U+`V!PlTFwiwptB&SlUH(1-6>24Qvmv zaj#g;*UYzRux|b$ZrzKShYs&F?>>`ySFh+2tS1lPFt*(DdA=hf0ON&eB!s+{bdgNl?O7yt>8Lt2rQ<-sw%oBdY`V2(eq4HDc_p|pZ!!p$5zZu`mT zbNw$zi#Bhhl(TH_gHJZhGU=X4CutF{gi*=EH>&{W`#7_A15K1{f0c$%2sJJzev)Ug z?QY!lcB3(g29_!M@@W#_8rvI$tM|A%_D(Z)Z&W!b>}3;|6J@Ce)B(MTlUjyV{GU%+ zl@5Xt_j?S?B!XS7CMWGy-v!>c{5qBU+$oSu{mS0>ao6mFvn6I^K2;eU|n*j$s*gFHU z;x2!->9#(FV{P{;hU)|L9NWG%MyT504we@I+x)v7+~l5q*r;Wk?+)a7(_VdeDgInH zKlYVw{91g~5Zb8Lf@)`WT$D)P5i&Wl`ZDc%`ojFg<;$RWY^~;<9s!8XSnK-@f7j%7 z#@d!tB4-$%)rfT9xtAB8fTmzpppR6Zp-to53T0bq-_8^x$L&kN&Efi#=|te})3;MC z(zb8lqvWAa4srPYFnXFV0i3osjb|pa?^*o%snrSa`i6#|McgLH zJGi`iWtCp#Gc|)BnTu~-tbToO{^rceHGq7k_+HMNGQUt)l14zC8VHUgf$UE%k~QsWBI) ziD+r%ujQ6B+wls3RG=vHiiSu&yN<~H3l?K!aE+duH|Eyk%q^PibJWv^MXG&M9|Ni8 z4~kHenE|?S0bBBp>dtoaaeigl#~*{z6T=UxMA;M_qgZS5S_a(?yAL*E(cOAD?QXUs z@a@18|2lLv>cpqRc=+S-bdCCF=4mhw zZx$u7&pOk{dd{kd+#8dO-}{`I0O!M7y7cvz@ycPz%yI@A0Z~EkzYJD#m%hL^OZd8T zCURRQ!y}}X^AW&&kRVnE@n3E$4JXq|k86(vDyn>zAYhR-8*=J9=nK(t_D~D~72zR% z>_Mgrcx{x6B27@07m?rBRx^m3o*yo;m6aFnSE*+BGLX>Fc+Oe(`{{#<7`_g(%v_&z zBr`#+RAiZD__|`1W%uu-13C8;A0L8LAuY@+wwO9e7#HBA=_mPzEQ#(@s~!GA-FuR( zjtvcM)8&_%__&O%1&;OWG6`S1;~2w^DPgLLUDOp8Jq3BRy5%8$$$ccEC|T9`zqn1_ zZv9H}c_I-l+3TcQEho+bye*7Rccjk8GM!g2+K{i*X0^tGWRFQ@%#%coQ z6;5(og&jyU3CG4#I~wu(P|aU?v&3*ReTf;Rf&i(&s;?(=HM&Qbm^qb?ej|*$6P8QS}E2B3taN^$queU{h+sXFh)1xi!p<-v}a|kRW`I@>o{#TDyZ}Rd#6G-1so|z zW$$R{l)OdlgLF+@uBZrvG%49Mq|Q@eaY!sc$MxFP`N3i6aiYgA?1fa2#@#*vIDNx5 z`YXd$<$hDI^~td#ie36l;Yp|QVlxjFpSW}5sH>K?Q|TDzSnXJBtHfk>1p+D7Z`=dj zkYMvLt>~z(rXbsbFc%KLlkK10{(PB_2Fj1tM$c%zKg-Z2jjXg=%7S!u{?UB=`cS6> z7)*59WQ-o0+11*U(zKNyS)NeZF>3ENHZI!!-W_ZD3TdGM(=ZulyRB?fqIV?SGW{N3 z+vyZ2`Lp);=-1g@o-hXM*&loF_!sh`fH{I)a8^>T)h}2b=iK1}yvj2aYEqB>*;I~R zav^b^+_;zh+gZJp(Tm);b+Ut(wfQ_&EX3@!ap;YuJ0x`R_>-eAgIH>gh3XZh8gz`F zlb3oAhT~+eS+4XeMDnI~#UoJzL`ENGMTl+jDJ-K_bwg#ihr-xw!h5)0kT#q4jStz+ z$EpR>>nCLrRJ=bNePL+1Q!JEo&jchsV2|fxkR}6irC?IyoUgPWv0(C1e9z_GmD#lN5Bm>Cxvuj$h8$u|B%%f` z%CBr0KSa8Ig^t^SS8iRG#$g^Fk1;3?e2u?<#O|)V$6(HJzq)nyb@EvmJ8c_j1PL_< zD^@X}?~YB-9o4(FNHA=JJg zE7ut|P2)z^;L8mqPai5F@RnH=j%Y_dv?dc9ivLBn|5?fd{f>Cgl*?&RG;NVRFx^T4 z`S|3*Vf@-nO7*pB-$V87t8%7u-)Q9Xm(M9C9d)Ig6dClhdHTBMgB^$V(l|L>@-04o zBW)}-^8{?lx)^!{i2h=r?Il-UzY?sZZ`gz6B4IsB+3BEQd6}hH=rb0iq2R!nwd5)2 z`U?aWb>q%-YqUEi95*9nuP;6~E;mn;<^fcB)J0pu0u4pg;!B}D8F| zwAC~FB+o6;!#+W0ZvJGP+U{6Z(}l$U{RP2!EdemuVly#pa?)ST*O&NFDsr7ZZ|HE3 zMUPaAK(O!m9Y65L?>Wu$kS>?E*E~EesW*h9Idr;x)o6nLLx)z$I#mC3;#y+F1?#|dnm=uP2 zQUD0g%39(t@NvVVf~4bVy7J?^MATD)E)Wzv{pdwG5~xWP7Vr+(wq)~h_FEDyzQ|y_ z{?5r~j-cH>AY~r5zI!RH=089ONiJbUc~f_T`<5F(Iq9a5sgZ*ja$(S0XwSTP zeDs0n+VJJEr1eKOJe!Y7?cWy`R0+()dBe`)etQr@GfiBcXH(T-?D_B>(&dnZ;~@ZR z4e~ku)>@?n&%G+S`nI%r4Vak^&)>E8`|{%l;IcHuc~R1J+rb|*oIIgX9`EyNxy4BK zD3u8Q7zad|yifX`CeI~Y54$s!+a{%44kOtpyR4k%dePe+9zyQ(TyTR-F5E9)B!B?1 z*qGGy4|0Qf!!6ph{uSPgzH57Fiv})`{tX0U^%SPBH9>LL#4nNsrrsb~+24PzZ~0?l zQVpL>hf=AY_T)2?HxcD4jkWtpnm>#wy{@5OEyL>9l+OpWacPki8%75gac-*@pdTue zype7Sj@t1NO)zUx7Gk3&3a&4_7X_uk3b(ZbXg-b+1EMnMg& zO+kVSGF(vL^7M@c7j(E_zy%X7Sa89H3l3ax;erPje7F#_)TR(>b))fc^YQWV@TOwx z5K;ZlM+>J85fipLuYeFYuYdqV5F#KaEg&N)C@3T-3E`80NI|4{rGoT0SbTs&O7i5o~v07I4!OyA36_fjXpLU+F2eDLjNU-)n9SN1VS3krvVOq?oN zeGY_!Dw};O&f5mZC?IUk-4wcoo$4@h%P!U=qqoh@?H#?x*&Hxamh8wqyU^<9NO`2< zrXR^#!4($!JQ7nJ#Z^s7?SH-h?DBek?JRt;e=RbhDlfd|1o9uU0gOLMke&iSR?v8m+MSH}rs{D`0;OT_f3(3MjcOKNCer_MO%mT{-4%44BXyhb|3 zKO$Rt-lJIugt$^E7oVex$WcnZl1M|Q&hS*TNCEwV9(4NzjYZ%PXO22H7A2T|XXpfS! zC$YLJ2y{iMj)mK}=8xr1rOIY>EJVAK*uH|>ECUg)NYObQ@QAihJ?3TtYm-+QU`*SB z?5ljM`~}4_fhH{F{^FHiy@zu8GQSvSWhM_P-6k*QA>okQ0YW{}=eo?(hm7G@k#IsE_Kmk(a}O(F$#;x^+=fJa3XH3G2{%%JC^M1h7cB~iG`UJubi8g%?S(1= zDB16d7Dub0wn5Ii@s^zOlSy*I@Rg>{;a3MayTLVXh2_Z5E4neGV@WK6=|gz-Ur3a9 zq#NOjcjVN=f-eH@|6@|a6XFH8W}3Wt$NJA|S;^>w;-@$N?8KLZ|FdhKD!YY;f_t!~ zusm%EFnzd#2=Od|@2%C~aPO&;T;N}{tzG`}W8t%e>!TQFYa;n<=rvG*`T z<;`7`{sWuV;4H|LN)TH0uADL`I}ghiWSaxSg3jyd$AAK21!2)e|> z#0CB2h)Ox|sgXk`p_{WFJ(65OCEK5adh-?!Z7*s6!<-(vZOAE^V2qr3>Hh{g_0#*8L{;i3xwhopF$a_aAXPUOw9%3M1FZUph9?nlq0Vm zxNi$}c2k?k098#tsGVvgy3N|So$->Ja ze;~gE8S?WcET|WHZyrgs{mQIe4?0wOOJ874l?>$wsyNHhL$)We1wYY5ctwPKO@HU} zbVkwAhqJgn?C||N%A~R@ZYH|&tA^@N5(agcPy!TdQsuTm6bmsWC=!{_QXi-uLNnn3 z&XJUQL6etH+6q)rLm;2g3?W*2Gif!0(Sc@(poydBejOo0%fqKCa2-Db%o7U3$If@q zNS;o@4(x?eX%-8k$NCjQsSZjY*@ImUO~0T=K`+*kGr%v3&4{D&S!is7w$|x0DCDiO z0E9#+6-u!lhN}Mzlq$bS1N5PbI0IaUzX5@!Xt3o6%kS_ajpti!SUq>{_JH`V7HN2r zcF96^cqH60z7^EFyH~7?u469YF9~l@wMNJ#y0i4ytP4|!lZ(c}sFx+Sb^}xC4G;vT z<|nC%RzWROH@fShbOy*{ma)aU-W_~E4?|tlfY+54#z}w|bT;=o?&+Y+&rYfFG4ZBu z3eL6AKLzqZ4rXU&E;eJQ%G6J&^H*C#2h8^O87}j`Pu=`kha-xKO>p7k)q^Su*}6bb z%M+ZGiCjau(50YvHK$;Vr_)*^S+)IlT5iE^|LBVu^@PzB8-R@RYi+1a>(fRdp+(Ql zw@WTyv`s>V?cu1}liQjcX;)c5YA&en@%WDJ)y)sVjr~#?gi-e_y7z^B3=C~gdduyL zTQH0XI&L*cB`>mS(c@c38U2FVJ_UwHCSksD#zAn5ayuz_e#$|3jD|LuG>H|2*8Lo! zWql&DK}#kD%zSXd2<_+n6C?F{9I?-Ko|5MU_DVLiV5I`fQjuR zU{qN$=8|Xs7!BfE;!*nV+`NU{pExap;zNQOr!zR36HT>!I#VRnmEedzUu+$IY*!wh zlo?r((7VIm1L%D9jZd8mf~2IE0F=#r6+?>Qf|UX*r}Q7b1_E}UF=;$skudN64RK-Z z$$T<-D@Af@3jvF$&eUvn1`cdAxuvN^H%1K*1eYeQpj=O;OogDI1d=qhL;tZ`^dt2b z>7Ysz9A&=pcOJ;ph1CVmo*<@-mGlo{O%g}&qG~f{39NN8bs>ZbQoHCCF50j|y?=Qk z!AmfE&_GL#eDrtdm0qLvD<12x6}Yk*M;S*Fn1$0vE#NM&@)`dI0{%kPX+_%5F{07M9vTylg2cLM>MLrEosVZ%y#3W$XzHGa6UJ+KlkUw}JIwHUv8K4(<8$g8)L zj450UA8hJsA(*y+d-ehZ5v7pn{HGxH<62T}?eNi%DGd3^wx0+OJ|)|Gg@dnhPp;Tl zy3~Ox>Wh4tfJ);BouLFz1s~`?l>y^N6`j3g%SJrz>}TX! zjoXA7i0}dHlrMlZ88SSOI5;6P$TZLL1K*of#wY~7SCzWwU{m6nAIDJo#DI_(k%SNE zOkMO8SQ_V$4Hg(hbpMNTHA>a|eNk1ygW&OkGj+yt!+k(GM|vxa+ap1Z=2RpjL&sh^ z(TO1XMesIRKBFr*+WxFaEFi%VMY$jNcl#{a{6477Z6L3HM)MD%>eW^U8bJb68aDD6 znq4hZ8k`xfB#+M^zIu#L44A<{GjYz4sxfj5FRtFh453dLg?#;(@E=Hnxjld|7)e!r z{W^mtYiyS}gxwAeJ+VuAX0wFnjlTR&}}uzVO@r$As#p$Mo>BsvNv1|5R0P*prqeTTz?~&&JZ1 z|CQBp9f0VG1t}ia8C(9{Jt^M)Bf-041v`dW%BA2EWa#x@uJ)-5oGIn9a59M*fuMgZ zn4$IXzgW=sKNi%o|Hpz>XLCu|@H5QsZwWj_&s(I_gYsn$B8;t0O8rm~1VOJ*LwyzC zEI4_H!=3`CK_G9@ew;!)KS6N|IZbAZNwFdM>B2HFQBkD0j-4j8)fx3Z3`UV9yQNxM zH1Yv8OM0t}c2R0C;5U?tgkJL>7qSy0_Pj9xM|-#a<3b8~%;W#aP_+F&GQ2$W4Enu} zO7Z)U5rec#8DhQcqvVdYEN6}g184|10$xF>gxhU`-sA}#;g;45uW0tb-wPC2>9)fwFctu^pd1UA3T58xHLafXoU%ZKlrHzg;=DHhDz?4gAnq5aNU8TI3ERo)`pw%^& zq}Tw^@FC1RfELsvU4e4&e|d(sZ_QQ`f-b)BqJ@`-K!d&am(QG>K_g zN!vfC&%<{wAzyy&X?7sFh!B>KgTkS z$Pah=6zf5uUUBk{jgVg_z?~A(QpYk|L*7tk!*|eQW8T#LI(i)W4p|(E1@e$P4`K-W zX(FGfIdK-KA^w}60`Kh_g(wn-zmsM5_u|j%pg5Lop131(T($t+#%BPhmAQxb`>Qmx ztV?A!gz-zpo6p-Lj*Z0VGDg?dM73!4UL;3|>&qX_NS4vuzFp{p;!3cRoiEtcy@UWocA^c)$-E>_DrJLQ+3v=eo19R75nyJ7w!Cz0)e$@o|VZ z9>Gy@2zAP)`Sc6_TV3q)*GA}y&gPtP^Y+c>sY@}!3vql>Mq#G3q)4CYxp70j>TIrB{4PPQL8>vg)%WrbY&8BezMs{Pd` z)i=E&?<^xmzpU`g=mM6)qe>sr|Gb>Bw$Q6qs!UEFKY!jh_E__5Pfc0XJYtl`VN%~h zSD?fno3hDRmmlZn=WoQRI|F%Wqvs}*_7*~%do2xj;P?4#slgvfAUf#}%$)v~uO9o; zRztACj^1%e4R-`hn(10EIWskXG-WFhI!cH@ZjC<_$A4*{!dCzcKqG@M8c)RMn913^ zz=yYAMR0kzg8V;XapOa7|3t`H=!f|U#u4Qc4FPp1>%wHmi0(KcUWmAX8^~RhTv1jg zt-X45Cvz$;%YDPQ8=?r6U8hEVk|yrrU0^P6d=peMYGfURkWl{|FJxF7a6=|&$L>RL zj{5Qe(w%?0-&Dtf4uq zTfdOBI?qxA3kkh`?QPeBvU_Zm#5(*IEaV}>975F}SBUF4D~a1EALG1R=e~XFyj>PX zwK##J3!ILJxyoKS{gol|^Bguk9EvY|vLVesx ziXpBL*)d5%MSSIA{wKUn7J54ty>jahnE1H&nfP@~!w`JZ5juI@?Xo~Bm;dIC5gv-a zG2mUXzqwvQw1|H(^O+CpVdUjBJQ`pTZ^M6kE5;B_SeAG%TwX2E_?YQNUe=}x(h%yU7RrNjh`@J06l|rqXDv4J=?-{~>M(eMOuH^{ zr`^j;@PSaVvY^ke1E6pf=?(Y*&)QF_0>1u8schMN728=|V)%+KZ^0BF=LYVaHm9Xr z<|L=3wm5D4cJf!*lJ^=BP5g@UzS`yQlNNo_X}!gt21+w${Ufjl45a zV-S5t9JtK#C+V>m@=IKh!<|l-e{2)cShS1VwTZf(^(Rfg?HqYqR z1^+E)84a(iA=vlS+_nLvB}pg(!ehTst+jT>ai>L@N3_tgXi`kq41~tAQ{5yqS*G8g z*&h&CXcgB~%|pgiWmd5?3kM@ z=)Ka0bmfj7vrtC|Y|Xdrc2JNZZXpo;I7b#l8mMyFi(Zn+J)%l$m=EWjF6owC*ZbA4 z`sNj$Wj7yeSdVnI)c@r=j}G$p>dOtzwPO(-gocP!P;1=s%=}+e)yyF@&p|!o-#)=_ zz#d0y2l3zX904(ebMG3EkSGm+_=o>QZ+jo_5_?hf7CxTr9aDbiIBS=% zJticPATYf1Q#VrM_tKhX9rEkFpN4+DocW(SZR2UiY}#)T#7`M@SKtZ0cS)?WOPEuh zZcRsacM(uu$7t`#8w7xW?Bk?Q8#^{3X)A=1Hdrbz`!Z9zq6RyCxgRbg&q9r(#IMZ^ zM8I3dojL_XCS_Zk9b~}o{u^V?l-CGZ4tGAJTRaV9-=q6)oH$eV(6&s7FZA?pIXaH* zxDc)a{$DSIz}|IqLDZN)_+vQ>naG4N3sl-o&IO>9bfa)t+Uj>!v^+Q7$BxUqAdC~x z7?`k;Xr7FmZ$82D9I&9p+7pR8b3$AOox5>>46-pkW`zscygHlkJSL1|*GT`02ZNp0 z8b<;5W*FFM1~0&GmOnLPtlJxoUyTXbolCeAmzYvg06S=s`RhQa5$ z3BXd*a`Ibzv~7WB&CTEM%bOD?5|m1ZoikV!+;o1_el;qj7gJ7{8N>Enm!BT;&A2Op zs|Rqb-ivpj#Z6w=+1ysQKoL~Gj=nEdfO>X*>JThMoX)1kMcy>8JrAQVRWQ#eP{7mu zvAb4s^x8W^sg$MYf{b>?iYJ@#Ujy*&C(d8xiWQc|-$2w;=OYvMA?YgZ2raOTh}T;L zDv9@yMb*S@NVw_*V#IT0?ji*@Pm*u?Kfa5XeW$&(YRD)-o=%d;AbdT#@Me6b;rA*E z&soAvQ&l^?7<+FGVH7(jGuQV`+d51_djB_oH{G~s|xr}Xyuc! zo)_`!-+d6DV>FexdilOnss!H9(XjWc#e1dnX>X^mfVPv0CtjWpUV%QF?cYRvsdM%Y z7OPc5BhhPUm?`KMpZPVtNfMDSm8-q?8eG4LN? zq~5W<03R95A0U?h%W~tD~8Ax?2**r#b(p875&Z4Ipio@_o78@~|XzlG&LfxNl$?d5=C6TPR)uFSh<6C45g(jOE&iwK(gXF8)5PKQDFat zRayM+CkISO>2KTLa8nia7P~1oYy~f~sZPpdM3wQzFj?_aoM4{%VUGc4OTxUF0-A7Z zjVs=Ckrw@nElvgV10Eg^9tLtT56fEF$GK}r1Yb{bb$s~HP?8X0jNZlCSKUl50`kw# znb`lH+gX)%QNh@Ozr{z>O{7BjwR#sX2a61m@VW|QJO4&O$O*L(gEis?lHOD6UD$Nq zO$wtq`@cn+)$`YTsb+&>UELK#QZ}ySH2?6!Cr2yq0`uAY!%Yi`E9hrMmjOvy7U+{! zUJa&(4kC?_zXF?@B?@Nwy3WO|{S@x@*TQv7B{Tm`)*;D3KnYw{(&Hx!{6)1@N+;Pa zrbJEB^&jSdJ6N`K9fD6=!68+m7hiG){`ChCos7voU9pc+6mwy<>R=_#e|;0I(M5Z* zDa>wS!LEdaU4=rZM*B9Hci#6@$W{rF6q4wPIxrhxbM#_uevd&S26MXswz#AU7$eQP zBQjY7rqt`=>}&`Ir6?605+zuXenH?Ags;;ex#=EFn_M5=_BO1&V33Pi-V)aPPCu)r zQ*;v&_H`Wg<(8{V)(a`ZI$V;Rbu0B}MOF6?0uB*42ctg|`rRJXp4ZfMgKYpqDO3D) z%&mr~88i7wxo@_=pUi=yJIu*S)*F`k>2bZ0;t0*Xp*4?g9EHSzqcF@VTE-g|`{{8jp;`TQ zSV9n6t~;(DR6A9+oIL1vm-s@WMq!a=kPUnv)=Y^;Btbo71sgH@5d!fp0<7BOf9BscpiE37{VXpuOcztxGfS{ z%lQMz)8G7zXu`(4f<6<0UhK_gW7<_i>qWy4HeddtqxbSAJq<~+u6TK54YQKXYn$2w zpPqe+?~hLvMK6m!^o?Belq*~~Z%C4G9ol?3p-`yU;C>8uNNE__j2)w{2X2sd1l}aS zXDMQPXk#l*VCY=Lh!ofT&RB{tzX1R4$It9P$|&Y<@Z75DUt2DYuK!|rX-vO3k#Nk` zJa?GYzviNoX@CR|zq(0thi28P(>N=6RnjkhWe0zoq?`|PAU5PC_d1gd0d3GT)dyo+ zw#bSvm9vu40~2UqzzR?=g3d*Qsog#~g%I3GLOeaU$|1xgpzsu=Vd^ zD=6l~e|T4gMCB4oOV5wD_6vwtC~rR`eM-y{@ML>X_9;L`nTG$`=>z6`d}YN(jN`Z( zhv?MOUkR@{@`*|U843@_Ek(nz3Y(OU1_^5+-YGm<3ELruNdSG5!_em1kNnd3?T`mi zvtvp&1RSs*>b_kaCS=Ik{M)AZ)g`u;KWgzb?JW*#f#je7hd+7bW6zxHtYn~)faTb} znmiwLGsGyYPv9rw=Z^+_rPuNN&sU;lB~~d~#>m}cQD3{k4TtOlHS;9TDXzza{xaH1 zVL&dDYgb-I0}4(mj~}AVUf9KL4+#4X(xRt1eFCq?`-7bx?m{tp^PUBL`|P5n_Y#Wq zb`o*33vq?kay(Ghb{V7D4|$J!m(PB-OKTFV*FvG-!Cc1K!e?6f|lcG4}cNU3kgQu&t`*Ti#I8#j9uig zku1`Ow-g~CwQ*;xXgLqD zPr(_(dnlmBDssAzRZBDJl;{UyAp~opu{eD_%aW=nDjV7OiGLj6a?e(YI8v}+t+L}r zEQheuTdhZBH@6831cy>K2`;DfZ#1*v3Ya^zl1KKk>rmNsZ^E`nLMdwnm+ksDTwme} z=sL6xNAy-}QrT5+!ldCt6@trh{TuwOxB{vUK&wYYFS|OGUHT@hkT{gGSa6xTf8&Z7 zS3uUGl?CpmMr9Yg2}6Ppd&7 z<6Q3R+n8s-ZQ*ceH4X25rkJ%^B(#=2KQN0Wz~<1Z9NufINM)z833DP0)lU#yP5}B? z4j=(PV@G!k|3*{B3oeHh&CI=57h4YO+Zca=>qhO+x)#=ZFGqFEvk6Nj2=$5-l>MAu zp1&#T^!55#N9yhTVu2KEjn0Qh@%<^JU(SzmvDA9I2STEHpT>c+D(Y(8U8bKGd+{#5 zXR@N6AG3igS5Za1T|D1>`5YJ72Sp(E-TWn_SpL`irI)(rMPIWFrtmSgj_SlH;*ay3 zYi7tVJ0O+%lg@QJ=fT^_Hu1|+PvVB-u@9HbA6FjhYUHcu?-Pyp-u8J)#b(|{dRK*C zT|)>SKQXrXzA1D+GSt1ixNgTSYyCLW4?EecbF&!PHe=8(;SlPh(V1Aa7q|v^?h+a0 zI)C(~KjyVNo<+Ij)YnkT?vER{@y6Qq7kZsc2L7t*|J7!nblg#xd9-&d+{T$^hb%|) zXH3-ij`MJF_tB8%$L9&n1TiC(++ValHE~xC=YO-de*dYt6yn>O*%cE#f;XsdbC0j~ zeR5;2;Z3B!H;4I!L;h{+q`6N`?P>k9KUM8w-x2>%IsI9X_u;TOcW}NnFx>lHn>(RN zB9@AEKZ;>3_YfgzGx3G(JZI{o8#`I&BLx4?mhDPY24Lo9}erloTt z*iVeK=O*o!&#~vaDk#+`lE@2qGuBsYva-h(e?u6~`Tu{T@c(trG2MxCr?Fw*^YE@m av`O`d*Qt3?M6{Uvg4~!446>?nnEw|YISNt$