From 87b26cc53b0a0248a0c7221aabd0c1771f26dfb1 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:10:03 -0400 Subject: [PATCH] [develop] Update ufs-weather-model hash and UPP hash and use upp-addon-env spack-stack environment (#1136) * Update ufs-weather-model hash to 38a29a6 (September 19) * Update UPP hash to 81b38a8 (August 13) * All Tier-1 modulefiles/build_*.lua files have been updated to use the upp-addon-env spack-stack environment * srw_common.lua was updated to use g2/3.5.1 and g2tmpl/1.13.0 (these are required for UPP) * .cicd/JENKINSFILE was updated to replace cheyenne entries with derecho. * The doc/tables/Tests.csv table had nco-mode WE2E tests removed * The doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst documentation was updated to updated ush/config_defaults.yaml file. * The .github/CODEOWNERS file was updated to add Bruce Kropp to the list of reviewers * The exregional_plot_allvars.py and exregional_plot_allvars_diff.py scripts were updated to address changes made to the postxconfig-NT-fv3lam.txt file. * Updated ush/config_defaults.yaml to update PE_MEMBER01 calculation and documentation for OMP_NUM_THREADS_RUN_FCST to allow for the run_fcst task to properly run on Tier-1 platforms after updates to allow threading to function properly. * The ush/machine/*.yaml files were updated to allow for the run_fcst task to properly run on Tier-1 platforms after updates to allow threading to function properly. * There are not enough resources on Jet to run the high resolution WE2E tests (136 (ReqNodeNotAvail)). Commented out the tests in the comprehensive.jet test suite and removed one test from the coverage.jet test suite. * The ufs-case-studies WE2E tests are currently failing on Derecho. The failure is due to the file not being available. This is an issue because the file in question is named correctly and is available, but the tests fail in the get_extrn_ics/lbs tasks stating that the files aren't present. Commented out these tests in comprehensive.derecho and moved WE2E tests to remove from coverage.derecho. Issue ufs-case-studies WE2E tests fail on Derecho in get_extrn_ics/lbcs #1144 was opened to track this issue on Derecho. --- .cicd/Jenkinsfile | 4 +- .github/CODEOWNERS | 2 +- Externals.cfg | 4 +- .../CustomizingTheWorkflow/ConfigWorkflow.rst | 6 +- doc/tables/Tests.csv | 8 +- modulefiles/build_derecho_intel.lua | 2 +- modulefiles/build_gaea_intel.lua | 6 +- modulefiles/build_hera_gnu.lua | 2 +- modulefiles/build_hera_intel.lua | 2 +- modulefiles/build_hercules_intel.lua | 2 +- modulefiles/build_jet_intel.lua | 2 +- modulefiles/build_noaacloud_intel.lua | 2 +- modulefiles/build_orion_intel.lua | 2 +- modulefiles/srw_common.lua | 4 +- modulefiles/tasks/gaea/python_srw.lua | 3 +- modulefiles/wflow_gaea.lua | 1 - parm/model_configure | 3 - parm/ufs.configure | 25 ++++-- scripts/exregional_plot_allvars.py | 10 ++- scripts/exregional_plot_allvars_diff.py | 9 ++- .../WE2E/machine_suites/comprehensive.derecho | 18 ++--- tests/WE2E/machine_suites/comprehensive.jet | 77 +++++++++++++++++++ tests/WE2E/machine_suites/coverage.derecho | 6 +- .../machine_suites/coverage.hera.intel.nco | 2 +- tests/WE2E/machine_suites/coverage.hercules | 2 +- tests/WE2E/machine_suites/coverage.jet | 2 +- ...fig.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 1 + ..._FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml | 2 - ...3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml | 2 - ...pact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml | 2 - ...m_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml | 2 - .../test_create_model_configure_file.py | 2 - ush/config.aqm.yaml | 1 + ush/config_defaults.yaml | 4 +- ush/create_model_configure_file.py | 2 - ush/create_ufs_configure_file.py | 7 +- ush/machine/gaea.yaml | 6 +- ush/machine/hera.yaml | 4 +- ush/machine/hercules.yaml | 13 +++- ush/machine/jet.yaml | 6 +- ush/machine/orion.yaml | 13 +++- 41 files changed, 191 insertions(+), 82 deletions(-) create mode 100644 tests/WE2E/machine_suites/comprehensive.jet diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 5b90ab1173..18d015b6e9 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -12,9 +12,9 @@ pipeline { parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 700cea255c..e1878a6447 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # These owners will be the default owners for everything in the repo. #* @defunkt -* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @dmwright526 @gspetro-NOAA @natalie-perlin @EdwardSnyder-NOAA @MichaelLueken @rickgrubin-noaa +* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @dmwright526 @gspetro-NOAA @natalie-perlin @EdwardSnyder-NOAA @MichaelLueken @rickgrubin-noaa @BruceKropp-Raytheon # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners diff --git a/Externals.cfg b/Externals.cfg index b57d63957e..9acd326b65 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = a1143cc +hash = 38a29a6 local_path = sorc/ufs-weather-model required = True @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = be0410e +hash = 81b38a8 local_path = sorc/UPP required = True diff --git a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst index 4d88173028..d9704b1ab6 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1097,7 +1097,7 @@ For each workflow task, certain parameter values must be passed to the job sched For more information, see the `Intel Development Reference Guide `__. ``OMP_NUM_THREADS_RUN_FCST``: (Default: 2) - The number of OpenMP threads to use for parallel regions. Corresponds to the ``atmos_nthreads`` value in ``model_configure``. + The number of OpenMP threads to use for parallel regions. Corresponds to the ``ATM_omp_num_threads`` value in ``ufs.configure``. ``OMP_STACKSIZE_RUN_FCST``: (Default: "1024m") Controls the size of the stack for threads created by the OpenMP implementation. @@ -1163,12 +1163,12 @@ Write-Component (Quilting) Parameters ``PRINT_ESMF``: (Default: false) Flag that determines whether to output extra (debugging) information from :term:`ESMF` routines. Note that the write component uses ESMF library routines to interpolate from the native forecast model grid to the user-specified output grid (which is defined in the model configuration file ``model_configure`` in the forecast run directory). Valid values: ``True`` | ``False`` -``PE_MEMBER01``: (Default: ``'{{ LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X}}'``) +``PE_MEMBER01``: (Default: ``'{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X)}}'``) The number of MPI processes required by the forecast. When QUILTING is true, it is calculated as: .. math:: - LAYOUT\_X * LAYOUT\_Y + WRTCMP\_write\_groups * WRTCMP\_write\_tasks\_per\_group + OMP\_NUM\_THREADS\_RUN\_FCST * (LAYOUT\_X * LAYOUT\_Y + WRTCMP\_write\_groups * WRTCMP\_write\_tasks\_per\_group) ``WRTCMP_write_groups``: (Default: "") The number of write groups (i.e., groups of :term:`MPI` tasks) to use in the write component. Each write group will write to one set of output files (a ``dynf${fhr}.nc`` and a ``phyf${fhr}.nc`` file, where ``${fhr}`` is the forecast hour). Each write group contains ``WRTCMP_write_tasks_per_group`` tasks. Usually, one write group is sufficient. This may need to be increased if the forecast is proceeding so quickly that a single write group cannot complete writing to its set of files before there is a need/request to start writing the next set of files at the next output time. diff --git a/doc/tables/Tests.csv b/doc/tables/Tests.csv index d063e5c3fa..325b4dad69 100644 --- a/doc/tables/Tests.csv +++ b/doc/tables/Tests.csv @@ -1,6 +1,5 @@ Fundamental,Comprehensive,Test Name,PREDEF_GRID_NAME,CCPP_PHYS_SUITE,EXTRN_MDL_NAME_ICS,EXTRN_MDL_NAME_LBCS,DATES (UTC),FCST_LEN_HRS (hrs),est. core hours, walltime (min),notes yes,yes,grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,8,22, -yes,yes,nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16,RRFS_CONUS_25km,FV3_GFS_v16,FV3GFS,FV3GFS,2022081012,6,10,15, yes,yes,grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,7,10, yes,yes,grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot,RRFS_CONUS_25km,FV3_GFS_v17_p8,FV3GFS,FV3GFS,2019070100,6,11,15, yes,yes,grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR,RRFS_CONUScompact_25km,FV3_HRRR,HRRR,HRRR,2020081000,24,26,20 @@ -50,11 +49,6 @@ yes,yes,grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16,RRFS_CONUS_25km,FV3_ ,yes,grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta,SUBCONUS_Ind_3km,FV3_RRFS_v1beta,RAP,RAP,2020080103,3,20,22, ,yes,MET_ensemble_verification_only_vx,RRFS_CONUS_25km,*none*,*none*,*none*,2019061500,6,1,15,Runs ensemble verification tasks on staged data without running the rest of the workflow ,yes,MET_verification_only_vx,RRFS_CONUS_25km,*none*,*none*,*none*,2019061500,6,1,8,Runs verification tasks on staged data without running the rest of the workflow -,yes,nco,RRFS_CONUS_25km,FV3_GFS_v16,FV3GFS,FV3GFS,2022040700,6,7,20, -,yes,nco_ensemble,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100 2019070112 2019070200 2019070212,6,55,20, -,yes,nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16,RRFS_CONUS_13km,FV3_GFS_v16,FV3GFS,FV3GFS,2019061500,6,26,20, -,yes,nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km,RRFS_CONUS_3km,FV3_GFS_v15_thompson_mynn_lam3km,FV3GFS,FV3GFS,2019061500,6,320,25, -,yes,nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR,RRFS_CONUScompact_25km,FV3_HRRR,HRRR,RAP,2020081000,6,12,17, ,yes,pregen_grid_orog_sfc_climo,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,17, ,yes,specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2021061500,6,6,19, ,yes,specify_template_filenames,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,28, @@ -74,4 +68,4 @@ yes,yes,grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16,RRFS_CONUS_25km,FV3_ ,,aqm_grid_AQM_NA13km_suite_GFS_v16,AQM_NA_13km,FV3_GFS_v16,FV3GFS,FV3GFS,2023021700 2023021706,6,,,This is an air-quality model test that requires special compilation to run; not supported in this release ,,grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta,RRFS_NA_3km,FV3_RRFS_v1beta,FV3GFS,FV3GFS,2019070100,3,,,The RRFS_NA_3km domain currently has segfault problems--this test is not run ,,subhourly_post,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,,,Subhourly post tasks are currently broken--these tests are not run -,,subhourly_post_ensemble_2mems,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,,,Subhourly post tasks are currently broken--these tests are not run \ No newline at end of file +,,subhourly_post_ensemble_2mems,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,,,Subhourly post tasks are currently broken--these tests are not run diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua index 491a94f912..1356fdb3e0 100644 --- a/modulefiles/build_derecho_intel.lua +++ b/modulefiles/build_derecho_intel.lua @@ -6,7 +6,7 @@ the CISL machine Derecho (Cray) using Intel@2021.10.0 whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===]) prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.10.0")) load(pathJoin("stack-cray-mpich", os.getenv("stack_cray_mpich_ver") or "8.1.25")) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index b47209194c..2a53acf15b 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -5,11 +5,11 @@ the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0 whatis([===[Loads libraries needed for building the UFS SRW App on Gaea C5 ]===]) -prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) -stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.25" +stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.28" load(pathJoin("stack-cray-mpich", stack_mpich_ver)) stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 8854108966..621c7581a6 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GN prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles") prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles") -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/ufs-wm-srw-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/upp-addon-env/install/modulefiles/Core") load("stack-gcc/13.3.0") load("stack-openmpi/4.1.6") diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index d8e793044c..d3d20a5bb5 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index b65890f1c4..e01b68905e 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -5,7 +5,7 @@ the MSU machine Hercules using intel-oneapi-compilers/2022.2.1 whatis([===[Loads libraries needed for building the UFS SRW App on Hercules ]===]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.9.0") load("stack-intel-oneapi-mpi/2021.9.0") diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index a0169a684a..78e70e0960 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -5,7 +5,7 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===]) -prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.5.0") load("stack-intel-oneapi-mpi/2021.5.1") diff --git a/modulefiles/build_noaacloud_intel.lua b/modulefiles/build_noaacloud_intel.lua index b1a6adbbf0..1f5855343d 100644 --- a/modulefiles/build_noaacloud_intel.lua +++ b/modulefiles/build_noaacloud_intel.lua @@ -5,7 +5,7 @@ the NOAA cloud using Intel-oneapi whatis([===[Loads libraries needed for building the UFS SRW App on NOAA cloud ]===]) -prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/apps/modules/modulefiles") load("gnu") load("stack-intel") diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index b2f3d85c00..92d268a4d7 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -5,7 +5,7 @@ the MSU machine Orion using intel-oneapi-compilers/2021.9.0 whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.9.0") load("stack-intel-oneapi-mpi/2021.9.0") diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index cb2047cbe1..d2bdbe6790 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -10,8 +10,8 @@ load("fms/2023.04") load("bacio/2.4.1") load("crtm/2.4.0.1") -load("g2/3.4.5") -load("g2tmpl/1.10.2") +load("g2/3.5.1") +load("g2tmpl/1.13.0") load("ip/4.3.0") load("sp/2.5.0") load("w3emc/2.10.0") diff --git a/modulefiles/tasks/gaea/python_srw.lua b/modulefiles/tasks/gaea/python_srw.lua index 5058b3f615..84df70682d 100644 --- a/modulefiles/tasks/gaea/python_srw.lua +++ b/modulefiles/tasks/gaea/python_srw.lua @@ -3,5 +3,6 @@ unload("python") load("conda") setenv("SRW_ENV", "srw_app") -setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") +setenv("LD_PRELOAD", "/usr/lib64/libstdc++.so.6") +setenv("FI_VERBS_PREFER_XRC", "0") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 6c24672c30..c2207ff57b 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -11,7 +11,6 @@ load("rocoto") load("conda") pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.1.0/") -setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: diff --git a/parm/model_configure b/parm/model_configure index aeb45f4719..d22adf3f3a 100644 --- a/parm/model_configure +++ b/parm/model_configure @@ -1,5 +1,3 @@ -total_member: 1 -PE_MEMBER01: {{ PE_MEMBER01 }} start_year: {{ start_year }} start_month: {{ start_month }} start_day: {{ start_day }} @@ -13,7 +11,6 @@ ENS_SPS: .false. dt_atmos: {{ dt_atmos }} calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: {{ atmos_nthreads }} restart_interval: {{ restart_interval }} output_1st_tstep_rst: .false. write_dopost: {{ write_dopost }} diff --git a/parm/ufs.configure b/parm/ufs.configure index d90b7447f4..48d2a66e8a 100644 --- a/parm/ufs.configure +++ b/parm/ufs.configure @@ -19,14 +19,14 @@ EARTH_attributes:: # ATM # ATM_model: fv3 -ATM_petlist_bounds: -1 -1 +ATM_petlist_bounds: 0 {{ pe_member01_m1 }} ATM_attributes:: Verbosity = 0 :: # AQM # AQM_model: aqm -AQM_petlist_bounds: -1 -1 +AQM_petlist_bounds: 0 {{ aqm_pe_member01_m1 }} AQM_attributes:: Verbosity = 0 Diagnostic = 0 @@ -45,8 +45,21 @@ runSeq:: {% else %} # EARTH # EARTH_component_list: ATM - ATM_model: fv3 - runSeq:: - ATM - :: +EARTH_attributes:: + Verbosity = 0 +:: + +# ATM # +ATM_model: fv3 +ATM_petlist_bounds: 0 {{ pe_member01_m1 }} +ATM_omp_num_threads: {{ atm_omp_num_threads }} +ATM_attributes:: + Verbosity = 0 + Diagnostic = 0 +:: + +# Run Sequence # +runSeq:: + ATM +:: {% endif %} diff --git a/scripts/exregional_plot_allvars.py b/scripts/exregional_plot_allvars.py index 040e17b012..14d15c07f0 100755 --- a/scripts/exregional_plot_allvars.py +++ b/scripts/exregional_plot_allvars.py @@ -429,7 +429,7 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth = ndimage.gaussian_filter(slp, 13.78) # 2-m temperature @@ -484,7 +484,13 @@ def setup_logging(debug=False): ) # Composite reflectivity - refc = data1.select(name="Maximum/Composite radar reflectivity")[0].values + # refc is the 37th entry in the GRIB2 post output file + # First rewind to the start of the GRIB2 file + data1.rewind() + # Advance 36 entries in the GRIB2 file + data1.seek(36) + # Read values from the 37th entry in the GRIB2 file + refc = data1.readline().values if fhr > 0: # Max/Min Hourly 2-5 km Updraft Helicity diff --git a/scripts/exregional_plot_allvars_diff.py b/scripts/exregional_plot_allvars_diff.py index 61efcdb82b..c493f68f65 100755 --- a/scripts/exregional_plot_allvars_diff.py +++ b/scripts/exregional_plot_allvars_diff.py @@ -446,9 +446,9 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp_1 = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp_1 = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth_1 = ndimage.gaussian_filter(slp_1, 13.78) - slp_2 = data2.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp_2 = data2.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth_2 = ndimage.gaussian_filter(slp_2, 13.78) slp_diff = slp_2 - slp_1 @@ -544,7 +544,12 @@ def setup_logging(debug=False): qpf_diff = qpf_2 - qpf_1 # Composite reflectivity + # refc is the 37th entry in the GRIB2 post output file + data1.rewind() + data1.seek(36) refc_1 = data1.select(name="Maximum/Composite radar reflectivity")[0].values + data2.rewind() + data2.seek(36) refc_2 = data2.select(name="Maximum/Composite radar reflectivity")[0].values if fhr > 0: diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho index 5930843582..b88ec31bba 100644 --- a/tests/WE2E/machine_suites/comprehensive.derecho +++ b/tests/WE2E/machine_suites/comprehensive.derecho @@ -1,12 +1,12 @@ -2020_CAD -2020_CAPE -2019_hurricane_barry -2019_halloween_storm -2019_hurricane_lorenzo -2019_memorial_day_heat_wave -2020_denver_radiation_inversion -2020_easter_storm -2020_jan_cold_blast +#2020_CAD +#2020_CAPE +#2019_hurricane_barry +#2019_halloween_storm +#2019_hurricane_lorenzo +#2019_memorial_day_heat_wave +#2020_denver_radiation_inversion +#2020_easter_storm +#2020_jan_cold_blast community custom_ESGgrid custom_ESGgrid_Central_Asia_3km diff --git a/tests/WE2E/machine_suites/comprehensive.jet b/tests/WE2E/machine_suites/comprehensive.jet new file mode 100644 index 0000000000..0e15479feb --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.jet @@ -0,0 +1,77 @@ +2020_CAD +2020_CAPE +2019_hurricane_barry +2019_halloween_storm +2019_hurricane_lorenzo +2019_memorial_day_heat_wave +2020_denver_radiation_inversion +2020_easter_storm +2020_jan_cold_blast +community +custom_ESGgrid +custom_ESGgrid_Central_Asia_3km +custom_ESGgrid_Great_Lakes_snow_8km +custom_ESGgrid_IndianOcean_6km +custom_ESGgrid_NewZealand_3km +custom_ESGgrid_Peru_12km +custom_ESGgrid_SF_1p1km +custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE +custom_GFDLgrid +deactivate_tasks +#get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021032018 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h +#get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_netcdf_2022040400_ensemble_2mems +get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS +get_from_HPSS_ics_HRRR_lbcs_RAP +get_from_HPSS_ics_RAP_lbcs_RAP +get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS +grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_RRFS_lbcs_RRFS_suite_RRFS_v1beta +grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 +grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot +long_fcst +MET_ensemble_verification_only_vx +MET_ensemble_verification_only_vx_time_lag +MET_ensemble_verification_winter_wx +MET_verification_only_vx +pregen_grid_orog_sfc_climo +specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS +specify_template_filenames diff --git a/tests/WE2E/machine_suites/coverage.derecho b/tests/WE2E/machine_suites/coverage.derecho index a948c76033..a10c865447 100644 --- a/tests/WE2E/machine_suites/coverage.derecho +++ b/tests/WE2E/machine_suites/coverage.derecho @@ -1,11 +1,11 @@ custom_ESGgrid_IndianOcean_6km grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR pregen_grid_orog_sfc_climo specify_template_filenames -2019_hurricane_barry -2019_memorial_day_heat_wave -2020_denver_radiation_inversion diff --git a/tests/WE2E/machine_suites/coverage.hera.intel.nco b/tests/WE2E/machine_suites/coverage.hera.intel.nco index d5ab0d6fe8..e4b02a90ba 100644 --- a/tests/WE2E/machine_suites/coverage.hera.intel.nco +++ b/tests/WE2E/machine_suites/coverage.hera.intel.nco @@ -1,8 +1,8 @@ +2019_memorial_day_heat_wave custom_ESGgrid_Peru_12km get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_netcdf_2022040400_ensemble_2mems get_from_HPSS_ics_HRRR_lbcs_RAP -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 diff --git a/tests/WE2E/machine_suites/coverage.hercules b/tests/WE2E/machine_suites/coverage.hercules index ec37d81a56..668a26e685 100644 --- a/tests/WE2E/machine_suites/coverage.hercules +++ b/tests/WE2E/machine_suites/coverage.hercules @@ -4,9 +4,9 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP -grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 MET_verification_only_vx specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS 2019_hurricane_lorenzo +2020_denver_radiation_inversion diff --git a/tests/WE2E/machine_suites/coverage.jet b/tests/WE2E/machine_suites/coverage.jet index 53308090b1..8c79a0b700 100644 --- a/tests/WE2E/machine_suites/coverage.jet +++ b/tests/WE2E/machine_suites/coverage.jet @@ -1,3 +1,4 @@ +2019_hurricane_barry community custom_ESGgrid custom_ESGgrid_Great_Lakes_snow_8km @@ -8,4 +9,3 @@ get_from_HPSS_ics_RAP_lbcs_RAP grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml index 2901d1ebf1..a9548f8b5f 100644 --- a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml @@ -34,6 +34,7 @@ task_get_extrn_lbcs: EXTRN_MDL_LBCS_OFFSET_HRS: 0 USE_USER_STAGED_EXTRN_FILES: true task_run_fcst: + OMP_NUM_THREADS_RUN_FCST: 1 DT_ATMOS: 180 LAYOUT_X: 50 LAYOUT_Y: 34 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml index de456cea73..f4c40bf722 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml @@ -20,5 +20,3 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 USE_USER_STAGED_EXTRN_FILES: true -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml index 4a340185f3..6d4dbc3b33 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml @@ -19,5 +19,3 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 USE_USER_STAGED_EXTRN_FILES: true -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml index 35be12a1ee..b00a24ae84 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml @@ -23,5 +23,3 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_FILES_LBCS: - '{yy}{jjj}{hh}00{fcst_hr:02d}00' -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml index 1265fa8e0c..44dfec5e75 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml @@ -24,5 +24,3 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_FILES_LBCS: - '{yy}{jjj}{hh}00{fcst_hr:02d}00' -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/test_python/test_create_model_configure_file.py b/tests/test_python/test_create_model_configure_file.py index d5aea79ed8..9475028505 100644 --- a/tests/test_python/test_create_model_configure_file.py +++ b/tests/test_python/test_create_model_configure_file.py @@ -43,11 +43,9 @@ def setUp(self): set_env_var("USHdir", USHdir) set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN) set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP) - set_env_var("PE_MEMBER01", 24) set_env_var("FCST_LEN_HRS", 72) set_env_var("FHROT", 0) set_env_var("DT_ATMOS", 1) - set_env_var("OMP_NUM_THREADS_RUN_FCST", 1) set_env_var("RESTART_INTERVAL", 4) set_env_var("ITASKS", 1) diff --git a/ush/config.aqm.yaml b/ush/config.aqm.yaml index 21a73591ee..77512ec7a2 100644 --- a/ush/config.aqm.yaml +++ b/ush/config.aqm.yaml @@ -46,6 +46,7 @@ task_get_extrn_lbcs: FV3GFS_FILE_FMT_LBCS: netcdf EXTRN_MDL_LBCS_OFFSET_HRS: 0 task_run_fcst: + OMP_NUM_THREADS_RUN_FCST: 1 DT_ATMOS: 180 LAYOUT_X: 50 LAYOUT_Y: 34 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 488c2d5d3d..6af52578b8 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1722,7 +1722,7 @@ task_run_fcst: #----------------------------------------------------------------------- # KMP_AFFINITY_RUN_FCST: "scatter" - OMP_NUM_THREADS_RUN_FCST: 2 # atmos_nthreads in model_configure + OMP_NUM_THREADS_RUN_FCST: 2 # ATM_omp_num_threads in ufs.configure OMP_STACKSIZE_RUN_FCST: "1024m" # #----------------------------------------------------------------------- @@ -1918,7 +1918,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X}}' + PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X)}}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 9b430b83ae..e0e420c71b 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -70,7 +70,6 @@ def create_model_configure_file( # ----------------------------------------------------------------------- # settings = { - "PE_MEMBER01": PE_MEMBER01, "start_year": cdate.year, "start_month": cdate.month, "start_day": cdate.day, @@ -78,7 +77,6 @@ def create_model_configure_file( "nhours_fcst": fcst_len_hrs, "fhrot": fhrot, "dt_atmos": DT_ATMOS, - "atmos_nthreads": OMP_NUM_THREADS_RUN_FCST, "restart_interval": RESTART_INTERVAL, "itasks": ITASKS, "write_dopost": f".{lowercase(str(WRITE_DOPOST))}.", diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index dc6a43420d..e97fa25c6a 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -50,6 +50,8 @@ def create_ufs_configure_file(run_dir): # Set output file path # ufs_config_fp = os.path.join(run_dir, UFS_CONFIG_FN) + pe_member01_m1 = str(int(PE_MEMBER01)-1) + aqm_pe_member01_m1 = str(int(LAYOUT_X*LAYOUT_Y)-1) # #----------------------------------------------------------------------- # @@ -62,7 +64,10 @@ def create_ufs_configure_file(run_dir): settings = { "dt_atmos": DT_ATMOS, "print_esmf": PRINT_ESMF, - "cpl_aqm": CPL_AQM + "cpl_aqm": CPL_AQM, + "pe_member01_m1": pe_member01_m1, + "aqm_pe_member01_m1": aqm_pe_member01_m1, + "atm_omp_num_threads": OMP_NUM_THREADS_RUN_FCST, } settings_str = cfg_to_yaml_str(settings) diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 1ec2ded2ef..92d33d7ad2 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -12,7 +12,7 @@ platform: QUEUE_HPSS: normal REMOVE_MEMORY: True PARTITION_HPSS: eslogin_c5 - RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL -n $nprocs RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs RUN_CMD_SERIAL: time @@ -47,9 +47,9 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + cores: native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' - nodes: + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' nnodes: nodesize: ppn: diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 80fbb8fc98..189689f30d 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -48,9 +48,9 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + cores: native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' - nodes: + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' nnodes: nodesize: ppn: diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index e29801dd49..6a325094da 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -13,7 +13,7 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL RUN_CMD_SERIAL: time @@ -51,3 +51,14 @@ cpl_aqm_parm: COMINfire_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/RAVE_fire COMINgefs_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GEFS_DATA NEXUS_GFS_SFC_DIR: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GFS_SFC_DATA + +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + cores: + native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' + nnodes: + nodesize: + ppn: diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 375711c61a..847530e4eb 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -7,7 +7,7 @@ platform: TEST_NDAS_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc TEST_NOHRSC_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/nohrsc/proc DOMAIN_PREGEN_BASEDIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen - PARTITION_DEFAULT: sjet,vjet,kjet,xjet + PARTITION_DEFAULT: vjet,kjet,xjet QUEUE_DEFAULT: batch PARTITION_FCST: xjet QUEUE_FCST: batch @@ -49,9 +49,9 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + cores: native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' - nodes: + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' nnodes: nodesize: ppn: diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 3f756e2836..5467160167 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -13,7 +13,7 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL RUN_CMD_SERIAL: time @@ -50,3 +50,14 @@ cpl_aqm_parm: COMINfire_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/RAVE_fire COMINgefs_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GEFS_DATA NEXUS_GFS_SFC_DIR: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GFS_SFC_DATA + +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + cores: + native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' + nnodes: + nodesize: + ppn: