diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 5b90ab1173..18d015b6e9 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -12,9 +12,9 @@ pipeline { parameters { // Allow job runner to filter based on platform // Use the line below to enable all PW clusters - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use') // Use the line below to enable the PW AWS cluster - // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') + // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 700cea255c..e1878a6447 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # These owners will be the default owners for everything in the repo. #* @defunkt -* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @dmwright526 @gspetro-NOAA @natalie-perlin @EdwardSnyder-NOAA @MichaelLueken @rickgrubin-noaa +* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @dmwright526 @gspetro-NOAA @natalie-perlin @EdwardSnyder-NOAA @MichaelLueken @rickgrubin-noaa @BruceKropp-Raytheon # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners diff --git a/Externals.cfg b/Externals.cfg index b57d63957e..9acd326b65 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = a1143cc +hash = 38a29a6 local_path = sorc/ufs-weather-model required = True @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = be0410e +hash = 81b38a8 local_path = sorc/UPP required = True diff --git a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst index 4d88173028..d9704b1ab6 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1097,7 +1097,7 @@ For each workflow task, certain parameter values must be passed to the job sched For more information, see the `Intel Development Reference Guide `__. ``OMP_NUM_THREADS_RUN_FCST``: (Default: 2) - The number of OpenMP threads to use for parallel regions. Corresponds to the ``atmos_nthreads`` value in ``model_configure``. + The number of OpenMP threads to use for parallel regions. Corresponds to the ``ATM_omp_num_threads`` value in ``ufs.configure``. ``OMP_STACKSIZE_RUN_FCST``: (Default: "1024m") Controls the size of the stack for threads created by the OpenMP implementation. @@ -1163,12 +1163,12 @@ Write-Component (Quilting) Parameters ``PRINT_ESMF``: (Default: false) Flag that determines whether to output extra (debugging) information from :term:`ESMF` routines. Note that the write component uses ESMF library routines to interpolate from the native forecast model grid to the user-specified output grid (which is defined in the model configuration file ``model_configure`` in the forecast run directory). Valid values: ``True`` | ``False`` -``PE_MEMBER01``: (Default: ``'{{ LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X}}'``) +``PE_MEMBER01``: (Default: ``'{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X)}}'``) The number of MPI processes required by the forecast. When QUILTING is true, it is calculated as: .. math:: - LAYOUT\_X * LAYOUT\_Y + WRTCMP\_write\_groups * WRTCMP\_write\_tasks\_per\_group + OMP\_NUM\_THREADS\_RUN\_FCST * (LAYOUT\_X * LAYOUT\_Y + WRTCMP\_write\_groups * WRTCMP\_write\_tasks\_per\_group) ``WRTCMP_write_groups``: (Default: "") The number of write groups (i.e., groups of :term:`MPI` tasks) to use in the write component. Each write group will write to one set of output files (a ``dynf${fhr}.nc`` and a ``phyf${fhr}.nc`` file, where ``${fhr}`` is the forecast hour). Each write group contains ``WRTCMP_write_tasks_per_group`` tasks. Usually, one write group is sufficient. This may need to be increased if the forecast is proceeding so quickly that a single write group cannot complete writing to its set of files before there is a need/request to start writing the next set of files at the next output time. diff --git a/doc/tables/Tests.csv b/doc/tables/Tests.csv index d063e5c3fa..325b4dad69 100644 --- a/doc/tables/Tests.csv +++ b/doc/tables/Tests.csv @@ -1,6 +1,5 @@ Fundamental,Comprehensive,Test Name,PREDEF_GRID_NAME,CCPP_PHYS_SUITE,EXTRN_MDL_NAME_ICS,EXTRN_MDL_NAME_LBCS,DATES (UTC),FCST_LEN_HRS (hrs),est. core hours, walltime (min),notes yes,yes,grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,8,22, -yes,yes,nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_timeoffset_suite_GFS_v16,RRFS_CONUS_25km,FV3_GFS_v16,FV3GFS,FV3GFS,2022081012,6,10,15, yes,yes,grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,7,10, yes,yes,grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot,RRFS_CONUS_25km,FV3_GFS_v17_p8,FV3GFS,FV3GFS,2019070100,6,11,15, yes,yes,grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR,RRFS_CONUScompact_25km,FV3_HRRR,HRRR,HRRR,2020081000,24,26,20 @@ -50,11 +49,6 @@ yes,yes,grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16,RRFS_CONUS_25km,FV3_ ,yes,grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta,SUBCONUS_Ind_3km,FV3_RRFS_v1beta,RAP,RAP,2020080103,3,20,22, ,yes,MET_ensemble_verification_only_vx,RRFS_CONUS_25km,*none*,*none*,*none*,2019061500,6,1,15,Runs ensemble verification tasks on staged data without running the rest of the workflow ,yes,MET_verification_only_vx,RRFS_CONUS_25km,*none*,*none*,*none*,2019061500,6,1,8,Runs verification tasks on staged data without running the rest of the workflow -,yes,nco,RRFS_CONUS_25km,FV3_GFS_v16,FV3GFS,FV3GFS,2022040700,6,7,20, -,yes,nco_ensemble,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100 2019070112 2019070200 2019070212,6,55,20, -,yes,nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16,RRFS_CONUS_13km,FV3_GFS_v16,FV3GFS,FV3GFS,2019061500,6,26,20, -,yes,nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km,RRFS_CONUS_3km,FV3_GFS_v15_thompson_mynn_lam3km,FV3GFS,FV3GFS,2019061500,6,320,25, -,yes,nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR,RRFS_CONUScompact_25km,FV3_HRRR,HRRR,RAP,2020081000,6,12,17, ,yes,pregen_grid_orog_sfc_climo,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,17, ,yes,specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2021061500,6,6,19, ,yes,specify_template_filenames,RRFS_CONUS_25km,FV3_GFS_v15p2,FV3GFS,FV3GFS,2019070100,6,6,28, @@ -74,4 +68,4 @@ yes,yes,grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16,RRFS_CONUS_25km,FV3_ ,,aqm_grid_AQM_NA13km_suite_GFS_v16,AQM_NA_13km,FV3_GFS_v16,FV3GFS,FV3GFS,2023021700 2023021706,6,,,This is an air-quality model test that requires special compilation to run; not supported in this release ,,grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta,RRFS_NA_3km,FV3_RRFS_v1beta,FV3GFS,FV3GFS,2019070100,3,,,The RRFS_NA_3km domain currently has segfault problems--this test is not run ,,subhourly_post,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,,,Subhourly post tasks are currently broken--these tests are not run -,,subhourly_post_ensemble_2mems,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,,,Subhourly post tasks are currently broken--these tests are not run \ No newline at end of file +,,subhourly_post_ensemble_2mems,RRFS_CONUScompact_25km,FV3_RRFS_v1beta,HRRR,RAP,2020081000,3,,,Subhourly post tasks are currently broken--these tests are not run diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua index 491a94f912..1356fdb3e0 100644 --- a/modulefiles/build_derecho_intel.lua +++ b/modulefiles/build_derecho_intel.lua @@ -6,7 +6,7 @@ the CISL machine Derecho (Cray) using Intel@2021.10.0 whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===]) prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.10.0")) load(pathJoin("stack-cray-mpich", os.getenv("stack_cray_mpich_ver") or "8.1.25")) diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua index b47209194c..2a53acf15b 100644 --- a/modulefiles/build_gaea_intel.lua +++ b/modulefiles/build_gaea_intel.lua @@ -5,11 +5,11 @@ the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0 whatis([===[Loads libraries needed for building the UFS SRW App on Gaea C5 ]===]) -prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") -stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) -stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.25" +stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.28" load(pathJoin("stack-cray-mpich", stack_mpich_ver)) stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 8854108966..621c7581a6 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GN prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles") prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles") -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/ufs-wm-srw-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/upp-addon-env/install/modulefiles/Core") load("stack-gcc/13.3.0") load("stack-openmpi/4.1.6") diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index d8e793044c..d3d20a5bb5 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index b65890f1c4..e01b68905e 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -5,7 +5,7 @@ the MSU machine Hercules using intel-oneapi-compilers/2022.2.1 whatis([===[Loads libraries needed for building the UFS SRW App on Hercules ]===]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.9.0") load("stack-intel-oneapi-mpi/2021.9.0") diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index a0169a684a..78e70e0960 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -5,7 +5,7 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===]) -prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.5.0") load("stack-intel-oneapi-mpi/2021.5.1") diff --git a/modulefiles/build_noaacloud_intel.lua b/modulefiles/build_noaacloud_intel.lua index b1a6adbbf0..1f5855343d 100644 --- a/modulefiles/build_noaacloud_intel.lua +++ b/modulefiles/build_noaacloud_intel.lua @@ -5,7 +5,7 @@ the NOAA cloud using Intel-oneapi whatis([===[Loads libraries needed for building the UFS SRW App on NOAA cloud ]===]) -prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/apps/modules/modulefiles") load("gnu") load("stack-intel") diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index b2f3d85c00..92d268a4d7 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -5,7 +5,7 @@ the MSU machine Orion using intel-oneapi-compilers/2021.9.0 whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") load("stack-intel/2021.9.0") load("stack-intel-oneapi-mpi/2021.9.0") diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua index cb2047cbe1..d2bdbe6790 100644 --- a/modulefiles/srw_common.lua +++ b/modulefiles/srw_common.lua @@ -10,8 +10,8 @@ load("fms/2023.04") load("bacio/2.4.1") load("crtm/2.4.0.1") -load("g2/3.4.5") -load("g2tmpl/1.10.2") +load("g2/3.5.1") +load("g2tmpl/1.13.0") load("ip/4.3.0") load("sp/2.5.0") load("w3emc/2.10.0") diff --git a/modulefiles/tasks/gaea/python_srw.lua b/modulefiles/tasks/gaea/python_srw.lua index 5058b3f615..84df70682d 100644 --- a/modulefiles/tasks/gaea/python_srw.lua +++ b/modulefiles/tasks/gaea/python_srw.lua @@ -3,5 +3,6 @@ unload("python") load("conda") setenv("SRW_ENV", "srw_app") -setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") +setenv("LD_PRELOAD", "/usr/lib64/libstdc++.so.6") +setenv("FI_VERBS_PREFER_XRC", "0") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 6c24672c30..c2207ff57b 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -11,7 +11,6 @@ load("rocoto") load("conda") pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2023.1.0/") -setenv("LD_PRELOAD", "/opt/cray/pe/gcc/12.2.0/snos/lib64/libstdc++.so.6") if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: diff --git a/parm/model_configure b/parm/model_configure index aeb45f4719..d22adf3f3a 100644 --- a/parm/model_configure +++ b/parm/model_configure @@ -1,5 +1,3 @@ -total_member: 1 -PE_MEMBER01: {{ PE_MEMBER01 }} start_year: {{ start_year }} start_month: {{ start_month }} start_day: {{ start_day }} @@ -13,7 +11,6 @@ ENS_SPS: .false. dt_atmos: {{ dt_atmos }} calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: {{ atmos_nthreads }} restart_interval: {{ restart_interval }} output_1st_tstep_rst: .false. write_dopost: {{ write_dopost }} diff --git a/parm/ufs.configure b/parm/ufs.configure index d90b7447f4..48d2a66e8a 100644 --- a/parm/ufs.configure +++ b/parm/ufs.configure @@ -19,14 +19,14 @@ EARTH_attributes:: # ATM # ATM_model: fv3 -ATM_petlist_bounds: -1 -1 +ATM_petlist_bounds: 0 {{ pe_member01_m1 }} ATM_attributes:: Verbosity = 0 :: # AQM # AQM_model: aqm -AQM_petlist_bounds: -1 -1 +AQM_petlist_bounds: 0 {{ aqm_pe_member01_m1 }} AQM_attributes:: Verbosity = 0 Diagnostic = 0 @@ -45,8 +45,21 @@ runSeq:: {% else %} # EARTH # EARTH_component_list: ATM - ATM_model: fv3 - runSeq:: - ATM - :: +EARTH_attributes:: + Verbosity = 0 +:: + +# ATM # +ATM_model: fv3 +ATM_petlist_bounds: 0 {{ pe_member01_m1 }} +ATM_omp_num_threads: {{ atm_omp_num_threads }} +ATM_attributes:: + Verbosity = 0 + Diagnostic = 0 +:: + +# Run Sequence # +runSeq:: + ATM +:: {% endif %} diff --git a/scripts/exregional_plot_allvars.py b/scripts/exregional_plot_allvars.py index 040e17b012..14d15c07f0 100755 --- a/scripts/exregional_plot_allvars.py +++ b/scripts/exregional_plot_allvars.py @@ -429,7 +429,7 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth = ndimage.gaussian_filter(slp, 13.78) # 2-m temperature @@ -484,7 +484,13 @@ def setup_logging(debug=False): ) # Composite reflectivity - refc = data1.select(name="Maximum/Composite radar reflectivity")[0].values + # refc is the 37th entry in the GRIB2 post output file + # First rewind to the start of the GRIB2 file + data1.rewind() + # Advance 36 entries in the GRIB2 file + data1.seek(36) + # Read values from the 37th entry in the GRIB2 file + refc = data1.readline().values if fhr > 0: # Max/Min Hourly 2-5 km Updraft Helicity diff --git a/scripts/exregional_plot_allvars_diff.py b/scripts/exregional_plot_allvars_diff.py index 61efcdb82b..c493f68f65 100755 --- a/scripts/exregional_plot_allvars_diff.py +++ b/scripts/exregional_plot_allvars_diff.py @@ -446,9 +446,9 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp_1 = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp_1 = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth_1 = ndimage.gaussian_filter(slp_1, 13.78) - slp_2 = data2.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp_2 = data2.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth_2 = ndimage.gaussian_filter(slp_2, 13.78) slp_diff = slp_2 - slp_1 @@ -544,7 +544,12 @@ def setup_logging(debug=False): qpf_diff = qpf_2 - qpf_1 # Composite reflectivity + # refc is the 37th entry in the GRIB2 post output file + data1.rewind() + data1.seek(36) refc_1 = data1.select(name="Maximum/Composite radar reflectivity")[0].values + data2.rewind() + data2.seek(36) refc_2 = data2.select(name="Maximum/Composite radar reflectivity")[0].values if fhr > 0: diff --git a/tests/WE2E/machine_suites/comprehensive.derecho b/tests/WE2E/machine_suites/comprehensive.derecho index 5930843582..b88ec31bba 100644 --- a/tests/WE2E/machine_suites/comprehensive.derecho +++ b/tests/WE2E/machine_suites/comprehensive.derecho @@ -1,12 +1,12 @@ -2020_CAD -2020_CAPE -2019_hurricane_barry -2019_halloween_storm -2019_hurricane_lorenzo -2019_memorial_day_heat_wave -2020_denver_radiation_inversion -2020_easter_storm -2020_jan_cold_blast +#2020_CAD +#2020_CAPE +#2019_hurricane_barry +#2019_halloween_storm +#2019_hurricane_lorenzo +#2019_memorial_day_heat_wave +#2020_denver_radiation_inversion +#2020_easter_storm +#2020_jan_cold_blast community custom_ESGgrid custom_ESGgrid_Central_Asia_3km diff --git a/tests/WE2E/machine_suites/comprehensive.jet b/tests/WE2E/machine_suites/comprehensive.jet new file mode 100644 index 0000000000..0e15479feb --- /dev/null +++ b/tests/WE2E/machine_suites/comprehensive.jet @@ -0,0 +1,77 @@ +2020_CAD +2020_CAPE +2019_hurricane_barry +2019_halloween_storm +2019_hurricane_lorenzo +2019_memorial_day_heat_wave +2020_denver_radiation_inversion +2020_easter_storm +2020_jan_cold_blast +community +custom_ESGgrid +custom_ESGgrid_Central_Asia_3km +custom_ESGgrid_Great_Lakes_snow_8km +custom_ESGgrid_IndianOcean_6km +custom_ESGgrid_NewZealand_3km +custom_ESGgrid_Peru_12km +custom_ESGgrid_SF_1p1km +custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE +custom_GFDLgrid +deactivate_tasks +#get_from_AWS_ics_GEFS_lbcs_GEFS_fmt_grib2_2022040400_ensemble_2mems +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021032018 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h +#get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_netcdf_2022040400_ensemble_2mems +get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS +get_from_HPSS_ics_HRRR_lbcs_RAP +get_from_HPSS_ics_RAP_lbcs_RAP +get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS +grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR +#grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta +grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_RRFS_lbcs_RRFS_suite_RRFS_v1beta +grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP +grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0 +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 +grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot +long_fcst +MET_ensemble_verification_only_vx +MET_ensemble_verification_only_vx_time_lag +MET_ensemble_verification_winter_wx +MET_verification_only_vx +pregen_grid_orog_sfc_climo +specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS +specify_template_filenames diff --git a/tests/WE2E/machine_suites/coverage.derecho b/tests/WE2E/machine_suites/coverage.derecho index a948c76033..a10c865447 100644 --- a/tests/WE2E/machine_suites/coverage.derecho +++ b/tests/WE2E/machine_suites/coverage.derecho @@ -1,11 +1,11 @@ custom_ESGgrid_IndianOcean_6km grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16 +grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_HRRR_suite_HRRR pregen_grid_orog_sfc_climo specify_template_filenames -2019_hurricane_barry -2019_memorial_day_heat_wave -2020_denver_radiation_inversion diff --git a/tests/WE2E/machine_suites/coverage.hera.intel.nco b/tests/WE2E/machine_suites/coverage.hera.intel.nco index d5ab0d6fe8..e4b02a90ba 100644 --- a/tests/WE2E/machine_suites/coverage.hera.intel.nco +++ b/tests/WE2E/machine_suites/coverage.hera.intel.nco @@ -1,8 +1,8 @@ +2019_memorial_day_heat_wave custom_ESGgrid_Peru_12km get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_netcdf_2022040400_ensemble_2mems get_from_HPSS_ics_HRRR_lbcs_RAP -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 diff --git a/tests/WE2E/machine_suites/coverage.hercules b/tests/WE2E/machine_suites/coverage.hercules index ec37d81a56..668a26e685 100644 --- a/tests/WE2E/machine_suites/coverage.hercules +++ b/tests/WE2E/machine_suites/coverage.hercules @@ -4,9 +4,9 @@ grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v17_p8_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP -grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 MET_verification_only_vx specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS 2019_hurricane_lorenzo +2020_denver_radiation_inversion diff --git a/tests/WE2E/machine_suites/coverage.jet b/tests/WE2E/machine_suites/coverage.jet index 53308090b1..8c79a0b700 100644 --- a/tests/WE2E/machine_suites/coverage.jet +++ b/tests/WE2E/machine_suites/coverage.jet @@ -1,3 +1,4 @@ +2019_hurricane_barry community custom_ESGgrid custom_ESGgrid_Great_Lakes_snow_8km @@ -8,4 +9,3 @@ get_from_HPSS_ics_RAP_lbcs_RAP grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16_plot grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml index 2901d1ebf1..a9548f8b5f 100644 --- a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml @@ -34,6 +34,7 @@ task_get_extrn_lbcs: EXTRN_MDL_LBCS_OFFSET_HRS: 0 USE_USER_STAGED_EXTRN_FILES: true task_run_fcst: + OMP_NUM_THREADS_RUN_FCST: 1 DT_ATMOS: 180 LAYOUT_X: 50 LAYOUT_Y: 34 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml index de456cea73..f4c40bf722 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml @@ -20,5 +20,3 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 USE_USER_STAGED_EXTRN_FILES: true -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml index 4a340185f3..6d4dbc3b33 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml @@ -19,5 +19,3 @@ task_get_extrn_lbcs: EXTRN_MDL_NAME_LBCS: FV3GFS LBC_SPEC_INTVL_HRS: 3 USE_USER_STAGED_EXTRN_FILES: true -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml index 35be12a1ee..b00a24ae84 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml @@ -23,5 +23,3 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_FILES_LBCS: - '{yy}{jjj}{hh}00{fcst_hr:02d}00' -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml index 1265fa8e0c..44dfec5e75 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml @@ -24,5 +24,3 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_FILES_LBCS: - '{yy}{jjj}{hh}00{fcst_hr:02d}00' -task_run_fcst: - OMP_NUM_THREADS_RUN_FCST: 3 diff --git a/tests/test_python/test_create_model_configure_file.py b/tests/test_python/test_create_model_configure_file.py index d5aea79ed8..9475028505 100644 --- a/tests/test_python/test_create_model_configure_file.py +++ b/tests/test_python/test_create_model_configure_file.py @@ -43,11 +43,9 @@ def setUp(self): set_env_var("USHdir", USHdir) set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN) set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP) - set_env_var("PE_MEMBER01", 24) set_env_var("FCST_LEN_HRS", 72) set_env_var("FHROT", 0) set_env_var("DT_ATMOS", 1) - set_env_var("OMP_NUM_THREADS_RUN_FCST", 1) set_env_var("RESTART_INTERVAL", 4) set_env_var("ITASKS", 1) diff --git a/ush/config.aqm.yaml b/ush/config.aqm.yaml index 21a73591ee..77512ec7a2 100644 --- a/ush/config.aqm.yaml +++ b/ush/config.aqm.yaml @@ -46,6 +46,7 @@ task_get_extrn_lbcs: FV3GFS_FILE_FMT_LBCS: netcdf EXTRN_MDL_LBCS_OFFSET_HRS: 0 task_run_fcst: + OMP_NUM_THREADS_RUN_FCST: 1 DT_ATMOS: 180 LAYOUT_X: 50 LAYOUT_Y: 34 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 488c2d5d3d..6af52578b8 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1722,7 +1722,7 @@ task_run_fcst: #----------------------------------------------------------------------- # KMP_AFFINITY_RUN_FCST: "scatter" - OMP_NUM_THREADS_RUN_FCST: 2 # atmos_nthreads in model_configure + OMP_NUM_THREADS_RUN_FCST: 2 # ATM_omp_num_threads in ufs.configure OMP_STACKSIZE_RUN_FCST: "1024m" # #----------------------------------------------------------------------- @@ -1918,7 +1918,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group if QUILTING else LAYOUT_Y * LAYOUT_X}}' + PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X)}}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 9b430b83ae..e0e420c71b 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -70,7 +70,6 @@ def create_model_configure_file( # ----------------------------------------------------------------------- # settings = { - "PE_MEMBER01": PE_MEMBER01, "start_year": cdate.year, "start_month": cdate.month, "start_day": cdate.day, @@ -78,7 +77,6 @@ def create_model_configure_file( "nhours_fcst": fcst_len_hrs, "fhrot": fhrot, "dt_atmos": DT_ATMOS, - "atmos_nthreads": OMP_NUM_THREADS_RUN_FCST, "restart_interval": RESTART_INTERVAL, "itasks": ITASKS, "write_dopost": f".{lowercase(str(WRITE_DOPOST))}.", diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index dc6a43420d..e97fa25c6a 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -50,6 +50,8 @@ def create_ufs_configure_file(run_dir): # Set output file path # ufs_config_fp = os.path.join(run_dir, UFS_CONFIG_FN) + pe_member01_m1 = str(int(PE_MEMBER01)-1) + aqm_pe_member01_m1 = str(int(LAYOUT_X*LAYOUT_Y)-1) # #----------------------------------------------------------------------- # @@ -62,7 +64,10 @@ def create_ufs_configure_file(run_dir): settings = { "dt_atmos": DT_ATMOS, "print_esmf": PRINT_ESMF, - "cpl_aqm": CPL_AQM + "cpl_aqm": CPL_AQM, + "pe_member01_m1": pe_member01_m1, + "aqm_pe_member01_m1": aqm_pe_member01_m1, + "atm_omp_num_threads": OMP_NUM_THREADS_RUN_FCST, } settings_str = cfg_to_yaml_str(settings) diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 1ec2ded2ef..92d33d7ad2 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -12,7 +12,7 @@ platform: QUEUE_HPSS: normal REMOVE_MEMORY: True PARTITION_HPSS: eslogin_c5 - RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL -n $nprocs RUN_CMD_PRDGEN: srun --export=ALL -n $nprocs RUN_CMD_SERIAL: time @@ -47,9 +47,9 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + cores: native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' - nodes: + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' nnodes: nodesize: ppn: diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 80fbb8fc98..189689f30d 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -48,9 +48,9 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + cores: native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' - nodes: + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' nnodes: nodesize: ppn: diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index e29801dd49..6a325094da 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -13,7 +13,7 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL RUN_CMD_SERIAL: time @@ -51,3 +51,14 @@ cpl_aqm_parm: COMINfire_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/RAVE_fire COMINgefs_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GEFS_DATA NEXUS_GFS_SFC_DIR: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GFS_SFC_DATA + +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + cores: + native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' + nnodes: + nodesize: + ppn: diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 375711c61a..847530e4eb 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -7,7 +7,7 @@ platform: TEST_NDAS_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc TEST_NOHRSC_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/nohrsc/proc DOMAIN_PREGEN_BASEDIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen - PARTITION_DEFAULT: sjet,vjet,kjet,xjet + PARTITION_DEFAULT: vjet,kjet,xjet QUEUE_DEFAULT: batch PARTITION_FCST: xjet QUEUE_FCST: batch @@ -49,9 +49,9 @@ rocoto: tasks: metatask_run_ensemble: task_run_fcst_mem#mem#: - cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}' + cores: native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' - nodes: + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' nnodes: nodesize: ppn: diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 3f756e2836..5467160167 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -13,7 +13,7 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01} + RUN_CMD_FCST: srun --export=ALL RUN_CMD_POST: srun --export=ALL RUN_CMD_PRDGEN: srun --export=ALL RUN_CMD_SERIAL: time @@ -50,3 +50,14 @@ cpl_aqm_parm: COMINfire_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/RAVE_fire COMINgefs_default: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GEFS_DATA NEXUS_GFS_SFC_DIR: /work/noaa/epic/SRW-AQM_DATA/aqm_data/GFS_SFC_DATA + +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + cores: + native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}' + nodes: '{{ task_run_fcst.NNODES_RUN_FCST // 1 }}:ppn={{ task_run_fcst.PPN_RUN_FCST // 1 }}' + nnodes: + nodesize: + ppn: