diff --git a/modulefiles/tasks/hera/task.analysis.lua b/modulefiles/tasks/hera/task.analysis.lua index 84d59acd..22b031e9 100644 --- a/modulefiles/tasks/hera/task.analysis.lua +++ b/modulefiles/tasks/hera/task.analysis.lua @@ -5,6 +5,9 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("prod_util", prod_util_ver)) + +load(pathJoin("py-jinja2", py_jinja2_ver)) load(pathJoin("py-netcdf4", py_netcdf4_ver)) load(pathJoin("py-numpy", py_numpy_ver)) +load(pathJoin("py-pyyaml", py_pyyaml_ver)) diff --git a/modulefiles/tasks/hera/task.forecast.lua b/modulefiles/tasks/hera/task.forecast.lua index cd767192..81925277 100644 --- a/modulefiles/tasks/hera/task.forecast.lua +++ b/modulefiles/tasks/hera/task.forecast.lua @@ -23,6 +23,9 @@ load(pathJoin("w3emc", w3emc_ver)) load(pathJoin("gftl-shared", gftl_shared_ver)) load(pathJoin("mapl", mapl_ver)) load(pathJoin("prod_util", prod_util_ver)) + +load(pathJoin("py-jinja2", py_jinja2_ver)) load(pathJoin("py-netcdf4", py_netcdf4_ver)) load(pathJoin("py-numpy", py_numpy_ver)) +load(pathJoin("py-pyyaml", py_pyyaml_ver)) diff --git a/modulefiles/tasks/hera/task.post_anal.lua b/modulefiles/tasks/hera/task.post_anal.lua index 3b8b3c7b..6c75de7b 100644 --- a/modulefiles/tasks/hera/task.post_anal.lua +++ b/modulefiles/tasks/hera/task.post_anal.lua @@ -8,6 +8,9 @@ load(pathJoin("hdf5", hdf5_ver)) load(pathJoin("netcdf-c", netcdf_c_ver)) load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) load(pathJoin("prod_util", prod_util_ver)) + +load(pathJoin("py-jinja2", py_jinja2_ver)) load(pathJoin("py-netcdf4", py_netcdf4_ver)) load(pathJoin("py-numpy", py_numpy_ver)) +load(pathJoin("py-pyyaml", py_pyyaml_ver)) diff --git a/modulefiles/tasks/hera/task.pre_anal.lua b/modulefiles/tasks/hera/task.pre_anal.lua index 31afb0c9..b77ea68d 100644 --- a/modulefiles/tasks/hera/task.pre_anal.lua +++ b/modulefiles/tasks/hera/task.pre_anal.lua @@ -8,3 +8,7 @@ load(pathJoin("hdf5", hdf5_ver)) load(pathJoin("netcdf-c", netcdf_c_ver)) load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) load(pathJoin("prod_util", prod_util_ver)) + +load(pathJoin("py-jinja2", py_jinja2_ver)) +load(pathJoin("py-pyyaml", py_pyyaml_ver)) + diff --git a/parm/jedi/GHCN.yaml b/parm/jedi/GHCN.yaml index 2b8c725d..9becb527 100644 --- a/parm/jedi/GHCN.yaml +++ b/parm/jedi/GHCN.yaml @@ -7,11 +7,11 @@ obsdatain: engine: type: H5File - obsfile: GHCN_XXYYYYXXMMXXDDXXHH.nc + obsfile: GHCN_{{ yyyymmddhh }}.nc obsdataout: engine: type: H5File - obsfile: output/DA/hofx/letkf_hofx_ghcn_XXYYYYXXMMXXDDXXHH.nc + obsfile: output/DA/hofx/letkf_hofx_ghcn_{{ yyyymmddhh }}.nc obs operator: name: Identity obs error: diff --git a/parm/jedi/letkfoi_snow.yaml b/parm/jedi/letkfoi_snow.yaml index 330bde46..f71d40b3 100644 --- a/parm/jedi/letkfoi_snow.yaml +++ b/parm/jedi/letkfoi_snow.yaml @@ -3,45 +3,45 @@ geometry: namelist filename: Data/fv3files/fmsmpp.nml field table filename: Data/fv3files/field_table akbk: Data/fv3files/akbk64.nc4 - npx: XXREP - npy: XXREP + npx: {{ resp1 }} + npy: {{ resp1 }} npz: 64 field metadata override: gfs-land.yaml time invariant fields: state fields: - datetime: XXYYYP-XXMP-XXDPTXXHP:00:00Z + datetime: {{ yyyp }}-{{ mp }}-{{ dp }}T{{ hp }}:00:00Z filetype: fms restart skip coupler file: true state variables: [orog_filt] - datapath: XXTPATH - filename_orog: XXTSTUB.nc + datapath: {{ tpath }} + filename_orog: {{ tstub }}.nc derived fields: [nominal_surface_pressure] time window: - begin: XXYYYP-XXMP-XXDPTXXHP:00:00Z + begin: {{ yyyp }}-{{ mp }}-{{ dp }}T{{ hp }}:00:00Z length: PT24H background: - date: &date XXYYYY-XXMM-XXDDTXXHH:00:00Z + date: &date {{ yyyy }}-{{ mm }}-{{ dd }}T{{ hh }}:00:00Z members: - - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z + - datetime: {{ yyyy }}-{{ mm }}-{{ dd }}T{{ hh }}:00:00Z filetype: fms restart state variables: [snwdph,vtype,slmsk] datapath: mem_pos/ - filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc - filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res - - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z + filename_sfcd: {{ yyyymmdd }}.{{ hh }}0000.sfc_data.nc + filename_cplr: {{ yyyymmdd }}.{{ hh }}0000.coupler.res + - datetime: {{ yyyy }}-{{ mm }}-{{ dd }}T{{ hh }}:00:00Z filetype: fms restart state variables: [snwdph,vtype,slmsk] datapath: mem_neg/ - filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc - filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res + filename_sfcd: {{ yyyymmdd }}.{{ hh }}0000.sfc_data.nc + filename_cplr: {{ yyyymmdd }}.{{ hh }}0000.coupler.res driver: save posterior mean: false save posterior mean increment: true save posterior ensemble: false - run as observer only: XXHOFX + run as observer only: {{ driver_obs_only }} local ensemble DA: solver: LETKF diff --git a/parm/templates/template.coupler.res b/parm/templates/template.coupler.res index a3dc1afe..652666da 100644 --- a/parm/templates/template.coupler.res +++ b/parm/templates/template.coupler.res @@ -1,4 +1,4 @@ 2 (Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4) - XXYYYP XXMP XXDP XXHP 0 0 Model start time: year, month, day, hour, minute, second - XXYYYY XXMM XXDD XXHH 0 0 Current model time: year, month, day, hour, minute, second + {{ yyyp }} {{ mp }} {{ dp }} {{ hp }} 0 0 Model start time: year, month, day, hour, minute, second + {{ yyyy }} {{ mm }} {{ dd }} {{ hh }} 0 0 Current model time: year, month, day, hour, minute, second diff --git a/parm/templates/template.diag_table b/parm/templates/template.diag_table index ac60ce36..b4206d0f 100644 --- a/parm/templates/template.diag_table +++ b/parm/templates/template.diag_table @@ -1,5 +1,5 @@ -XXYYYYMMDD.00Z.1760x880.64bit.non-mono -XXYYYY XXMM XXDD XXHH 0 0 +{{ yyyymmdd }}.00Z.1760x880.64bit.non-mono +{{ yyyy }} {{ mm }} {{ dd }} {{ hh }} 0 0 "fv3_history", 0, "hours", 1, "hours", "time" "fv3_history2d", 0, "hours", 1, "hours", "time" diff --git a/parm/templates/template.jedi2ufs b/parm/templates/template.jedi2ufs index efc66308..4f31a316 100644 --- a/parm/templates/template.jedi2ufs +++ b/parm/templates/template.jedi2ufs @@ -8,17 +8,17 @@ ! FV3 resolution and path to oro files for restart/perturbation conversion - tile_size = XXRES - tile_path = "FIXlandda/FV3_fix_tiled/CXXRES/" - tile_fstub = "XXTSTUB" + tile_size = {{ res }} + tile_path = "{{ fix_landda }}/FV3_fix_tiled/C{{ res }}/" + tile_fstub = "{{ tstub }}" !------------------- only restart conversion ------------------- ! Time stamp for conversion for restart conversion - restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + restart_date = "{{ yyyy }}-{{ mm }}-{{ dd }} {{ hh }}:00:00" ! Path for static file - static_filename = "FIXlandda/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "{{ fix_landda }}/static/ufs-land_C{{ res }}_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/parm/templates/template.model_configure b/parm/templates/template.model_configure index 15e4cf42..07883611 100644 --- a/parm/templates/template.model_configure +++ b/parm/templates/template.model_configure @@ -1,9 +1,9 @@ -start_year: XXYYYY -start_month: XXMM -start_day: XXDD -start_hour: XXHH +start_year: {{ yyyy }} +start_month: {{ mm }} +start_day: {{ dd }} +start_hour: {{ hh }} start_minute: 0 start_second: 0 -nhours_fcst: XXFCSTHR -dt_atmos: XXDT_ATMOS +nhours_fcst: {{ fcsthr }} +dt_atmos: {{ dt_atmos }} fhrot: 0 diff --git a/parm/templates/template.ufs.configure b/parm/templates/template.ufs.configure index 0e1cb40b..792bd679 100644 --- a/parm/templates/template.ufs.configure +++ b/parm/templates/template.ufs.configure @@ -14,7 +14,7 @@ EARTH_attributes:: # MED # MED_model: cmeps -MED_petlist_bounds: 0 XXNPROCS_ATM_M1 +MED_petlist_bounds: 0 {{ nprocs_atm_m1 }} MED_omp_num_threads: 1 MED_attributes:: Verbosity = 1 @@ -30,7 +30,7 @@ MED_attributes:: # ATM # ATM_model: datm -ATM_petlist_bounds: 0 XXNPROCS_ATM_M1 +ATM_petlist_bounds: 0 {{ nprocs_atm_m1 }} ATM_omp_num_threads: 1 ATM_attributes:: Verbosity = 0 @@ -39,7 +39,7 @@ ATM_attributes:: # LND # LND_model: noahmp -LND_petlist_bounds: XXNPROCS_FORECAST_ATM XXNPROCS_ATM_LND_M1 +LND_petlist_bounds: {{ nprocs_forecast_atm }} {{ nprocs_atm_lnd_m1 }} LND_omp_num_threads: 1 LND_attributes:: Verbosity = 1 @@ -47,7 +47,7 @@ LND_attributes:: mosaic_file = INPUT/C96_mosaic.nc input_dir = INPUT/ ic_type = custom - layout = XXLND_LAYOUT_X:XXLND_LAYOUT_Y # need to be consistent with number of PEs (6*Lx*Ly) + layout = {{ lnd_layout_x }}:{{ lnd_layout_y }} # need to be consistent with number of PEs (6*Lx*Ly) num_soil_levels = 4 forcing_height = 10 soil_level_thickness = 0.10:0.30:0.60:1.00 @@ -67,7 +67,7 @@ LND_attributes:: surface_evap_resistance_option = 1 # not used, it is fixed to 4 in sfc_noahmp_drv.F90 glacier_option = 1 surface_thermal_roughness_option = 2 - output_freq = XXLND_OUTPUT_FREQ_SEC + output_freq = {{ lnd_output_freq_sec }} restart_freq = -1 calc_snet = .true. initial_albedo = 0.25 @@ -75,7 +75,7 @@ LND_attributes:: # cold runSeq:: -@XXDT_RUNSEQ +@{{ dt_runseq }} MED med_phases_prep_atm MED -> ATM :remapMethod=redist ATM diff --git a/parm/templates/template.ufs2jedi b/parm/templates/template.ufs2jedi index 82576340..9c4c5a81 100644 --- a/parm/templates/template.ufs2jedi +++ b/parm/templates/template.ufs2jedi @@ -8,17 +8,17 @@ ! FV3 resolution and path to oro files for restart/perturbation conversion - tile_size = XXRES - tile_path = "FIXlandda/FV3_fix_tiled/CXXRES/" - tile_fstub = "XXTSTUB" + tile_size = {{ res }} + tile_path = "{{ fix_landda }}/FV3_fix_tiled/C{{ res }}/" + tile_fstub = "{{ tstub }}" !------------------- only restart conversion ------------------- ! Time stamp for conversion for restart conversion - restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" + restart_date = "{{ yyyy }}-{{ mm }}-{{ dd }} {{ hh }}:00:00" ! Path for static file - static_filename = "FIXlandda/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "{{ fix_landda }}/static/ufs-land_C{{ res }}_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh index 6f76cb98..c768e114 100755 --- a/scripts/exlandda_analysis.sh +++ b/scripts/exlandda_analysis.sh @@ -43,17 +43,21 @@ do done ln -nsf ${COMIN}/obs/*_${YYYY}${MM}${DD}${HH}.nc . -cres_file=${DATA}/${FILEDATE}.coupler.res -cp ${PARMlandda}/templates/template.coupler.res $cres_file - -sed -i -e "s/XXYYYY/${YYYY}/g" $cres_file -sed -i -e "s/XXMM/${MM}/g" $cres_file -sed -i -e "s/XXDD/${DD}/g" $cres_file -sed -i -e "s/XXHH/${HH}/g" $cres_file -sed -i -e "s/XXYYYP/${YYYP}/g" $cres_file -sed -i -e "s/XXMP/${MP}/g" $cres_file -sed -i -e "s/XXDP/${DP}/g" $cres_file -sed -i -e "s/XXHP/${HP}/g" $cres_file +# update coupler.res file +settings="\ + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} + 'yyyp': !!str ${YYYP} + 'mp': !!str ${MP} + 'dp': !!str ${DP} + 'hp': !!str ${HP} +" # End of settins variable + +fp_template="${PARMlandda}/templates/template.coupler.res" +fn_namelist="${DATA}/${FILEDATE}.coupler.res" +${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" ################################################ # CREATE BACKGROUND ENSEMBLE (LETKFOI) @@ -95,11 +99,7 @@ fi do_DA="YES" do_HOFX="NO" - -if [[ $do_DA == "NO" && $do_HOFX == "NO" ]]; then - echo "do_landDA:No obs found, not calling JEDI" - exit 0 -fi +RESP1=$((RES+1)) mkdir -p output/DA/hofx # if yaml is specified by user, use that. Otherwise, build the yaml @@ -116,20 +116,28 @@ if [[ $do_DA == "YES" ]]; then cp ${PARMlandda}/jedi/${YAML_DA} ${DATA}/letkf_land.yaml fi - sed -i -e "s/XXYYYY/${YYYY}/g" letkf_land.yaml - sed -i -e "s/XXMM/${MM}/g" letkf_land.yaml - sed -i -e "s/XXDD/${DD}/g" letkf_land.yaml - sed -i -e "s/XXHH/${HH}/g" letkf_land.yaml - sed -i -e "s/XXYYYP/${YYYP}/g" letkf_land.yaml - sed -i -e "s/XXMP/${MP}/g" letkf_land.yaml - sed -i -e "s/XXDP/${DP}/g" letkf_land.yaml - sed -i -e "s/XXHP/${HP}/g" letkf_land.yaml - sed -i -e "s/XXTSTUB/${TSTUB}/g" letkf_land.yaml - sed -i -e "s#XXTPATH#${TPATH}#g" letkf_land.yaml - sed -i -e "s/XXRES/${RES}/g" letkf_land.yaml - RESP1=$((RES+1)) - sed -i -e "s/XXREP/${RESP1}/g" letkf_land.yaml - sed -i -e "s/XXHOFX/false/g" letkf_land.yaml # do DA + # update jedi yaml file + settings="\ + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} + 'yyyymmdd': !!str ${PDY} + 'yyyymmddhh': !!str ${PDY}${cyc} + 'yyyp': !!str ${YYYP} + 'mp': !!str ${MP} + 'dp': !!str ${DP} + 'hp': !!str ${HP} + 'tstub': ${TSTUB} + 'tpath': ${TPATH} + 'res': ${RES} + 'resp1': ${RESP1} + 'driver_obs_only': false + " # End of settins variable + + fp_template="${DATA}/letkf_land.yaml" + fn_namelist="${DATA}/letkf_land.yaml" + ${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" fi if [[ $do_HOFX == "YES" ]]; then @@ -145,21 +153,28 @@ if [[ $do_HOFX == "YES" ]]; then cp ${PARMlandda}/jedi/${YAML_HOFX} ${DATA}/hofx_land.yaml fi - sed -i -e "s/XXYYYY/${YYYY}/g" hofx_land.yaml - sed -i -e "s/XXMM/${MM}/g" hofx_land.yaml - sed -i -e "s/XXDD/${DD}/g" hofx_land.yaml - sed -i -e "s/XXHH/${HH}/g" hofx_land.yaml - sed -i -e "s/XXYYYP/${YYYP}/g" hofx_land.yaml - sed -i -e "s/XXMP/${MP}/g" hofx_land.yaml - sed -i -e "s/XXDP/${DP}/g" hofx_land.yaml - sed -i -e "s/XXHP/${HP}/g" hofx_land.yaml - sed -i -e "s#XXTPATH#${TPATH}#g" hofx_land.yaml - sed -i -e "s/XXTSTUB/${TSTUB}/g" hofx_land.yaml - sed -i -e "s/XXRES/${RES}/g" hofx_land.yaml - RESP1=$((RES+1)) - sed -i -e "s/XXREP/${RESP1}/g" hofx_land.yaml - sed -i -e "s/XXHOFX/true/g" hofx_land.yaml # do HOFX - + # update jedi yaml file + settings="\ + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} + 'yyyymmdd': !!str ${PDY} + 'yyyymmddhh': !!str ${PDY}${cyc} + 'yyyp': !!str ${YYYP} + 'mp': !!str ${MP} + 'dp': !!str ${DP} + 'hp': !!str ${HP} + 'tstub': ${TSTUB} + 'tpath': ${TPATH} + 'res': ${RES} + 'resp1': ${RESP1} + 'driver_obs_only': true + " # End of settins variable + + fp_template="${DATA}/hofx_land.yaml" + fn_namelist="${DATA}/hofx_land.yaml" + ${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" fi if [[ "$GFSv17" == "NO" ]]; then diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh index a660d935..3ecdc317 100755 --- a/scripts/exlandda_forecast.sh +++ b/scripts/exlandda_forecast.sh @@ -51,33 +51,49 @@ cp -p ${PARMlandda}/templates/template.fd_ufs.yaml fd_ufs.yaml cp -p ${PARMlandda}/templates/template.data_table data_table # Set ufs.configure -cp -p ${PARMlandda}/templates/template.ufs.configure ufs.configure nprocs_atm_m1=$(( NPROCS_FORECAST_ATM - 1 )) -sed -i -e "s/XXNPROCS_ATM_M1/${nprocs_atm_m1}/g" ufs.configure -sed -i -e "s/XXNPROCS_FORECAST_ATM/${NPROCS_FORECAST_ATM}/g" ufs.configure nprocs_atm_lnd_m1=$(( NPROCS_FORECAST_ATM + NPROCS_FORECAST_LND - 1 )) -sed -i -e "s/XXNPROCS_ATM_LND_M1/${nprocs_atm_lnd_m1}/g" ufs.configure -sed -i -e "s/XXLND_LAYOUT_X/${LND_LAYOUT_X}/g" ufs.configure -sed -i -e "s/XXLND_LAYOUT_Y/${LND_LAYOUT_Y}/g" ufs.configure -sed -i -e "s/XXLND_OUTPUT_FREQ_SEC/${LND_OUTPUT_FREQ_SEC}/g" ufs.configure -sed -i -e "s/XXDT_RUNSEQ/${DT_RUNSEQ}/g" ufs.configure + +settings="\ + 'nprocs_atm_m1': ${nprocs_atm_m1} + 'nprocs_forecast_atm': ${NPROCS_FORECAST_ATM} + 'nprocs_atm_lnd_m1': ${nprocs_atm_lnd_m1} + 'lnd_layout_x': ${LND_LAYOUT_X} + 'lnd_layout_y': ${LND_LAYOUT_Y} + 'lnd_output_freq_sec': ${LND_OUTPUT_FREQ_SEC} + 'dt_runseq': ${DT_RUNSEQ} +" # End of settins variable + +fp_template="${PARMlandda}/templates/template.ufs.configure" +fn_namelist="ufs.configure" +${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" # Set model_configure -cp -p ${PARMlandda}/templates/template.model_configure model_configure -sed -i -e "s/XXYYYY/${YYYY}/g" model_configure -sed -i -e "s/XXMM/${MM}/g" model_configure -sed -i -e "s/XXDD/${DD}/g" model_configure -sed -i -e "s/XXHH/${HH}/g" model_configure -sed -i -e "s/XXFCSTHR/${FCSTHR}/g" model_configure -sed -i -e "s/XXDT_ATMOS/${DT_ATMOS}/g" model_configure +settings="\ + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} + 'fcsthr': ${FCSTHR} + 'dt_atmos': ${DT_ATMOS} +" # End of settins variable + +fp_template="${PARMlandda}/templates/template.model_configure" +fn_namelist="model_configure" +${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" # set diag table -cp -p ${PARMlandda}/templates/template.diag_table diag_table -sed -i -e "s/XXYYYYMMDD/${YYYYMMDD}/g" diag_table -sed -i -e "s/XXYYYY/${YYYY}/g" diag_table -sed -i -e "s/XXMM/${MM}/g" diag_table -sed -i -e "s/XXDD/${DD}/g" diag_table -sed -i -e "s/XXHH/${HH}/g" diag_table +settings="\ + 'yyyymmdd': !!str ${YYYYMMDD} + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} +" # End of settins variable + +fp_template="${PARMlandda}/templates/template.diag_table" +fn_namelist="diag_table" +${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" # Set up the run directory mkdir -p RESTART diff --git a/scripts/exlandda_post_anal.sh b/scripts/exlandda_post_anal.sh index 7cd62552..7fcb419c 100755 --- a/scripts/exlandda_post_anal.sh +++ b/scripts/exlandda_post_anal.sh @@ -50,25 +50,30 @@ do cp ${DATA_SHARE}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${itile}.nc . done - cp ${PARMlandda}/templates/template.jedi2ufs jedi2ufs.namelist - - sed -i "s|FIXlandda|${FIXlandda}|g" jedi2ufs.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist - sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist - sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist - sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist - sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist +# update tile2tile namelist +settings="\ + 'fix_landda': ${FIXlandda} + 'res': ${RES} + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} + 'tstub': ${TSTUB} +" # End of settins variable - export pgm="tile2tile_converter.exe" - . prep_step - ${EXEClandda}/$pgm jedi2ufs.namelist >>$pgmout 2>errfile - export err=$?; err_chk - cp errfile errfile_tile2tile - if [[ $err != 0 ]]; then - echo "tile2tile failed" - exit 10 - fi +fp_template="${PARMlandda}/templates/template.jedi2ufs" +fn_namelist="jedi2ufs.namelist" +${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" + +export pgm="tile2tile_converter.exe" +. prep_step +${EXEClandda}/$pgm jedi2ufs.namelist >>$pgmout 2>errfile +export err=$?; err_chk +cp errfile errfile_tile2tile +if [[ $err != 0 ]]; then + echo "tile2tile failed" + exit 10 +fi # save analysis restart for itile in {1..6} diff --git a/scripts/exlandda_pre_anal.sh b/scripts/exlandda_pre_anal.sh index 7a9e7cfd..9007d92f 100755 --- a/scripts/exlandda_pre_anal.sh +++ b/scripts/exlandda_pre_anal.sh @@ -33,28 +33,31 @@ do cp -p ${rst_fn} ${DATA_SHARE} done - # update tile2tile namelist - cp ${PARMlandda}/templates/template.ufs2jedi ufs2jedi.namelist +# update tile2tile namelist +settings="\ + 'fix_landda': ${FIXlandda} + 'res': ${RES} + 'yyyy': !!str ${YYYY} + 'mm': !!str ${MM} + 'dd': !!str ${DD} + 'hh': !!str ${HH} + 'tstub': ${TSTUB} +" # End of settins variable - sed -i "s|FIXlandda|${FIXlandda}|g" ufs2jedi.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist - sed -i -e "s/XXMM/${MM}/g" ufs2jedi.namelist - sed -i -e "s/XXDD/${DD}/g" ufs2jedi.namelist - sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist - sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist - sed -i -e "s/XXRES/${RES}/g" ufs2jedi.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" ufs2jedi.namelist +fp_template="${PARMlandda}/templates/template.ufs2jedi" +fn_namelist="ufs2jedi.namelist" +${USHlandda}/fill_jinja_template.py -u "${settings}" -t "${fp_template}" -o "${fn_namelist}" - # submit tile2tile - export pgm="tile2tile_converter.exe" - . prep_step - ${EXEClandda}/$pgm ufs2jedi.namelist >>$pgmout 2>errfile - cp errfile errfile_tile2tile - export err=$?; err_chk - if [[ $err != 0 ]]; then - echo "tile2tile failed" - exit 22 - fi +# submit tile2tile +export pgm="tile2tile_converter.exe" +. prep_step +${EXEClandda}/$pgm ufs2jedi.namelist >>$pgmout 2>errfile +cp errfile errfile_tile2tile +export err=$?; err_chk +if [[ $err != 0 ]]; then + echo "tile2tile failed" + exit 22 +fi #stage restarts for applying JEDI update to intermediate directory for itile in {1..6} diff --git a/ush/fill_jinja_template.py b/ush/fill_jinja_template.py new file mode 100755 index 00000000..f8101367 --- /dev/null +++ b/ush/fill_jinja_template.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 + +""" +This utility fills in a user-supplied Jinja template from either a YAML file, or +command line arguments. + +The user configuration file and commandline arguments should be YAML-formatted. +This script will support a single- or two-level YAML config file. For example: + + 1. expt1: + date_first_cycl: !datetime 2019043000 + date_last_cycl: !datetime 2019050100 + cycl_freq: !!str 12:00:00 + + expt2: + date_first_cycl: !datetime 2019061012 + date_last_cycl: !datetime 2019061212 + cycl_freq: !!str 12:00:00 + + 2. date_first_cycl: !datetime 2019043000 + date_last_cycl: !datetime 2019050100 + cycl_freq: !!str 12:00:00 + + In Case 1, provide the name of the file and the section title, e.g. expt2, + to the -c command line argument. Only provide the name of the file in -c + option if it's configured as in Case 2 above. + + +Supported YAML Tags: + + The script supports additional YAML configuration tags. + + !datetime Converts an input string formatted as YYYYMMDDHH[mm[ss]] to a + Python datetime object + !join Uses os.path.join to join a list as a path. + +Expected behavior: + + - The template file is required. Script fails if not provided. + - Command line arguments in the -u setting override the -c settings. + +""" + +import datetime as dt +import os +import sys + +import argparse +import jinja2 as j2 +from jinja2 import meta +import yaml + + +def join(loader, node): + + """Uses os to join a list as a path.""" + + return os.path.join(*loader.construct_sequence(node)) + + +def to_datetime(loader, node): + + """Converts a date string with format YYYYMMDDHH[MM[SS]] to a datetime + object.""" + + value = loader.construct_scalar(node) + val_len = len(value) + + # Check that the input string contains only numbers and is expected length. + if val_len not in [10, 12, 14] or not value.isnumeric(): + msg = f"{value} does not conform to input format YYYYMMDDHH[MM[SS]]" + raise ValueError(msg) + + # Use a subset of the string corresponding to the input length of the string + # 2 chosen here since Y is a 4 char year. + date_format = "%Y%m%d%H%M%S"[0 : val_len - 2] + + return dt.datetime.strptime(value, date_format) + + +yaml.add_constructor("!datetime", to_datetime, Loader=yaml.SafeLoader) +yaml.add_constructor("!join", join, Loader=yaml.SafeLoader) + + +def file_exists(arg): + + """Checks whether a file exists, and returns the path if it does.""" + + if not os.path.exists(arg): + msg = f"{arg} does not exist!" + raise argparse.ArgumentTypeError(msg) + + return arg + + +def config_exists(arg): + + """ + Checks whether the config file exists and if it contains the input + section. Returns the config as a Python dict. + """ + + if len(arg) > 2: + msg = f"{len(arg)} arguments were provided for config. Only 2 allowed!" + raise argparse.ArgumentTypeError(msg) + + file_name = file_exists(arg[0]) + section_name = arg[1] if len(arg) == 2 else None + + # Load the YAML file into a dictionary + with open(file_name, "r") as fn: + cfg = yaml.load(fn, Loader=yaml.SafeLoader) + + if section_name: + try: + cfg = cfg[section_name] + except KeyError: + msg = f"Section {section_name} does not exist in top level of {file_name}" + raise argparse.ArgumentTypeError(msg) + + return cfg + + +def load_config(arg): + + """ + Check to ensure that the provided config file exists. If it does, load it + with YAML's safe loader and return the resulting dict. + """ + + # Check for existence of file + if not os.path.exists(arg): + msg = f"{arg} does not exist!" + raise argparse.ArgumentTypeError(msg) + + return yaml.safe_load(arg) + + +def load_str(arg): + + """Load a dict string safely using YAML. Return the resulting dict.""" + + return yaml.load(arg, Loader=yaml.SafeLoader) + + +def path_ok(arg): + + """ + Check whether the path to the file exists, and is writeable. Return the path + if it passes all checks, otherwise raise an error. + """ + + # Get the absolute path provided by arg + dir_name = os.path.abspath(os.path.dirname(arg)) + + # Ensure the arg path exists, and is writable. Raise error if not. + if os.path.lexists(dir_name) and os.access(dir_name, os.W_OK): + return arg + + msg = f"{arg} is not a writable path!" + raise argparse.ArgumentTypeError(msg) + + +def parse_args(argv): + + """ + Function maintains the arguments accepted by this script. Please see + Python's argparse documenation for more information about settings of each + argument. + """ + + parser = argparse.ArgumentParser(description="Fill in a Rocoto XML template.") + + # Optional + parser.add_argument( + "-c", + "--config", + help="Full path to a YAML user config file, and a \ + top-level section to use (optional).", + nargs="*", + type=load_config, + ) + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress all output", + ) + parser.add_argument( + "-u", + "--user_config", + help="Command-line user config options in YAML-formatted string", + type=load_str, + ) + # Required + parser.add_argument( + "-t", + "--xml_template", + dest="template", + help="Full path to the jinja template", + required=True, + type=file_exists, + ) + parser.add_argument( + "-o", + "--outxml", + dest="outxml", + help="Full path to the output Rocoto XML file.", + required=True, + type=path_ok, + ) + return parser.parse_args(argv) + + +def update_dict(dest, newdict, quiet=False): + + """ + Overwrites all values in dest dictionary section with key/value pairs from + newdict. Does not support multi-layer update. + + Turn off print statements with quiet=True. + + Input: + dest A dict that is to be updated. + newdict A dict containing sections and keys corresponding to + those in dest and potentially additional ones, that will be used to + update the dest dict. + quiet An optional boolean flag to turn off output. + Output: + None + Result: + The dest dict is updated in place. + """ + + if not quiet: + print("*" * 50) + + for key, value in newdict.items(): + if not quiet: + print(f"Overriding {key:>20} = {value}") + + # Set key in dict + dest[key] = value + + if not quiet: + print("*" * 50) + + +def fill_jinja_template(argv, config_dict=None): + + """ + Loads a Jinja template, determines its necessary undefined variables, + retrives them from user supplied settings, and renders the final result. + """ + + # parse args + cla = parse_args(argv) + if cla.config: + cla.config = config_exists(cla.config) + + # Create a Jinja Environment to load the template. + env = j2.Environment(loader=j2.FileSystemLoader(cla.template, + encoding='utf-8')) + template_source = env.loader.get_source(env, "") + template = env.get_template("") + parsed_content = env.parse(template_source) + + # Gather all of the undefined variables in the template. + template_vars = meta.find_undeclared_variables(parsed_content) + + # Read in the config options from the provided (optional) YAML file + cfg = cla.config if cla.config is not None else {} + + if config_dict is not None: + update_dict(cfg, config_dict, quiet=cla.quiet) + + # Update cfg with (optional) command-line entries, overriding those in YAML file + if cla.user_config: + update_dict(cfg, cla.user_config, quiet=cla.quiet) + + # Loop through all the undefined Jinja template variables, and grab the + # required values from the config file. + tvars = {} + for var in template_vars: + + if cfg.get(var, "NULL") == "NULL": + raise KeyError(f"{var} does not exist in user-supplied settings!") + + if not cla.quiet: + print(f"{var:>25}: {cfg.get(var)}") + + tvars[var] = cfg.get(var) + + # Fill in XML template + xml_contents = template.render(**tvars) + with open(cla.outxml, "w") as fn: + fn.write(xml_contents) + + +if __name__ == "__main__": + + fill_jinja_template(sys.argv[1:])