Add time-history plots for stats (#151)

* add python for plotting time-history * update scripts * fix wrong parms * fix incorrect var names * fix missing env * remove legend from plot * add nobs plot * change we2e_vav to we2e_test * remove templates for plot and set yaml with cat eof
ufs-community · Oct 22, 2024 · 5f5b152 · 5f5b152
1 parent d8b7ea8
commit 5f5b152
Show file tree

Hide file tree

Showing 14 changed files with 388 additions and 61 deletions.
diff --git a/modulefiles/tasks/hera/task.plot_stats.lua b/modulefiles/tasks/hera/task.plot_stats.lua
@@ -9,6 +9,7 @@ load(pathJoin("py-cartopy", py_cartopy_ver))
 load(pathJoin("py-matplotlib", py_matplotlib_ver))
 load(pathJoin("py-netcdf4", py_netcdf4_ver))
 load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pandas", py_pandas_ver))
 load(pathJoin("py-pyyaml", py_pyyaml_ver))
 load(pathJoin("py-scipy", py_scipy_ver))
 load(pathJoin("py-xarray", py_xarray_ver))

diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -34,7 +34,7 @@ workflow:
     OBS_TYPES: "GHCN"
     DAtype: "letkfoi_snow"
     TSTUB: "oro_C96.mx100"
-    WE2E_VAV: "YES"
+    WE2E_TEST: "NO"
     WE2E_ATOL: "1e-7"
     WE2E_LOG_FN: "we2e.log"
     NET: "landda"
@@ -137,7 +137,7 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -172,7 +172,7 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -208,7 +208,7 @@ workflow:
         ATMOS_FORC: "&ATMOS_FORC;"
         RES: "&RES;"
         WARMSTART_DIR: "&WARMSTART_DIR;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -259,6 +259,9 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
+        LOGDIR: "&LOGDIR;"
+        NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"'
       jobname: plot_stats

diff --git a/parm/land_analysis_hercules.yaml b/parm/land_analysis_hercules.yaml
@@ -34,7 +34,7 @@ workflow:
     OBS_TYPES: "GHCN"
     DAtype: "letkfoi_snow"
     TSTUB: "oro_C96.mx100"
-    WE2E_VAV: "YES"
+    WE2E_TEST: "NO"
     WE2E_ATOL: "1e-7"
     WE2E_LOG_FN: "we2e.log"
     NET: "landda"
@@ -137,7 +137,7 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -172,7 +172,7 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -208,7 +208,7 @@ workflow:
         ATMOS_FORC: "&ATMOS_FORC;"
         RES: "&RES;"
         WARMSTART_DIR: "&WARMSTART_DIR;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -259,6 +259,9 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
+        LOGDIR: "&LOGDIR;"
+        NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"'
       jobname: plot_stats

diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -34,7 +34,7 @@ workflow:
     OBS_TYPES: "GHCN"
     DAtype: "letkfoi_snow"
     TSTUB: "oro_C96.mx100"
-    WE2E_VAV: "YES"
+    WE2E_TEST: "NO"
     WE2E_ATOL: "1e-7"
     WE2E_LOG_FN: "we2e.log"
     NET: "landda"
@@ -137,7 +137,7 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -172,7 +172,7 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -208,7 +208,7 @@ workflow:
         ATMOS_FORC: "&ATMOS_FORC;"
         RES: "&RES;"
         WARMSTART_DIR: "&WARMSTART_DIR;"
-        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_TEST: "&WE2E_TEST;"
         WE2E_ATOL: "&WE2E_ATOL;"
         WE2E_LOG_FN: "&WE2E_LOG_FN;"
         LOGDIR: "&LOGDIR;"
@@ -259,6 +259,9 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
+        LOGDIR: "&LOGDIR;"
+        NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"'
       jobname: plot_stats

diff --git a/parm/templates/template.plot_hofx.yaml b/parm/templates/template.plot_hofx.yaml
diff --git a/parm/templates/template.plot_restart.yaml b/parm/templates/template.plot_restart.yaml
diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh
@@ -251,8 +251,8 @@ if [[ -d output/DA/hofx ]]; then
   ln -nsf ${COMOUThofx}/* ${DATA_HOFX}
 fi
 
-# WE2E V&V
-if [[ "${WE2E_VAV}" == "YES" ]]; then
+# WE2E test
+if [[ "${WE2E_TEST}" == "YES" ]]; then
   path_fbase="${FIXlandda}/test_base/we2e_com/${RUN}.${PDY}"
   fn_sfc="${FILEDATE}.sfc_data.tile"
   fn_inc="${FILEDATE}.xainc.sfc_data.tile"

diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh
@@ -155,8 +155,8 @@ do
   ln -nsf ${COMOUT}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${itile}.nc ${DATA_RESTART}
 done
 
-# WE2E V&V
-if [[ "${WE2E_VAV}" == "YES" ]]; then
+# WE2E test
+if [[ "${WE2E_TEST}" == "YES" ]]; then
   path_fbase="${FIXlandda}/test_base/we2e_com/${RUN}.${PDY}"
   fn_res="ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile"
   we2e_log_fp="${LOGDIR}/${WE2E_LOG_FN}"

diff --git a/scripts/exlandda_plot_stats.sh b/scripts/exlandda_plot_stats.sh
@@ -7,12 +7,15 @@ MM=${PDY:4:2}
 DD=${PDY:6:2}
 HH=${cyc}
 
+nYYYY=${NTIME:0:4}
+nMM=${NTIME:4:2}
+nDD=${NTIME:6:2}
+nHH=${NTIME:8:2}
+
 ############################################################
 # Stats Plot
 ############################################################
 
-cp -p ${PARMlandda}/templates/template.plot_hofx.yaml plot_hofx.yaml
-
 # Path to the directory containing the input file
 INPUTFP="${DATA_HOFX}"
 # Field variable
@@ -29,15 +32,16 @@ FIGTITLE="GHCN Snow Depth (mm)::Obs-Ana::${PDY}"
 # Prefix of output file name
 PREOUTFN="hofx_oma_${PDY}"
 
-sed -i "s|INPUTFP|${INPUTFP}|g" plot_hofx.yaml
-sed -i -e "s/XXFIELDVAR/${FIELDVAR}/g" plot_hofx.yaml
-sed -i -e "s/XXFRLOW/${FRLOW}/g" plot_hofx.yaml
-sed -i -e "s/XXFRHIGH/${FRHIGH}/g" plot_hofx.yaml
-sed -i -e "s/XXNBINS/${NBINS}/g" plot_hofx.yaml
-sed -i -e "s/XXPLOTTYPE/${PLOTTYPE}/g" plot_hofx.yaml
-sed -i -e "s/XXFIGTITLE/${FIGTITLE}/g" plot_hofx.yaml
-sed -i -e "s/XXPREOUTFN/${PREOUTFN}/g" plot_hofx.yaml
-sed -i -e "s/XXMACHINE/${MACHINE}/g" plot_hofx.yaml
+cat > plot_hofx.yaml <<EOF
+hofx_files: '${INPUTFP}'
+field_var: '${FIELDVAR}'
+field_range: [${FRLOW},${FRHIGH}]
+nbins: ${NBINS}
+plottype: '${PLOTTYPE}'
+title_fig: '${FIGTITLE}'
+output_prefix: '${PREOUTFN}'
+machine: '${MACHINE}'
+EOF
 
 ${USHlandda}/hofx_analysis_stats.py
 if [[ $? != 0 ]]; then
@@ -50,32 +54,64 @@ cp -p ${PREOUTFN}* ${COMOUTplot}
 
 
 ############################################################
-# Restart Plot
+# Time-history Plot
 ############################################################
 
-nYYYY=${NTIME:0:4}
-nMM=${NTIME:4:2}
-nDD=${NTIME:6:2}
-nHH=${NTIME:8:2}
+FN_DATA_ANAL_PREFIX="analysis_"
+FN_DATA_ANAL_SUFFIX=".log"
+FN_DATA_FCST_PREFIX="forecast_"
+FN_DATA_FCST_SUFFIX=".log"
+OUT_TITLE_ANAL_BASE="Land-DA::Analysis::QC SnowDepthGHCN::"
+OUT_FN_ANAL_BASE="landda_timehistory_"
+OUT_TITLE_TIME="Land-DA::Wall-clock time"
+OUT_FN_TIME="landda_timehistory_wtime"
+
+cat > plot_timehistory.yaml <<EOF
+path_data: '${LOGDIR}'
+work_dir: '${DATA}'
+fn_data_anal_prefix: '${FN_DATA_ANAL_PREFIX}'
+fn_data_anal_suffix: '${FN_DATA_ANAL_SUFFIX}'
+fn_data_fcst_prefix: '${FN_DATA_FCST_PREFIX}'
+fn_data_fcst_suffix: '${FN_DATA_FCST_SUFFIX}'
+nprocs_anal: '${NPROCS_ANALYSIS}'
+nprocs_fcst: '${NPROCS_FORECAST}'
+out_title_anal_base: '${OUT_TITLE_ANAL_BASE}'
+out_fn_anal_base: '${OUT_FN_ANAL_BASE}'
+out_title_time: '${OUT_TITLE_TIME}'
+out_fn_time: '${OUT_FN_TIME}'
+EOF
+
+${USHlandda}/plot_analysis_timehistory.py
+if [[ $? != 0 ]]; then
+  echo "FATAL ERROR: Time-history plots failed"
+  exit 44
+fi
+
+# Copy result files to COMOUT
+cp -p ${OUT_FN_ANAL_BASE}* ${COMOUTplot}
+cp -p ${OUT_FN_TIME}* ${COMOUTplot}
+
 
-cp -p ${PARMlandda}/templates/template.plot_restart.yaml plot_restart.yaml
+############################################################
+# Restart Plot
+############################################################
 
-PATH_DATA="${COMIN}"
-WORK_DIR="${DATA}"
 FN_DATA_BASE="ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile"
 FN_DATA_EXT=".nc"
 SOIL_LEVEL_NUMBER="1"
 OUT_TITLE_BASE="Land-DA::restart::${nYYYY}-${nMM}-${nDD}_${nHH}::"
 OUT_FN_BASE="landda_out_restart_${nYYYY}-${nMM}-${nDD}_${nHH}_"
 
-sed -i "s|PATH_DATA|${PATH_DATA}|g" plot_restart.yaml
-sed -i "s|WORK_DIR|${WORK_DIR}|g" plot_restart.yaml
-sed -i -e "s/XXFN_DATA_BASE/${FN_DATA_BASE}/g" plot_restart.yaml
-sed -i -e "s/XXFN_DATA_EXT/${FN_DATA_EXT}/g" plot_restart.yaml
-sed -i -e "s/XXSOIL_LEVEL_NUMBER/${SOIL_LEVEL_NUMBER}/g" plot_restart.yaml
-sed -i -e "s/XXOUT_TITLE_BASE/${OUT_TITLE_BASE}/g" plot_restart.yaml
-sed -i -e "s/XXOUT_FN_BASE/${OUT_FN_BASE}/g" plot_restart.yaml
-sed -i -e "s/XXMACHINE/${MACHINE}/g" plot_restart.yaml
+cat > plot_restart.yaml <<EOF
+path_data: '${COMIN}'
+work_dir: '${DATA}'
+fn_data_base: '${FN_DATA_BASE}'
+fn_data_ext: '${FN_DATA_EXT}'
+soil_lvl_number: '${SOIL_LEVEL_NUMBER}'
+out_title_base: '${OUT_TITLE_BASE}'
+out_fn_base: '${OUT_FN_BASE}'
+machine: '${MACHINE}'
+EOF
 
 ${USHlandda}/plot_forecast_restart.py
 if [[ $? != 0 ]]; then

diff --git a/scripts/exlandda_post_anal.sh b/scripts/exlandda_post_anal.sh
@@ -76,8 +76,8 @@ do
   cp -p ${DATA}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${itile}.nc ${COMOUT}/ufs_land_restart.anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${itile}.nc
 done
 
-# WE2E V&V
-if [[ "${WE2E_VAV}" == "YES" ]]; then
+# WE2E test
+if [[ "${WE2E_TEST}" == "YES" ]]; then
   path_fbase="${FIXlandda}/test_base/we2e_com/${RUN}.${PDY}"
   fn_res="ufs_land_restart.anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile"
   we2e_log_fp="${LOGDIR}/${WE2E_LOG_FN}"