Add WE2E verification capability (ufs-community#131)

* add we2e_vav flag * add we2e v&v to scripts
chan-hoo · Aug 9, 2024 · bfe321d · bfe321d
1 parent b90329e
commit bfe321d
Show file tree

Hide file tree

Showing 12 changed files with 142 additions and 20 deletions.
diff --git a/modulefiles/tasks/hera/task.analysis.lua b/modulefiles/tasks/hera/task.analysis.lua
@@ -5,4 +5,6 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/hera/task.forecast.lua b/modulefiles/tasks/hera/task.forecast.lua
@@ -23,4 +23,6 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/hera/task.post_anal.lua b/modulefiles/tasks/hera/task.post_anal.lua
@@ -8,3 +8,6 @@ load(pathJoin("hdf5", hdf5_ver))
 load(pathJoin("netcdf-c", netcdf_c_ver))
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/orion/task.analysis.lua b/modulefiles/tasks/orion/task.analysis.lua
@@ -5,4 +5,6 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/orion/task.forecast.lua b/modulefiles/tasks/orion/task.forecast.lua
@@ -23,4 +23,6 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
-load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/orion/task.post_anal.lua b/modulefiles/tasks/orion/task.post_anal.lua
@@ -8,3 +8,6 @@ load(pathJoin("hdf5", hdf5_ver))
 load(pathJoin("netcdf-c", netcdf_c_ver))
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -25,11 +25,13 @@ workflow:
     OBSDIR_SUBDIR: ""
     OBS_TYPES: "GHCN"
     DAtype: "letkfoi_snow"
-    SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
+    WE2E_VAV: "YES"
+    WE2E_ATOL: "1e-7"
+    WE2E_LOG_FN: "we2e.log"
     NET: "landda"
     envir: "test"
-    model_ver: "v1.2.1"
+    model_ver: "v2.0.0"
     RUN: "landda"
     HOMElandda: "&EXP_BASEDIR;/land-DA_workflow"
     PTMP: "&EXP_BASEDIR;/ptmp"
@@ -127,6 +129,10 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         HOMElandda: "&HOMElandda;"
         COMROOT: "&COMROOT;"
@@ -135,7 +141,6 @@ workflow:
         PDY: "&PDY;"
         cyc: "&cyc;"
         DAtype: "&DAtype;"
-        SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
         JEDI_INSTALL: "&JEDI_INSTALL;"
       account: "&ACCOUNT;"
@@ -159,6 +164,10 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         RUN: "&RUN;"
         HOMElandda: "&HOMElandda;"
@@ -218,6 +227,10 @@ workflow:
         ATMOS_FORC: "&ATMOS_FORC;"
         RES: "&RES;"
         WARMSTART_DIR: "&WARMSTART_DIR;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         HOMElandda: "&HOMElandda;"
         COMROOT: "&COMROOT;"

diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -25,11 +25,13 @@ workflow:
     OBSDIR_SUBDIR: ""
     OBS_TYPES: "GHCN"
     DAtype: "letkfoi_snow"
-    SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
+    WE2E_VAV: "YES"
+    WE2E_ATOL: "1e-7"
+    WE2E_LOG_FN: "we2e.log"
     NET: "landda"
     envir: "test"
-    model_ver: "v1.2.1"
+    model_ver: "v2.0.0"
     RUN: "landda"
     HOMElandda: "&EXP_BASEDIR;/land-DA_workflow"
     PTMP: "&EXP_BASEDIR;/ptmp"
@@ -127,6 +129,10 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         HOMElandda: "&HOMElandda;"
         COMROOT: "&COMROOT;"
@@ -135,7 +141,6 @@ workflow:
         PDY: "&PDY;"
         cyc: "&cyc;"
         DAtype: "&DAtype;"
-        SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
         JEDI_INSTALL: "&JEDI_INSTALL;"
       account: "&ACCOUNT;"
@@ -159,6 +164,10 @@ workflow:
         ACCOUNT: "&ACCOUNT;"
         RES: "&RES;"
         TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         RUN: "&RUN;"
         HOMElandda: "&HOMElandda;"
@@ -218,6 +227,10 @@ workflow:
         ATMOS_FORC: "&ATMOS_FORC;"
         RES: "&RES;"
         WARMSTART_DIR: "&WARMSTART_DIR;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
         model_ver: "&model_ver;"
         HOMElandda: "&HOMElandda;"
         COMROOT: "&COMROOT;"

diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh
@@ -32,7 +32,6 @@ case $MACHINE in
     ;;
 esac
 
-#SNOWDEPTHVAR=snwdph
 YAML_DA=construct
 GFSv17="NO"
 B=30 # back ground error std for LETKFOI
@@ -252,3 +251,26 @@ if [[ -d output/DA/hofx ]]; then
   ln -nsf ${COMOUThofx}/* ${DATA_HOFX}
 fi
 
+# WE2E V&V
+if [[ "${WE2E_VAV}" == "YES" ]]; then
+  path_fbase="${FIXlandda}/test_base/we2e_com/${RUN}.${PDY}"
+  fn_sfc="${FILEDATE}.sfc_data.tile"
+  fn_inc="${FILEDATE}.xainc.sfc_data.tile"
+  fn_hofx="letkf_hofx_ghcn_${PDY}${cyc}.nc"
+  we2e_log_fp="${LOGDIR}/${WE2E_LOG_FN}"
+  if [[ ! -e "${we2e_log_fp}" ]]; then
+    touch ${we2e_log_fp}
+  fi
+  # surface data tiles
+  for itile in {1..6}
+  do
+    ${USHlandda}/compare.py "${path_fbase}/${fn_sfc}${itile}.nc" "${COMOUT}/${fn_sfc}${itile}.nc" ${WE2E_ATOL} ${we2e_log_fp} "ANALYSIS" ${FILEDATE} "sfc_data.tile${itile}"
+  done
+  # increment tiles
+  for itile in {1..6}
+  do
+    ${USHlandda}/compare.py "${path_fbase}/${fn_inc}${itile}.nc" "${COMOUT}/${fn_inc}${itile}.nc" ${WE2E_ATOL} ${we2e_log_fp} "ANALYSIS" ${FILEDATE} "xinc.tile${itile}"
+  done
+  # H(x)
+  ${USHlandda}/compare.py "${path_fbase}/hofx/${fn_hofx}" "${COMOUT}/hofx/${fn_hofx}" ${WE2E_ATOL} ${we2e_log_fp} "ANALYSIS" ${FILEDATE} "HofX"
+fi
diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh
@@ -12,6 +12,8 @@ nMM=${NTIME:4:2}
 nDD=${NTIME:6:2}
 nHH=${NTIME:8:2}
 
+FILEDATE=${YYYY}${MM}${DD}.${HH}0000
+
 case $MACHINE in
   "hera")
     RUN_CMD="srun"
@@ -123,16 +125,33 @@ export layout_y=1
     exit 10
   fi
 
-  # copy model ouput to COM
+# copy model ouput to COM
+for itile in {1..6}
+do
+  cp -p ${DATA}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${itile}.nc ${COMOUT}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${itile}.nc
+done
+cp -p ${DATA}/ufs.cpld.datm.r.${nYYYY}-${nMM}-${nDD}-00000.nc ${COMOUT}
+cp -p ${DATA}/RESTART/ufs.cpld.cpl.r.${nYYYY}-${nMM}-${nDD}-00000.nc ${COMOUT}
+
+# link restart for next cycle
+for itile in {1..6}
+do
+  ln -nsf ${COMOUT}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${itile}.nc ${DATA_RESTART}
+done
+
+# WE2E V&V
+if [[ "${WE2E_VAV}" == "YES" ]]; then
+  path_fbase="${FIXlandda}/test_base/we2e_com/${RUN}.${PDY}"
+  fn_res="ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile"
+  we2e_log_fp="${LOGDIR}/${WE2E_LOG_FN}"
+
+  if [[ ! -e "${we2e_log_fp}" ]]; then
+    touch ${we2e_log_fp}
+  fi
+  # restart files
   for itile in {1..6}
   do
-    cp -p ${DATA}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${itile}.nc ${COMOUT}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${itile}.nc
+    ${USHlandda}/compare.py "${path_fbase}/${fn_res}${itile}.nc" "${COMOUT}/${fn_res}${itile}.nc" ${WE2E_ATOL} ${we2e_log_fp} "FORECAST" ${FILEDATE} "ufs_land_restart.tile${itile}"
   done
-  cp -p ${DATA}/ufs.cpld.datm.r.${nYYYY}-${nMM}-${nDD}-00000.nc ${COMOUT}
-  cp -p ${DATA}/RESTART/ufs.cpld.cpl.r.${nYYYY}-${nMM}-${nDD}-00000.nc ${COMOUT}
+fi
 
-  # link restart for next cycle
-  for itile in {1..6}
-  do
-    ln -nsf ${COMOUT}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${itile}.nc ${DATA_RESTART}
-  done
diff --git a/scripts/exlandda_post_anal.sh b/scripts/exlandda_post_anal.sh
@@ -75,3 +75,19 @@ for itile in {1..6}
 do
   cp -p ${DATA}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${itile}.nc ${COMOUT}/ufs_land_restart.anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${itile}.nc
 done
+
+# WE2E V&V
+if [[ "${WE2E_VAV}" == "YES" ]]; then
+  path_fbase="${FIXlandda}/test_base/we2e_com/${RUN}.${PDY}"
+  fn_res="ufs_land_restart.anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile"
+  we2e_log_fp="${LOGDIR}/${WE2E_LOG_FN}"
+  if [[ ! -e "${we2e_log_fp}" ]]; then
+    touch ${we2e_log_fp}
+  fi
+  # restart files
+  for itile in {1..6}
+  do
+    ${USHlandda}/compare.py "${path_fbase}/${fn_res}${itile}.nc" "${COMOUT}/${fn_res}${itile}.nc" ${WE2E_ATOL} ${we2e_log_fp} "POST_ANAL" ${FILEDATE} "ufs_land_restart.anal.tile${itile}"
+  done
+fi
+
diff --git a/ush/compare.py b/ush/compare.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import sys
+import numpy as np
+from netCDF4 import Dataset
+
+with Dataset(sys.argv[1]) as nc1, Dataset(sys.argv[2]) as nc2:
+  f = open(sys.argv[4], "a")
+  # Check if the list of variables are the same
+  if nc1.variables.keys()!=nc2.variables.keys():
+    print("FATAL ERROR: list of variables are different")
+    f.write("FATAL ERROR: list of variables are different.\n")
+    sys.exit(2)
+
+  for varname in nc1.variables.keys():
+    # First check if each variable has the same dimension
+    if np.shape(nc1[varname][:])!=np.shape(nc2[varname][:]):
+      print(varname,"FATAL ERROR: dimension is different")
+      f.write("FATAL ERROR: dimension is different.\n")
+      sys.exit(2)
+    # If dimension is the same, compare data
+    else:
+      np.testing.assert_allclose(nc1[varname][:], nc2[varname][:], rtol=1e-8, atol=float(sys.argv[3]))
+
+  f.write("{} :: {} :: {} :: TEST was complete successfully.\n".format(sys.argv[6],sys.argv[5],sys.argv[7]))
+  f.close()