Add new task to plot analysis (ufs-community#107)

* remove ptime ntime from rocoto yaml * add ptime ntime to j-job * load task modules from job card level * add a new task plot_stats * add plot_stats to xml * fix typo * fix conda issue * change wtime * comment out print * simplify python script * change cmap for scatter plot * fix typo * fix another typo in python script * update doc * change colormap * adjust colormap * adjust blue colormap * set xlimit of histogram adjustable * change coastline res to 50m * remove unnecessary line from gitignore
chan-hoo · Jun 5, 2024 · 54086b9 · 54086b9
1 parent 66b096c
commit 54086b9
Show file tree

Hide file tree

Showing 19 changed files with 409 additions and 8 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,7 +7,6 @@ sorc/lib/
 sorc/lib64
 parm/conda_loc
 parm/*.xml
-!parm/land_analysis_test.xml
 parm/*.db
 parm/*.out
 parm/*.err

diff --git a/doc/source/BuildingRunningTesting/BuildRunLandDA.rst b/doc/source/BuildingRunningTesting/BuildRunLandDA.rst
@@ -192,6 +192,8 @@ Each Land DA experiment includes multiple tasks that must be run in order to sat
      - Runs :term:`JEDI` and adds the increment to the surface data files
    * - JLANDDA_POST_ANAL
      - Transfers the JEDI result from the surface data files to the restart files
+   * - JLANDDA_PLOT_STATS
+     - Plots the JEDI result (scatter/histogram)
    * - JLANDDA_FORECAST
      - Runs the forecast model
 

diff --git a/jobs/JLANDDA_ANALYSIS b/jobs/JLANDDA_ANALYSIS
@@ -71,10 +71,14 @@ else
 fi
 
 mkdir -p ${COMOUT}
+export COMOUThofx="${COMOUThofx:-${COMOUT}/hofx}"
+mkdir -p ${COMOUThofx}
 
 # Create a teomporary share directory
 export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
+export DATA_HOFX="${DATA_HOFX:-${DATAROOT}/DATA_SHARE/hofx}"
+mkdir -p ${DATA_HOFX}
 
 # Set other dates
 export PTIME=$($NDATE -24 $PDY$cyc)

diff --git a/jobs/JLANDDA_PLOT_STATS b/jobs/JLANDDA_PLOT_STATS
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+date
+export PS4='+ $SECONDS + '
+set -xue
+#
+#-----------------------------------------------------------------------
+#
+# Set the NCO standard environment variables (Table 1, pp.4)
+#
+#-----------------------------------------------------------------------
+#
+export USHlandda="${HOMElandda}/ush"
+export EXEClandda="${HOMElandda}/exec"
+export PARMlandda="${HOMElandda}/parm"
+export FIXlandda="${HOMElandda}/fix"
+export SCRIPTSlandda="${HOMElandda}/scripts"
+#
+#-----------------------------------------------------------------------
+#
+# Define job and jobid by default for rocoto
+#
+#-----------------------------------------------------------------------
+#
+WORKFLOW_MANAGER="${WORKFLOW_MANAGER:-rocoto}"
+if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then
+  if [ "${SCHED}" = "slurm" ]; then
+    job=${SLURM_JOB_NAME}
+    pid=${SLURM_JOB_ID}
+  elif [ "${SCHED}" = "pbspro" ]; then
+    job=${PBS_JOBNAME}
+    pid=${PBS_JOBID}
+  else
+    job="task"
+    pid=$$
+  fi
+  jobid="${job}.${PDY}${cyc}.${pid}"
+fi
+#
+#-----------------------------------------------------------------------
+#
+# Create a temp working directory (DATA) and cd into it.
+#
+#-----------------------------------------------------------------------
+#
+export DATA="${DATA:-${DATAROOT}/${jobid}}"
+mkdir -p $DATA
+cd $DATA
+#
+#-----------------------------------------------------------------------
+#
+# Define NCO environment variables and set COM type definitions.
+#
+#-----------------------------------------------------------------------
+#
+export NET="${NET:-landda}"
+export RUN="${RUN:-landda}"
+
+# Run setpdy to initialize PDYm and PDYp variables
+export cycle="${cycle:-t${cyc}z}"
+setpdy.sh
+. ./PDY
+
+[[ "$WORKFLOW_MANAGER" = "rocoto" ]] && export COMROOT=$COMROOT
+if [ "${MACHINE}" = "WCOSS2" ]; then
+  export COMIN="${COMIN:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY})}"
+  export COMOUT="${COMOUT:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY})}"
+else
+  export COMIN="${COMIN:-${COMROOT}/${NET}/${model_ver}/${RUN}.${PDY}}"
+  export COMOUT="${COMOUT:-${COMROOT}/${NET}/${model_ver}/${RUN}.${PDY}}"
+fi
+
+mkdir -p ${COMOUT}
+export COMOUTplot="${COMOUTplot:-${COMOUT}/plot}"
+mkdir -p ${COMOUTplot}
+
+# Create a teomporary share directory
+export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
+mkdir -p ${DATA_SHARE}
+export DATA_HOFX="${DATA_HOFX:-${DATAROOT}/DATA_SHARE/hofx}"
+mkdir -p ${DATA_HOFX}
+
+#
+#-----------------------------------------------------------------------
+#
+# Call the ex-script for this J-job.
+#
+#-----------------------------------------------------------------------
+#
+export pgmout="${DATA}/OUTPUT.$$"
+env
+
+${SCRIPTSlandda}/exlandda_plot_stats.sh
+export err=$?; err_chk
+
+if [ -e "$pgmout" ]; then
+  cat $pgmout
+fi
+#
+#-----------------------------------------------------------------------
+#
+# Whether or not working directory DATA should be kept.
+#
+#-----------------------------------------------------------------------
+#
+if [ "${KEEPDATA}" = "NO" ]; then
+  rm -rf ${DATA}
+fi
+date
diff --git a/jobs/JLANDDA_PREP_OBS b/jobs/JLANDDA_PREP_OBS
@@ -71,6 +71,8 @@ else
 fi
 
 mkdir -p ${COMOUT}
+export COMOUTobs=${COMOUTobs:-${COMOUT}/obs}
+mkdir -p ${COMOUTobs}
 
 # Create a teomporary share directory
 export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"

diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua
@@ -39,5 +39,5 @@ setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "orion.intel")
 
-setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA")
+setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev")
 setenv("JEDI_INSTALL", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6")
diff --git a/modulefiles/tasks/hera/task.plot_stats.lua b/modulefiles/tasks/hera/task.plot_stats.lua
@@ -0,0 +1,6 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
diff --git a/modulefiles/tasks/orion/task.plot_stats.lua b/modulefiles/tasks/orion/task.plot_stats.lua
@@ -0,0 +1,6 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
diff --git a/parm/conda_environment.yml b/parm/conda_environment.yml
@@ -6,3 +6,9 @@ dependencies:
   - pylint=2.17*
   - pytest=7.2*
   - uwtools=2.2.0
+  - cartopy=0.23.*
+  - matplotlib=3.8.*
+  - netcdf4=1.6.*
+  - numpy=1.23.*
+  - scipy=1.10.*
+  - xarray=2022.11.*
diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -188,6 +188,34 @@ workflow:
         taskdep:
           attrs:
             task: analysis
+    task_plot_stats:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        EXP_NAME: "&EXP_NAME;"
+        model_ver: "&model_ver;"
+        RUN: "&RUN;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"'
+      jobname: plot_stats
+      cores: 1
+      walltime: 00:10:00
+      queue: batch
+      join: "&LOGDIR;/plot_stats&LOGFN_SUFFIX;"
+      dependency:
+        taskdep:
+          attrs:
+            task: analysis
     task_forecast:
       attrs:
         cycledefs: cycled

diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -188,6 +188,34 @@ workflow:
         taskdep:
           attrs:
             task: analysis
+    task_plot_stats:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        EXP_NAME: "&EXP_NAME;"
+        model_ver: "&model_ver;"
+        RUN: "&RUN;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"'
+      jobname: plot_stats
+      cores: 1
+      walltime: 00:10:00
+      queue: batch
+      join: "&LOGDIR;/plot_stats&LOGFN_SUFFIX;"
+      dependency:
+        taskdep:
+          attrs:
+            task: analysis
     task_forecast:
       attrs:
         cycledefs: cycled

diff --git a/parm/task_load_modules_run_jjob.sh b/parm/task_load_modules_run_jjob.sh
@@ -47,5 +47,17 @@ else
   echo "FATAL ERROR: task module file does not exist !!!"
 fi
 
+# Load conda env for plotting task
+if [ "${task_name}" = "plot_stats" ]; then
+  module unload python
+  module use ${home_dir}/modulefiles
+  module load conda
+  set +u
+  conda deactivate
+  conda activate land_da
+  conda list
+  set -u
+fi
+
 # Run J-job script
 ${home_dir}/jobs/JLANDDA_${task_name_upper}
diff --git a/parm/templates/template.plot_hofx.yaml b/parm/templates/template.plot_hofx.yaml
@@ -0,0 +1,8 @@
+hofx_files: 'INPUTFP'
+field_var: 'XXFIELDVAR'
+field_range: [XXFRLOW,XXFRHIGH]
+nbins: XXNBINS
+plottype: 'XXPLOTTYPE'
+title_fig: 'XXFIGTITLE'
+output_prefix: 'XXPREOUTFN'
+machine: 'XXMACHINE'
diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh
@@ -235,6 +235,7 @@ do
 done
 
 if [[ -d output/DA/hofx ]]; then
-  cp -rp output/DA/hofx ${COMOUT}
+  cp -p output/DA/hofx/* ${COMOUThofx}
+  ln -nsf ${COMOUThofx}/* ${DATA_HOFX}
 fi
 
diff --git a/scripts/exlandda_plot_stats.sh b/scripts/exlandda_plot_stats.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -xue
+
+############################################################
+# Scatter Plot
+############################################################
+
+cp ${PARMlandda}/templates/template.plot_hofx.yaml plot_hofx.yaml
+
+# Path to the directory containing the input file
+INPUTFP="${DATA_HOFX}"
+# Field variable
+FIELDVAR="OMA"
+# Field Range for scatter plot: [Low,High]
+FRLOW=-300
+FRHIGH=300
+# Number of bins in histogram plot
+NBINS=100
+# Plot type (scatter/histogram/both)
+PLOTTYPE="both"
+# Figure title
+FIGTITLE="GHCN Snow Depth (mm)::Obs-Ana::${PDY}"
+# Prefix of output file name
+PREOUTFN="hofx_oma_${PDY}"
+
+sed -i "s|INPUTFP|${INPUTFP}|g" plot_hofx.yaml
+sed -i -e "s/XXFIELDVAR/${FIELDVAR}/g" plot_hofx.yaml
+sed -i -e "s/XXFRLOW/${FRLOW}/g" plot_hofx.yaml
+sed -i -e "s/XXFRHIGH/${FRHIGH}/g" plot_hofx.yaml
+sed -i -e "s/XXNBINS/${NBINS}/g" plot_hofx.yaml
+sed -i -e "s/XXPLOTTYPE/${PLOTTYPE}/g" plot_hofx.yaml
+sed -i -e "s/XXFIGTITLE/${FIGTITLE}/g" plot_hofx.yaml
+sed -i -e "s/XXPREOUTFN/${PREOUTFN}/g" plot_hofx.yaml
+sed -i -e "s/XXMACHINE/${MACHINE}/g" plot_hofx.yaml
+
+${USHlandda}/hofx_analysis_stats.py
+if [[ $? != 0 ]]; then
+  echo "Scatter/Histogram plots failed"
+  exit 33
+fi
+
+# Copy result files to COMOUT
+cp -p ${PREOUTFN}* ${COMOUTplot}
diff --git a/scripts/exlandda_pre_anal.sh b/scripts/exlandda_pre_anal.sh
@@ -32,7 +32,7 @@ if [[ $ATMOS_FORC == "era5" ]]; then
   echo 'calling vector2tile' 
 
   # update vec2tile and tile2vec namelists
-  cp  ${PARMlandda}/templates/template.vector2tile vector2tile.namelist
+  cp ${PARMlandda}/templates/template.vector2tile vector2tile.namelist
 
   sed -i "s|FIXlandda|${FIXlandda}|g" vector2tile.namelist
   sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist

diff --git a/scripts/exlandda_prep_obs.sh b/scripts/exlandda_prep_obs.sh
@@ -12,8 +12,6 @@ MP=${PTIME:4:2}
 DP=${PTIME:6:2}
 HP=${PTIME:8:2}
 
-mkdir -p "${COMOUT}/obs"
-
 ################################################
 # 2. PREPARE OBS FILES
 ################################################
@@ -42,7 +40,7 @@ for obs in "${OBS_TYPES[@]}"; do
   # check obs are available
   if [[ -e $obsfile ]]; then
     echo "do_landDA: $obs observations found: $obsfile"
-    cp -p $obsfile ${COMOUT}/obs/${obs}_${YYYY}${MM}${DD}${HH}.nc
+    cp -p $obsfile ${COMOUTobs}/${obs}_${YYYY}${MM}${DD}${HH}.nc
   else
     echo "${obs} observations not found: $obsfile"
   fi

diff --git a/sorc/app_build.sh b/sorc/app_build.sh
@@ -396,7 +396,7 @@ fi
 if [ "${PLATFORM}" = "hera" ]; then
   landda_fix_orig="/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs"
 elif [ "${PLATFORM}" = "orion" ]; then
-  landda_fix_orig="/work/noaa/epic/UFS_Land-DA/inputs"
+  landda_fix_orig="/work/noaa/epic/UFS_Land-DA_Dev/inputs"
 fi
 ln -nsf ${landda_fix_orig}/* ${HOME_DIR}/fix