Skip to content

Commit

Permalink
Add ex-scripts and com-type nco environment (ufs-community#77)
Browse files Browse the repository at this point in the history
* add exscript

* remove unnecessary files

* add prod-util

* add task module files

* fix module issues

* update scripts

* update config yaml files

* update document

* fix without-rocoto scritp

* update doc

* fix doc

* update doc
  • Loading branch information
chan-hoo authored Apr 12, 2024
1 parent ea3a664 commit 8fe7a80
Show file tree
Hide file tree
Showing 37 changed files with 1,428 additions and 1,028 deletions.
32 changes: 16 additions & 16 deletions do_submit_cycle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export KEEPWORKDIR="YES"

############################
# ensure necessary envars are set
envars=("exp_name" "STARTDATE" "ENDDATE" "LANDDAROOT" "LANDDA_INPUTS" "CYCLEDIR" \
envars=("exp_name" "STARTDATE" "ENDDATE" "LANDDAROOT" "LANDDA_INPUTS" "HOMElandda" \
"LANDDA_EXPTS" "BUILDDIR" "atmos_forc" "OBSDIR" "WORKDIR" \
"OUTDIR" "TEST_BASEDIR" "JEDI_EXECDIR" "JEDI_STATICDIR" "ensemble_size" \
"FCSTHR" "RES" "TPATH" "TSTUB" "cycles_per_job" "ICSDIR" "DA_config" \
Expand All @@ -46,7 +46,7 @@ fi
# check that modules are loaded in the environment

if [[ ! $BASELINE =~ 'hera.internal' ]]; then
${CYCLEDIR}/module_check.sh
${HOMElandda}/module_check.sh
fi

if [[ $? -ne 0 ]]; then
Expand Down Expand Up @@ -74,37 +74,37 @@ fi
############################
# set executables

if [[ -e ${CYCLEDIR}/exec/vector2tile_converter.exe ]]; then #prefer cmake-built executables
export vec2tileexec=${CYCLEDIR}/exec/vector2tile_converter.exe
if [[ -e ${HOMElandda}/exec/vector2tile_converter.exe ]]; then #prefer cmake-built executables
export vec2tileexec=${HOMElandda}/exec/vector2tile_converter.exe
else
export vec2tileexec=${CYCLEDIR}/sorc/vector2tile/vector2tile_converter.exe
export vec2tileexec=${HOMElandda}/sorc/vector2tile/vector2tile_converter.exe
fi
if [[ -e ${CYCLEDIR}/exec/tile2tile_converter.exe ]]; then #prefer cmake-built executables
export tile2tileexec=${CYCLEDIR}/exec/tile2tile_converter.exe
if [[ -e ${HOMElandda}/exec/tile2tile_converter.exe ]]; then #prefer cmake-built executables
export tile2tileexec=${HOMElandda}/exec/tile2tile_converter.exe
else
export tile2tileexec=${CYCLEDIR}/sorc/tile2tile/tile2tile_converter.exe
export tile2tileexec=${HOMElandda}/sorc/tile2tile/tile2tile_converter.exe
fi
if [[ -e ${CYCLEDIR}/exec/ufsLand.exe ]]; then
export LSMexec=${CYCLEDIR}/exec/ufsLand.exe
if [[ -e ${HOMElandda}/exec/ufsLand.exe ]]; then
export LSMexec=${HOMElandda}/exec/ufsLand.exe
else
export LSMexec=${CYCLEDIR}/sorc/ufsLand.fd/run/ufsLand.exe
export LSMexec=${HOMElandda}/sorc/ufsLand.fd/run/ufsLand.exe
fi

export DADIR=${CYCLEDIR}/sorc/DA_update/
export DADIR=${HOMElandda}/sorc/DA_update/
export DAscript=${DADIR}/do_landDA.sh
export MPIEXEC=`which mpiexec`
export LANDDADIR=${DADIR}

export analdate=${CYCLEDIR}/analdates.sh
export incdate=${CYCLEDIR}/incdate.sh
export analdate=${HOMElandda}/analdates.sh
export incdate=${HOMElandda}/incdate.sh

export BUILDDIR=${CYCLEDIR}/sorc/build
export BUILDDIR=${HOMElandda}/sorc/build
export INCR_EXECDIR=${DADIR}/add_jedi_incr/exec/

############################
# read in dates

export logfile=${CYCLEDIR}/cycle.log
export logfile=${HOMElandda}/cycle.log
touch $logfile
echo "***************************************" >> $logfile
echo "cycling from $STARTDATE to $ENDDATE" >> $logfile
Expand Down
33 changes: 17 additions & 16 deletions doc/source/BuildingRunningTesting/BuildRunLandDA.rst
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,6 @@ Users will need to configure certain elements of their experiment in ``land_anal
* ``EXP_BASEDIR:`` The full path to the directory where land-DA_workflow was cloned (i.e., ``$LANDDAROOT``)
* ``JEDI_INSTALL:`` The full path to the system's ``jedi-bundle`` installation
* ``LANDDA_INPUTS:`` The full path to the experiment data. See :ref:`Data <GetData>` below for information on prestaged data on Level 1 platforms.
* ``OUTDIR:`` The full path to the directory where experiment will write its output. By default, this is set to ``"&EXP_BASEDIR;/landda_expts/DA_<forcing>_test"``, but users can change the ``DA_<forcing>_test`` portion to a name of their choice. If users do not change the name, the new experiment will overwrite data from the previous experiment.

.. note::

Expand Down Expand Up @@ -193,9 +192,9 @@ Each Land DA experiment includes multiple tasks that must be run in order to sat
- Sets up the observation files
* - JLANDDA_PREP_BMAT
- Sets up the :term:`JEDI` run
* - JLANDDA_RUN_ANA
* - JLANDDA_ANALYSIS
- Runs JEDI
* - JLANDDA_RUN_FCST
* - JLANDDA_FORECAST
- Runs forecast

Users may run these tasks :ref:`using the Rocoto workflow manager <run-w-rocoto>` or :ref:`using a batch script <run-batch-script>`.
Expand Down Expand Up @@ -263,18 +262,20 @@ As the experiment progresses, it will generate a number of directories to hold i
$LANDDAROOT: Base directory
├── land-DA_workflow(<CYCLEDIR>): Home directory of the land DA workflow
├── com
│ ├── landda (<NET>)
│ │ └── vX.Y.Z (<model_ver>)
│ │ └── DA_<forcing> (<OUTDIR>)
│ │ ├── DA: Directory containing the output files of JEDI run
│ │ │ ├── hofx
│ │ │ ├── jedi_incr
│ │ │ └── logs
│ │ └── mem000: Directory containing the output files
│ └── output
│ └── logs
│ └── run_<forcing> (<LOGDIR>): Directory containing the log file of the Rocoto workflow
├── ptmp (<PTMP>)
│ └── test (<envir>)
│ └── com
│ ├── landda (<NET>)
│ │ └── vX.Y.Z (<model_ver>)
│ │ └── landda.YYYYMMDD (<RUN>.<PDY>)
│ │ └── HH (<cyc>)
│ │ ├── DA: Directory containing the output files of JEDI run
│ │ │ ├── hofx
│ │ │ └── jedi_incr
│ │ └── mem000: Directory containing the output files
│ └── output
│ └── logs
│ └── run_<forcing> (<LOGDIR>): Directory containing the log file of the Rocoto workflow
└── workdir(<WORKDIR>)
└── run_<forcing>
└── mem000: Working directory
Expand All @@ -285,7 +286,7 @@ Check for the background and analysis files in the experiment directory:

.. code-block:: console
ls -l $LANDDAROOT/com/landda/v1.2.1/run_<forcing>/mem000/restarts/<vector_or_tile>
ls -l $LANDDAROOT/ptmp/test/com/landda/v1.2.1/landda.<PDY>/<cyc>/run_<forcing>/mem000/restarts/<vector_or_tile>
where:

Expand Down
103 changes: 103 additions & 0 deletions jobs/JLANDDA_ANALYSIS
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/bin/bash

date
export PS4='+ $SECONDS + '
set -xue
#
#-----------------------------------------------------------------------
#
# Set the NCO standard environment variables (Table 1, pp.4)
#
#-----------------------------------------------------------------------
#
export USHlandda="${HOMElandda}/ush"
export EXEClandda="${HOMElandda}/exec"
export PARMlandda="${HOMElandda}/parm"
export SCRIPTSlandda="${HOMElandda}/scripts"
#
#-----------------------------------------------------------------------
#
# Define job and jobid by default for rocoto
#
#-----------------------------------------------------------------------
#
WORKFLOW_MANAGER="${WORKFLOW_MANAGER:-rocoto}"
if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then
if [ "${SCHED}" = "slurm" ]; then
job=${SLURM_JOB_NAME}
pid=${SLURM_JOB_ID}
elif [ "${SCHED}" = "pbspro" ]; then
job=${PBS_JOBNAME}
pid=${PBS_JOBID}
else
job="task"
pid=$$
fi
jobid="${job}.${PDY}${cyc}.${pid}"
fi
#
#-----------------------------------------------------------------------
#
# Create a temp working directory (DATA) and cd into it.
#
#-----------------------------------------------------------------------
#
export DATA="${DATA:-${DATAROOT}/${jobid}}"
mkdir -p $DATA
cd $DATA
#
#-----------------------------------------------------------------------
#
# Define NCO environment variables and set COM type definitions.
#
#-----------------------------------------------------------------------
#
export NET="${NET:-landda}"
export RUN="${RUN:-landda}"

[[ "$WORKFLOW_MANAGER" = "rocoto" ]] && export COMROOT=$COMROOT
if [ "${MACHINE}" = "WCOSS2" ]; then
export COMIN="${COMIN:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR})}"
export COMOUT="${COMOUT:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR})}"
else
export COMIN="${COMIN:-${COMROOT}/${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR}}"
export COMOUT="${COMOUT:-${COMROOT}/${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR}}"
fi

mkdir -p ${COMOUT}

# Create a teomporary share directory
export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}${cyc}}"
mkdir -p ${DATA_SHARE}

# Run setpdy to initialize PDYm and PDYp variables
export cycle="${cycle:-t${cyc}z}"
setpdy.sh
. ./PDY
#
#-----------------------------------------------------------------------
#
# Call the ex-script for this J-job.
#
#-----------------------------------------------------------------------
#
export pgmout="${DATA}/OUTPUT.$$"
env

${SCRIPTSlandda}/exlandda_analysis.sh
export err=$?; err_chk

if [ -e "$pgmout" ]; then
cat $pgmout
fi
#
#-----------------------------------------------------------------------
#
# Whether or not working directory DATA should be kept.
#
#-----------------------------------------------------------------------
#
if [ "${KEEPDATA}" = "NO" ]; then
rm -rf ${DATA}
fi
date
103 changes: 103 additions & 0 deletions jobs/JLANDDA_FORECAST
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/bin/bash

date
export PS4='+ $SECONDS + '
set -xue
#
#-----------------------------------------------------------------------
#
# Set the NCO standard environment variables (Table 1, pp.4)
#
#-----------------------------------------------------------------------
#
export USHlandda="${HOMElandda}/ush"
export EXEClandda="${HOMElandda}/exec"
export PARMlandda="${HOMElandda}/parm"
export SCRIPTSlandda="${HOMElandda}/scripts"
#
#-----------------------------------------------------------------------
#
# Define job and jobid by default for rocoto
#
#-----------------------------------------------------------------------
#
WORKFLOW_MANAGER="${WORKFLOW_MANAGER:-rocoto}"
if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then
if [ "${SCHED}" = "slurm" ]; then
job=${SLURM_JOB_NAME}
pid=${SLURM_JOB_ID}
elif [ "${SCHED}" = "pbspro" ]; then
job=${PBS_JOBNAME}
pid=${PBS_JOBID}
else
job="task"
pid=$$
fi
jobid="${job}.${PDY}${cyc}.${pid}"
fi
#
#-----------------------------------------------------------------------
#
# Create a temp working directory (DATA) and cd into it.
#
#-----------------------------------------------------------------------
#
export DATA="${DATA:-${DATAROOT}/${jobid}}"
mkdir -p $DATA
cd $DATA
#
#-----------------------------------------------------------------------
#
# Define NCO environment variables and set COM type definitions.
#
#-----------------------------------------------------------------------
#
export NET="${NET:-landda}"
export RUN="${RUN:-landda}"

[[ "$WORKFLOW_MANAGER" = "rocoto" ]] && export COMROOT=$COMROOT
if [ "${MACHINE}" = "WCOSS2" ]; then
export COMIN="${COMIN:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR})}"
export COMOUT="${COMOUT:-$(compath.py -o ${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR})}"
else
export COMIN="${COMIN:-${COMROOT}/${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR}}"
export COMOUT="${COMOUT:-${COMROOT}/${NET}/${model_ver}/${RUN}.${PDY}/${cyc}${SLASH_ENSMEM_SUBDIR}}"
fi

mkdir -p ${COMOUT}

# Create a teomporary share directory
export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}${cyc}}"
mkdir -p ${DATA_SHARE}

# Run setpdy to initialize PDYm and PDYp variables
export cycle="${cycle:-t${cyc}z}"
setpdy.sh
. ./PDY
#
#-----------------------------------------------------------------------
#
# Call the ex-script for this J-job.
#
#-----------------------------------------------------------------------
#
export pgmout="${DATA}/OUTPUT.$$"
env

${SCRIPTSlandda}/exlandda_forecast.sh
export err=$?; err_chk

if [ -e "$pgmout" ]; then
cat $pgmout
fi
#
#-----------------------------------------------------------------------
#
# Whether or not working directory DATA should be kept.
#
#-----------------------------------------------------------------------
#
if [ "${KEEPDATA}" = "NO" ]; then
rm -rf ${DATA}
fi
date
Loading

0 comments on commit 8fe7a80

Please sign in to comment.