Load modules for each task on job-card level (rocoto XML) (ufs-commun…

…ity#105) * remove ptime ntime from rocoto yaml * add ptime ntime to j-job * load task modules from job card level
chan-hoo · Jun 3, 2024 · 66b096c · 66b096c
1 parent 2f8d537
commit 66b096c
Show file tree

Hide file tree

Showing 22 changed files with 71 additions and 51 deletions.
diff --git a/jobs/JLANDDA_ANALYSIS b/jobs/JLANDDA_ANALYSIS
@@ -75,6 +75,9 @@ mkdir -p ${COMOUT}
 # Create a teomporary share directory
 export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
+
+# Set other dates
+export PTIME=$($NDATE -24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_FORECAST b/jobs/JLANDDA_FORECAST
@@ -79,6 +79,9 @@ export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
 export DATA_RESTART="${DATA_RESTART:-${DATAROOT}/DATA_SHARE/RESTART}"
 mkdir -p ${DATA_RESTART}
+
+# Set other dates
+export NTIME=$($NDATE 24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_POST_ANAL b/jobs/JLANDDA_POST_ANAL
@@ -77,6 +77,9 @@ export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
 export DATA_RESTART="${DATA_RESTART:-${DATAROOT}/DATA_SHARE/RESTART}"
 mkdir -p ${DATA_RESTART}
+
+# Set other dates
+export NTIME=$($NDATE 24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_PREP_OBS b/jobs/JLANDDA_PREP_OBS
@@ -75,6 +75,9 @@ mkdir -p ${COMOUT}
 # Create a teomporary share directory
 export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
+
+# Set other dates
+export PTIME=$($NDATE -24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_PRE_ANAL b/jobs/JLANDDA_PRE_ANAL
@@ -77,6 +77,9 @@ export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
 export DATA_RESTART="${DATA_RESTART:-${DATAROOT}/DATA_SHARE/RESTART}"
 mkdir -p ${DATA_RESTART}
+
+# Set other dates
+export PTIME=$($NDATE -24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua
@@ -32,9 +32,7 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
-
-load("ufs-pyenv")
-load("atlas")
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
 
 setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")

diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua
@@ -32,8 +32,7 @@ load(pathJoin("w3emc", w3emc_ver))
 load(pathJoin("gftl-shared", gftl_shared_ver))
 load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
-
-load("ufs-pyenv")
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
 
 setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")

diff --git a/modulefiles/tasks/hera/task.analysis.lua b/modulefiles/tasks/hera/task.analysis.lua
@@ -5,3 +5,4 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
diff --git a/modulefiles/tasks/hera/task.forecast.lua b/modulefiles/tasks/hera/task.forecast.lua
@@ -4,4 +4,23 @@ load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
diff --git a/modulefiles/tasks/hera/task.post_anal.lua b/modulefiles/tasks/hera/task.post_anal.lua
@@ -4,4 +4,7 @@ load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
diff --git a/modulefiles/tasks/hera/task.pre_anal.lua b/modulefiles/tasks/hera/task.pre_anal.lua
@@ -4,4 +4,7 @@ load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
diff --git a/modulefiles/tasks/orion/task.analysis.lua b/modulefiles/tasks/orion/task.analysis.lua
@@ -5,3 +5,4 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
diff --git a/modulefiles/tasks/orion/task.forecast.lua b/modulefiles/tasks/orion/task.forecast.lua
@@ -4,4 +4,23 @@ load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
 load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("ufs-pyenv", ufs_pyenv_ver))
diff --git a/modulefiles/tasks/orion/task.post_anal.lua b/modulefiles/tasks/orion/task.post_anal.lua
@@ -4,4 +4,7 @@ load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
diff --git a/modulefiles/tasks/orion/task.pre_anal.lua b/modulefiles/tasks/orion/task.pre_anal.lua
@@ -4,4 +4,7 @@ load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 load(pathJoin("prod_util", prod_util_ver))
diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -42,8 +42,6 @@ workflow:
     PATHRT: "&EXP_BASEDIR;"
     PDY:  "<cyclestr>@Y@m@d</cyclestr>"
     cyc: "<cyclestr>@H</cyclestr>"
-    PTIME:  "<cyclestr offset='-24:00:00'>@Y@m@d@H</cyclestr>"
-    NTIME:  "<cyclestr offset='24:00:00'>@Y@m@d@H</cyclestr>"
     DATADEP_FILE1: "<cyclestr>&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc</cyclestr>"
     DATADEP_FILE2: "<cyclestr>&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.nc</cyclestr>"
     DATADEP_FILE3: "<cyclestr>&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc</cyclestr>"
@@ -70,7 +68,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "prep_obs" "&HOMElandda;" "&MACHINE;"'
       jobname: prep_obs
@@ -99,7 +96,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "pre_anal" "&HOMElandda;" "&MACHINE;"'
       jobname: pre_anal
@@ -145,8 +141,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
-        NTIME: "&NTIME;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROC_JEDI: "&NPROCS_ANALYSIS;"
@@ -182,8 +176,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
-        NTIME: "&NTIME;"
         FCSTHR: "&FCSTHR;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "post_anal" "&HOMElandda;" "&MACHINE;"'
@@ -216,7 +208,6 @@ workflow:
         LOGDIR: "&LOGDIR;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        NTIME: "&NTIME;"
         DAtype: "&DAtype;"
         FCSTHR: "&FCSTHR;"
       account: "&ACCOUNT;"

diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -42,8 +42,6 @@ workflow:
     PATHRT: "&EXP_BASEDIR;"
     PDY:  "<cyclestr>@Y@m@d</cyclestr>"
     cyc: "<cyclestr>@H</cyclestr>"
-    PTIME:  "<cyclestr offset='-24:00:00'>@Y@m@d@H</cyclestr>"
-    NTIME:  "<cyclestr offset='24:00:00'>@Y@m@d@H</cyclestr>"
     DATADEP_FILE1: "<cyclestr>&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc</cyclestr>"
     DATADEP_FILE2: "<cyclestr>&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.nc</cyclestr>"
     DATADEP_FILE3: "<cyclestr>&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc</cyclestr>"
@@ -70,7 +68,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "prep_obs" "&HOMElandda;" "&MACHINE;"'
       jobname: prep_obs
@@ -99,7 +96,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "pre_anal" "&HOMElandda;" "&MACHINE;"'
       jobname: pre_anal
@@ -145,8 +141,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
-        NTIME: "&NTIME;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROC_JEDI: "&NPROCS_ANALYSIS;"
@@ -182,8 +176,6 @@ workflow:
         KEEPDATA: "&KEEPDATA;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        PTIME: "&PTIME;"
-        NTIME: "&NTIME;"
         FCSTHR: "&FCSTHR;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "post_anal" "&HOMElandda;" "&MACHINE;"'
@@ -216,7 +208,6 @@ workflow:
         LOGDIR: "&LOGDIR;"
         PDY: "&PDY;"
         cyc: "&cyc;"
-        NTIME: "&NTIME;"
         DAtype: "&DAtype;"
         FCSTHR: "&FCSTHR;"
       account: "&ACCOUNT;"

diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh
@@ -17,14 +17,6 @@ FILEDATE=${YYYY}${MM}${DD}.${HH}0000
 JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data
 JEDI_EXECDIR=${JEDI_INSTALL}/build/bin
 
-# load modulefiles
-BUILD_VERSION_FILE="${HOMElandda}/versions/build.ver_${MACHINE}"
-if [ -e ${BUILD_VERSION_FILE} ]; then
-  . ${BUILD_VERSION_FILE}
-fi
-mkdir -p modulefiles
-cp ${HOMElandda}/modulefiles/build_${MACHINE}_intel.lua $DATA/modulefiles/modules.landda.lua
-module use modulefiles; module load modules.landda
 MPIEXEC=`which mpiexec`
 
 #SNOWDEPTHVAR=snwdph

diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh
@@ -14,18 +14,8 @@ nMM=${NTIME:4:2}
 nDD=${NTIME:6:2}
 nHH=${NTIME:8:2}
 
-# load modulefiles
-BUILD_VERSION_FILE="${HOMElandda}/versions/build.ver_${MACHINE}"
-if [ -e ${BUILD_VERSION_FILE} ]; then
-  . ${BUILD_VERSION_FILE}
-fi
-mkdir -p modulefiles
-cp ${HOMElandda}/modulefiles/build_${MACHINE}_intel.lua $DATA/modulefiles/modules.landda.lua
-module use modulefiles; module load modules.landda
-
 MPIEXEC=`which mpiexec`
 
-
 #  convert back to UFS tile, run model (all members)
 if [[ ${ATMOS_FORC} == "gswp3" ]]; then  
 

diff --git a/scripts/exlandda_pre_anal.sh b/scripts/exlandda_pre_anal.sh
@@ -14,16 +14,6 @@ HP=${PTIME:8:2}
 
 FILEDATE=${YYYY}${MM}${DD}.${HH}0000
 
-mkdir -p modulefiles
-cp ${HOMElandda}/modulefiles/build_${MACHINE}_intel.lua $DATA/modulefiles/modules.landda.lua
-
-# load modulefiles
-BUILD_VERSION_FILE="${HOMElandda}/versions/build.ver_${MACHINE}"
-if [ -e ${BUILD_VERSION_FILE} ]; then
-  . ${BUILD_VERSION_FILE}
-fi
-module use modulefiles; module load modules.landda
-
 if [[ $ATMOS_FORC == "era5" ]]; then
   # vector2tile for DA
   # copy restarts into work directory

diff --git a/versions/build.ver_hera b/versions/build.ver_hera
@@ -24,6 +24,7 @@ export sp_ver="2.5.0"
 export stack_intel_oneapi_mpi_ver="2021.5.1"
 export stack_intel_ver="2021.5.0"
 export stack_python_ver="3.10.13"
+export ufs_pyenv_ver="1.0.0"
 export w3emc_ver="2.10.0"
 export zlib_ver="1.2.13"
 
diff --git a/versions/build.ver_orion b/versions/build.ver_orion
@@ -24,6 +24,7 @@ export sp_ver="2.5.0"
 export stack_intel_oneapi_mpi_ver="2021.5.1"
 export stack_intel_ver="2022.0.2"
 export stack_python_ver="3.10.13"
+export ufs_pyenv_ver="1.0.0"
 export w3emc_ver="2.10.0"
 export zlib_ver="1.2.13"