Update Land DA container to Spack-Stack 1.6.0 (#147)

* update to ss160

* update python and cmake vars

* updated info

* update file to work with new setup container script

* added singularity files

* fixed typo

* fix typos

* removed extra dir

* update run container script

* init commit

* added logic to use staged data for the singularity container

* remove wflow singularity lua file

* update docs to reflect new container workflow

* update container name to v2.0.0. release

* updated container info

* added changes to work with the new spack-stack

* init commit

* removed out of date analysis file

* added fix data path for container

* update to new spack stack

* updated workflow process

* update sub var name

* update land da data link

* update land da data path

* fixed typo

---------

Co-authored-by: Parallel Works app-run user <Edward.Snyder@mgmt-edwardsnyder-aws-00194.pw-noaa-us-east-1.pw.local>

doc/source/BackgroundInfo/TechnicalOverview.rst

@@ -83,9 +83,9 @@ Preconfigured (Level 1) systems for Land DA already have the required external l
      - /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
      - /work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules
    * - Container
-     - intel-oneapi-compilers/2021.8.0
-     - intel-oneapi-mpi/2021.8.0
-     - /opt/spack-stack/ (inside the container)
+     - intel-oneapi-compilers/2021.10.0
+     - intel-oneapi-mpi/2021.9.0
+     - /opt/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core (inside the container)
      - /opt/jedi-bundle (inside the container)
 
 Level 2-4 Systems
@@ -215,4 +215,4 @@ Unlike the standalone Noah-MP land driver, the Noah-MP :term:`NUOPC cap` is able
 
 Unified Workflow (UW) Tools
 ============================
-The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes rocoto, template, and configuration (config) tools, which are being incorporated into the Land DA workflow. Additional tools are under development. More details about UW tools can be found in the `uwtools <https://github.com/ufs-community/uwtools>`_ GitHub repository and in the :uw:`UW Documentation <>`.
+The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes rocoto, template, and configuration (config) tools, which are being incorporated into the Land DA workflow. Additional tools are under development. More details about UW tools can be found in the `uwtools <https://github.com/ufs-community/uwtools>`_ GitHub repository and in the :uw:`UW Documentation <>`.

modulefiles/build_singularity_intel.lua

@@ -1,25 +1,25 @@
 help([[
-loads UFS Model prerequisites for Hera/Intel
+loads UFS Model prerequisites for Singularity container
 ]])
 
 setenv("EPICHOME", "/opt")
 
-prepend_path("MODULEPATH", pathJoin(os.getenv("EPICHOME"),"spack-stack/spack-stack-1.3.0/envs/unified-dev/install/modulefiles/Core"))
+prepend_path("MODULEPATH", pathJoin(os.getenv("EPICHOME"),"spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core"))
 
-stack_intel_ver=os.getenv("stack_intel_ver") or "2021.8.0"
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-load("intel-oneapi-mpi/2021.8.0")
-stack_intel_oneapi_mpi_ver=os.getenv("stack_intel_oneapi_mpi_ver") or "2021.8.0"
+load("intel-oneapi-mpi/2021.9.0")
+stack_intel_oneapi_mpi_ver=os.getenv("stack_intel_oneapi_mpi_ver") or "2021.9.0"
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.8.10"
-load(pathJoin("stack-python", stack_python_ver))
+--stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
+--load(pathJoin("stack-python", stack_python_ver))
 
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
-ecbuild_ver=os.getenv("ecbuild_ver") or "3.6.5"
+ecbuild_ver=os.getenv("ecbuild_ver") or "3.7.2"
 load(pathJoin("ecbuild", ecbuild_ver))
 
 jasper_ver=os.getenv("jasper_ver") or "2.0.32"
@@ -37,16 +37,16 @@ load(pathJoin("hdf5", hdf5_ver))
 netcdf_c_ver=os.getenv("netcdf_ver") or "4.9.2"
 load(pathJoin("netcdf-c", netcdf_c_ver))
 
-netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0"
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
 load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 
-pio_ver=os.getenv("pio_ver") or "2.5.9"
+pio_ver=os.getenv("pio_ver") or "2.5.10"
 load(pathJoin("parallelio", pio_ver))
 
-esmf_ver=os.getenv("esmf_ver") or "8.3.0b09"
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
 load(pathJoin("esmf", esmf_ver))
 
-fms_ver=os.getenv("fms_ver") or "2022.04"
+fms_ver=os.getenv("fms_ver") or "2024.01"
 load(pathJoin("fms",fms_ver))
 
 bacio_ver=os.getenv("bacio_ver") or "2.4.1"
@@ -55,39 +55,47 @@ load(pathJoin("bacio", bacio_ver))
 crtm_ver=os.getenv("crtm_ver") or "2.4.0"
 load(pathJoin("crtm", crtm_ver))
 
-g2_ver=os.getenv("g2_ver") or "3.4.5"
+g2_ver=os.getenv("g2_ver") or "3.5.1"
 load(pathJoin("g2", g2_ver))
 
-g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.10.2"
+g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.13.0"
 load(pathJoin("g2tmpl", g2tmpl_ver))
 
-ip_ver=os.getenv("ip_ver") or "3.3.3"
+ip_ver=os.getenv("ip_ver") or "4.3.0"
 load(pathJoin("ip", ip_ver))
 
-sp_ver=os.getenv("sp_ver") or "2.3.3"
+sp_ver=os.getenv("sp_ver") or "2.5.0"
 load(pathJoin("sp", sp_ver))
 
-w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
 load(pathJoin("w3emc", w3emc_ver))
 
-gftl_shared_ver=os.getenv("gftl_shared_ver") or "1.5.0"
+gftl_shared_ver=os.getenv("gftl_shared_ver") or "1.6.1"
 load(pathJoin("gftl-shared", gftl_shared_ver))
 
-mapl_ver=os.getenv("mapl_ver") or "2.22.0-esmf-8.3.0b09"
+mapl_ver=os.getenv("mapl_ver") or "2.40.3-esmf-8.6.0"
 load(pathJoin("mapl", mapl_ver))
 
+scotch_ver=os.getenv("scotch_ver") or "7.0.4"
+load(pathJoin("scotch", scotch_ver))
+
 load("py-cftime/1.0.3.4")
-load("py-cython/0.29.32")
+load("py-cython/0.29.36")
 load("py-f90nml/1.4.3")
-load("py-jinja2/3.1.2")
-load("py-netcdf4/1.5.3")
+load("py-jinja2/3.0.3")
+load("py-netcdf4/1.5.8")
 load("py-numpy/1.22.3")
-load("py-pandas/1.4.0")
+load("py-pandas/1.5.3")
 load("py-python-dateutil/2.8.2")
 load("py-pyyaml/6.0")
 
-load("atlas")
+setenv("CFLAGS","-diag-disable=10448")
+setenv("FFLAGS","-diag-disable=10448")
 
+prepend_path("PATH","/opt/intel/oneapi/compiler/2024.0/bin:/opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64")
+--setenv("CMAKE_C_COMPILER","mpiicc")
+--setenv("CMAKE_CXX_COMPILER","mpicxx")
+--setenv("CMAKE_Fortran_COMPILER","mpif90")
 setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")

modulefiles/tasks/singularity/task.analysis.lua

@@ -0,0 +1,2 @@
+load("COMPILER")
+load("MPI")

modulefiles/tasks/singularity/task.forecast.lua

@@ -0,0 +1,2 @@
+load("COMPILER")
+load("MPI")

modulefiles/tasks/singularity/task.plot_stats.lua

@@ -0,0 +1,2 @@
+load("COMPILER")
+load("MPI")

modulefiles/tasks/singularity/task.post_anal.lua

@@ -0,0 +1,2 @@
+load("COMPILER")
+load("MPI")

modulefiles/tasks/singularity/task.pre_anal.lua

@@ -0,0 +1,2 @@
+load("COMPILER")
+load("MPI")

modulefiles/tasks/singularity/task.prep_obs.lua

@@ -0,0 +1,2 @@
+load("COMPILER")
+load("MPI")

parm/parm_xml_singularity.yaml

@@ -0,0 +1,7 @@
+machine: singularity
+account: epic
+# exp_basedir: /path/to/parent/directory/of/land-DA_workflow
+exp_basedir: SINGULARITY_WORKING_DIR 
+jedi_install: SINGULARITY_WORKING_DIR
+warmstart_dir: SINGULARITY_WORKING_DIR/land-DA_workflow/fix/DATA_RESTART
+we2e_test: 'NO'

parm/run_container_executable.sh

@@ -2,18 +2,21 @@
 
 export SINGULARITYENV_FI_PROVIDER=tcp
 export SINGULARITY_SHELL=/bin/bash
+SINGULARITYBIN=`which singularity`
 BINDDIR="/"`pwd | awk -F"/" '{print $2}'`
-CONTAINERLOC=${EPICCONTAINERS:-${HOME}}
-img=${img:-${CONTAINERLOC}/ubuntu20.04-intel-ue-landda.img}
+img=IMAGE
 CONTAINERBASE="/"`echo $img | xargs realpath | awk -F"/" '{print $2}'`
 cmd=$(basename "$0")
 arg="$@"
-if [ ! -z "$FIXlandda" ]; then
-  INPUTBASE="/"`echo $FIXlandda | xargs realpath | awk -F"/" '{print $2}'`
+if [ ! -z "$LANDDAROOT" ]; then
+  INPUTBASE="/"`echo $LANDDAROOT | xargs realpath | awk -F"/" '{print $2}'`
   INPUTBIND="-B $INPUTBASE:$INPUTBASE"
 else
   INPUTBIND=""
 fi
-echo running: ${SINGULARITYBIN} exec $img $cmd $arg
+# Remove echo for ndate command as it messes with the PTIME variable 
+if [ $cmd != "ndate" ]; then
+  echo running: ${SINGULARITYBIN} exec -B $BINDDIR:$BINDDIR -B $CONTAINERBASE:$CONTAINERBASE $INPUTBIND $img $cmd $arg
+fi
 ${SINGULARITYBIN} exec -B $BINDDIR:$BINDDIR -B $CONTAINERBASE:$CONTAINERBASE $INPUTBIND $img $cmd $arg
 

sorc/test/ci/Dockerfile

@@ -1,20 +1,34 @@
-From noaaepic/ubuntu20.04-intel-landda:develop
+From noaaepic/ubuntu22.04-intel21.10-landda:ue160-fms202401-dev
 
 CMD ["/bin/bash"]
 
 ENV HOME=/opt
 WORKDIR $HOME
 
-#remove org land-offline_workflow
-RUN rm -rf $HOME/land-offline_workflow
-COPY . $HOME/land-offline_workflow
+# Get Land DA data
+RUN wget https://noaa-ufs-land-da-pds.s3.amazonaws.com/develop-20241024/inputs.tar.gz && \
+    tar -xvzf inputs.tar.gz && mv inputs/* /opt/land-DA_workflow/fix/
 
 # set env vars
 ENV FIXlandda=$HOME/land-DA_workflow/fix
-ENV EPICHOME=/opt
-ENV JEDI_INSTALL=${EPICHOME}/jedi_skylabv7.0
-ENV TEST_BASEDIR=${EPICHOME}/test_base/restarts/vector"}
+ENV JEDI_INSTALL=$HOME
+ENV FIXdir=$FIXlandda
+ENV JEDI_EXECDIR=/opt/jedi-bundle/install/bin
+
+# Fix UFS WM RT File paths
+#RUN ln -s /opt/land-DA_workflow/install/bin/ufs_model /opt/land-DA_workflow/build/ufs_model.fd/src/ufs_model.fd-build/
+RUN sed -i '18 i PLATFORM=jet' /opt/land-DA_workflow/sorc/test/run_ufs_datm_lnd.sh
+RUN mv /opt/land-DA_workflow/fix/DATM_input_data /opt/land-DA_workflow/fix/DATM_GSWP3_input_data && \
+    ln -s /opt/land-DA_workflow/fix/DATM_GSWP3_input_data/gswp3/* /opt/land-DA_workflow/fix/DATM_GSWP3_input_data/
+RUN mkdir -p /opt/land-DA_workflow/fix/FV3_input_data/INPUT && \ 
+    ln -s /opt/land-DA_workflow/fix/FV3_fix_tiled/C96/* /opt/land-DA_workflow/fix/FV3_input_data/INPUT
+RUN mkdir -p /opt/land-DA_workflow/fix/NOAHMP_IC/CLMNCEP && \ 
+    mv /opt/land-DA_workflow/fix/NOAHMP_IC/*.nc /opt/land-DA_workflow/fix/NOAHMP_IC/CLMNCEP/
 
 #build & unit testing
-WORKDIR $HOME/land-offline_workflow
-RUN source /opt/spack-stack/.bashenv; mkdir build; cd build; pwd; ecbuild ..; make -j2; ctest -V --stop-on-failure
+WORKDIR $HOME/land-DA_workflow
+#RUN source /opt/spack-stack/.bashenv; mkdir build; cd build; pwd; ecbuild ..; make -j2; ctest -V --stop-on-failure
+RUN source /opt/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/.bashenv-fms && \
+    module load bacio cmake crtm ecbuild esmf fms gftl-shared g2 g2tmpl hdf5 ip jasper libpng mapl netcdf-c netcdf-fortran parallelio && \
+    module load prod_util py-netcdf4 py-numpy py-pyyaml py-jinja2 py-xarray sp ufs-pyenv w3emc zlib scotch && \
+    cd build; pwd; ctest -V --stop-on-failure

ush/hofx_analysis_stats.py

@@ -68,6 +68,8 @@ def plot_scatter():
         cartopy.config['data_dir']='/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs/NaturalEarth'
     elif yaml_data['machine']=='orion' or yaml_data['machine']=='hercules':
         cartopy.config['data_dir']='/work/noaa/epic/UFS_Land-DA_Dev/inputs/NaturalEarth'
+    elif yaml_data['machine']=='singularity':
+        cartopy.config['data_dir']='SINGULARITY_WORKING_DIR/land-DA_workflow/fix/NaturalEarth'
 
     field_mean=float("{:.2f}".format(np.mean(np.absolute(field))))
     field_std=float("{:.2f}".format(np.std(np.absolute(field))))

ush/plot_forecast_restart.py

@@ -50,6 +50,8 @@ def main():
         cartopy.config['data_dir']='/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs/NaturalEarth'
     elif yaml_data['machine']=='orion' or yaml_data['machine']=='hercules':
         cartopy.config['data_dir']='/work/noaa/epic/UFS_Land-DA_Dev/inputs/NaturalEarth'
+    elif yaml_data['machine']=='singularity':
+        cartopy.config['data_dir']='SINGULARITY_WORKING_DIR/land-DA_workflow/fix/NaturalEarth'
 
     #var_list=["snwdph","smc"]
     var_list=["snwdph"]