Data structure for ctest is re-organized and cleaned up (ufs-communit…

…y#115) * clean up ctest scripts * fix issues on paths
chan-hoo · Jun 17, 2024 · 3391683 · 3391683
1 parent b9cd30a
commit 3391683
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 32 deletions.
diff --git a/sorc/test/ci/Dockerfile b/sorc/test/ci/Dockerfile
@@ -13,7 +13,7 @@ COPY . $HOME/land-offline_workflow
 ENV FIXlandda=$HOME/land-DA_workflow/fix
 ENV EPICHOME=/opt
 ENV JEDI_INSTALL=${EPICHOME}/jedi_skylabv7.0
-ENV TEST_BASEDIR=${EPICHOME}/test_base/mem000/restarts/vector"}
+ENV TEST_BASEDIR=${EPICHOME}/test_base/restarts/vector"}
 
 #build & unit testing
 WORKDIR $HOME/land-offline_workflow

diff --git a/sorc/test/retrieve_data.sh b/sorc/test/retrieve_data.sh
@@ -22,23 +22,19 @@ pip3 install awscli --upgrade --user
 export PATH=${HOME}/.local/bin:${PATH}
 
 # set envs
-DATA_ROOT=${project_source_dir}/../../inputs
-INPUTDATA_ROOT=${DATA_ROOT}/NEMSfv3gfs
-source ${PATHRT}/bl_date.conf
-INPUTDATA_DATE=20221101
-[[ ! -d ${INPUTDATA_ROOT}/develop-${BL_DATE}/${RT_COMPILER^^} ]] && mkdir -p ${INPUTDATA_ROOT}/develop-${BL_DATE}/${RT_COMPILER^^}
-[[ ! -d ${INPUTDATA_ROOT}/input-data-${INPUTDATA_DATE} ]] && mkdir -p ${INPUTDATA_ROOT}/input-data-${INPUTDATA_DATE}
-RTPWD=${INPUTDATA_ROOT}/develop-${BL_DATE}/${RT_COMPILER^^}
-INPUTDATA_ROOT=${INPUTDATA_ROOT}/input-data-${INPUTDATA_DATE}
-AWS_URL=s3://noaa-ufs-regtests-pds
-SRC_DIR=${AWS_URL}/input-data-${INPUTDATA_DATE}
+DATA_ROOT=${project_source_dir}/../fix
+INPUTDATA_ROOT=${DATA_ROOT}/UFS_WM
+
+[[ ! -d ${INPUTDATA_ROOT} ]] && mkdir -p ${INPUTDATA_ROOT}
+RTPWD=${DATA_ROOT}/test_base
+AWS_URL=s3://noaa-ufs-land-da-pds/develop/inputs
 
 # baseline data
-DES_DIR=${RTPWD}/datm_cdeps_lnd_gswp3
+DES_DIR=${RTPWD}/datm_cdeps_lnd_gswp3_intel
 [[ ! -d ${DES_DIR} ]] && mkdir -p ${DES_DIR}
 echo ${DES_DIR}
 cd $DES_DIR
-aws s3 sync --no-sign-request ${AWS_URL}/develop-${BL_DATE}/datm_cdeps_lnd_gswp3_${RT_COMPILER,,} .
+aws s3 sync --no-sign-request ${AWS_URL}/test_base/datm_cdeps_lnd_gswp3_intel .
 cd ${project_source_dir}
 
 # DATM data
@@ -54,7 +50,7 @@ DES_DIR=${INPUTDATA_ROOT}/FV3_fix_tiled/C96
 [[ ! -d ${DES_DIR} ]] && mkdir -p ${DES_DIR}
 echo ${DES_DIR}
 cd $DES_DIR
-aws s3 sync --no-sign-request ${SRC_DIR}/FV3_fix_tiled/C96 .
+aws s3 sync --no-sign-request ${AWS_URL}/FV3_fix_tiled/C96 .
 
 # input data
 DES_DIR=${INPUTDATA_ROOT}/FV3_input_data/INPUT

diff --git a/sorc/test/run_ufs_datm_lnd.sh b/sorc/test/run_ufs_datm_lnd.sh
@@ -13,6 +13,8 @@ echo ${project_source_dir}
 MACHINE_ID=${MACHINE_ID:-hera}
 TEST_NAME=datm_cdeps_lnd_gswp3
 PATHRT=${project_source_dir}/ufs_model.fd/tests
+FIXdir=${project_source_dir}/../fix
+INPUTDATA_ROOT=${FIXdir}/UFS_WM
 RT_COMPILER=${RT_COMPILER:-intel}
 ATOL="1e-7"
 source ${PATHRT}/detect_machine.sh
@@ -21,23 +23,10 @@ source ${PATHRT}/default_vars.sh
 source ${PATHRT}/tests/$TEST_NAME
 source ${PATHRT}/atparse.bash
 
-# Set inputdata location for each machines
-echo "MACHINE_ID: $MACHINE_ID"
-if [[ $MACHINE_ID = orion ]]; then
-  DISKNM=/work/noaa/epic/UFS-WM_RT
-elif [[ $MACHINE_ID = hera ]]; then
-  DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT
-else
-  echo "Warning: MACHINE_ID is default, users will have to define INPUTDATA_ROOT and RTPWD by themselves"
-fi
-
-source ${PATHRT}/bl_date.conf
-#BL_DATE=20230815
-RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${TEST_NAME}_${RT_COMPILER}}
-INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101}
+RTPWD=${RTPWD:-$FIXdir/test_base/${TEST_NAME}_${RT_COMPILER}}
 
-if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then
-  echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!"
+if [[ ! -d ${RTPWD} ]]; then
+  echo "Error: cannot find RTPWD, please check!"
   exit 1
 fi  
 

diff --git a/sorc/test/test_ufs_land_driver.sh b/sorc/test/test_ufs_land_driver.sh
@@ -9,7 +9,7 @@ project_source_dir=$2
 source ${project_source_dir}/test/runtime_vars.sh ${project_binary_dir} ${project_source_dir}
 
 # set baseline dir
-export TEST_BASEDIR=${TEST_BASEDIR:-"${EPICHOME}/test_base/mem000/restarts/vector"}
+export TEST_BASEDIR=${TEST_BASEDIR:-"${EPICHOME}/inputs/test_base/restarts/vector"}
 
 # compute the restart frequency, run_days and run_hours
 FREQ=$(( 3600 * $FCSTHR ))

diff --git a/sorc/test/test_vector2tile.sh b/sorc/test/test_vector2tile.sh
@@ -11,7 +11,7 @@ prefix=$4 #bkg or ana
 source ${project_source_dir}/test/runtime_vars.sh ${project_binary_dir} ${project_source_dir}
 
 # set baseline dir
-export TEST_BASEDIR=${TEST_BASEDIR:-"${EPICHOME}/test_base/mem000/restarts/vector"}
+export TEST_BASEDIR=${TEST_BASEDIR:-"${EPICHOME}/inputs/test_base/restarts/vector"}
 
 # set executables
 TEST_EXEC="vector2tile_converter.exe"