[develop] Build conda and environments in SRW. (ufs-community#938)

Modifies devbuild.sh to add the option to install miniforge (a version of miniconda that manages channels more strictly) in a user's specified location and defaults to inside the user's clone. It also installs two environments needed for SRW - srw_app, which is similar to the old workflow_tools environment, and srw_graphics, which is sufficient to support the plotting scripts in SRW.  If the SRW App is being built for AQM, then another environment, srw_aqm, will also be installed.

A few additional details:

* Does the conda installation right away so that MacOS users have the necessary bash utilities like readlink to use the devbuild.sh script.
* Adds a conda module file that points to the user-installed location of miniconda.
* Modifies the GitHub Actions workflows to use the same environments that are built by devbuild.sh.

---------

Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com>
Co-authored-by: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com>
Co-authored-by: Michael Kavulich <kavulich@ucar.edu>

.cicd/Jenkinsfile

@@ -137,16 +137,6 @@ pipeline {
                         }
                     }
 
-                    // Run the unittest functional tests that require an HPC platform
-                    stage('Functional UnitTests') {
-                        steps {
-                            dir ("${env.SRW_PLATFORM}") {
-                            echo "Running unittest on retrieve_data.py"
-                            sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"'
-                            }
-                        }
-                    }
-
                     // Run the unified build script; if successful create a tarball of the build and upload to S3
                     stage('Build') {
                         steps {
@@ -164,6 +154,16 @@ pipeline {
                         }
                     }
 
+                    // Run the unittest functional tests that require an HPC platform
+                    stage('Functional UnitTests') {
+                        steps {
+                            dir ("${env.SRW_PLATFORM}") {
+                            echo "Running unittest on retrieve_data.py"
+                            sh 'bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_unittest.sh"'
+                            }
+                        }
+                    }
+
                     // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage
                     stage('Functional WorkflowTaskTests') {
                         steps {

.cicd/scripts/srw_ftest.sh

@@ -85,7 +85,7 @@ module load build_${platform,,}_${SRW_COMPILER}
 module load wflow_${platform,,}
 
 [[ ${FORGIVE_CONDA} == true ]] && set +e +u    # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests
-conda activate workflow_tools
+conda activate srw_app
 set -e -u
 
 export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src

.github/workflows/python_tests.yaml

@@ -0,0 +1,58 @@
+name: Python Tests
+on:
+  push:
+    branches:
+      - develop
+      - 'release/*'
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+jobs:
+  srw_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Micromamba and srw_app environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          environment-name: srw_app
+          init-shell: bash
+          cache-downloads: true
+          cache-environment: true
+
+      - name: Checkout externals
+        run: |
+          ./manage_externals/checkout_externals ufs-weather-model workflow-tools
+
+      - name: Lint the python code
+        run: |
+          micromamba activate srw_app
+          export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src
+          pylint --ignore-imports=yes tests/test_python/
+          pylint ush/create_*.py
+          pylint ush/generate_FV3LAM_wflow.py
+
+      - name: Run python unittests
+        run: |
+          # exclude test_retrieve_data that is tested in functional test
+          micromamba activate srw_app
+          export UNIT_TEST=True
+          export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src
+          python -m unittest -b tests/test_python/*.py
+
+      - name: Run python functional tests
+        run: |
+          micromamba activate srw_app
+          export CI=true
+          export PYTHONPATH=${PWD}/ush
+          python3 -m unittest -b tests/test_python/test_retrieve_data.py

.gitignore

@@ -15,6 +15,7 @@ ush/config.yaml
 ush/python_utils/__pycache__/
 ush/python_utils/workflow-tools/
 ush/*.swp
+conda_loc
 
 *.swp
 __pycache__

aqm_environment.yml

@@ -0,0 +1,13 @@
+name: srw_aqm
+channels:
+  - conda-forge
+  - ufs-community
+dependencies:
+  - esmpy
+  - netcdf4=1.6.*
+  - numpy=1.23.*
+  - pylint=2.17*
+  - pytest=7.2*
+  - scipy=1.10.*
+  - uwtools=1.0.0
+  - xarray=2022.11.*

devbuild.sh

@@ -41,6 +41,8 @@ OPTIONS
       installation prefix
   --bin-dir=BIN_DIR
       installation binary directory name ("exec" by default; any name is available)
+  --conda-dir=CONDA_DIR
+      installation location for miniconda (SRW clone conda subdirectory by default)
   --build-type=BUILD_TYPE
       build type; defaults to RELEASE
       (e.g. DEBUG | RELEASE | RELWITHDEBINFO)
@@ -52,9 +54,11 @@ OPTIONS
       build with verbose output
 
 TARGETS
-   default = builds the default list of apps (also not passing any target does the same)
-   all = builds all apps
-   Or any combinations of (ufs, ufs_utils, upp)
+   default = builds the default list of components for the specified application
+            (also not passing any target does the same)
+   all = builds all standard components for ATM
+   conda_only = installs miniconda, but no other 
+   Or any combinations of (ufs, ufs_utils, upp, nexus, aqm_utils)
 
 NOTE: See User's Guide for detailed build instructions
 
@@ -99,11 +103,8 @@ usage_error () {
 
 # default settings
 LCL_PID=$$
-SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
-MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh
-BUILD_DIR="${SRW_DIR}/build"
-INSTALL_DIR=${SRW_DIR}
 BIN_DIR="exec"
+CONDA_BUILD_DIR="conda"
 COMPILER=""
 APPLICATION=""
 CCPP_SUITES=""
@@ -117,6 +118,7 @@ VERBOSE=false
 
 # Turn off all apps to build and choose default later
 DEFAULT_BUILD=true
+BUILD_CONDA="on"
 BUILD_UFS="off"
 BUILD_UFS_UTILS="off"
 BUILD_UPP="off"
@@ -164,6 +166,8 @@ while :; do
     --install-dir|--install-dir=) usage_error "$1 requires argument." ;;
     --bin-dir=?*) BIN_DIR=${1#*=} ;;
     --bin-dir|--bin-dir=) usage_error "$1 requires argument." ;;
+    --conda-dir=?*) CONDA_BUILD_DIR=${1#*=} ;;
+    --conda-dir|--conda-dir=) usage_error "$1 requires argument." ;;
     --build-type=?*) BUILD_TYPE=${1#*=} ;;
     --build-type|--build-type=) usage_error "$1 requires argument." ;;
     --build-jobs=?*) BUILD_JOBS=$((${1#*=})) ;;
@@ -175,6 +179,7 @@ while :; do
     default) ;;
     all) DEFAULT_BUILD=false; BUILD_UFS="on";
          BUILD_UFS_UTILS="on"; BUILD_UPP="on";;
+    conda_only) DEFAULT_BUILD=false;;
     ufs) DEFAULT_BUILD=false; BUILD_UFS="on" ;;
     ufs_utils) DEFAULT_BUILD=false; BUILD_UFS_UTILS="on" ;;
     upp) DEFAULT_BUILD=false; BUILD_UPP="on" ;;
@@ -188,10 +193,9 @@ while :; do
 done
 
 # Ensure uppercase / lowercase ============================================
-APPLICATION="${APPLICATION^^}"
-PLATFORM="${PLATFORM,,}"
-COMPILER="${COMPILER,,}"
-EXTERNALS="${EXTERNALS^^}"
+APPLICATION=$(echo ${APPLICATION} | tr '[a-z]' '[A-Z]')
+PLATFORM=$(echo ${PLATFORM} | tr '[A-Z]' '[a-z]')
+COMPILER=$(echo ${COMPILER} | tr '[A-Z]' '[a-z]')
 
 # check if PLATFORM is set
 if [ -z $PLATFORM ] ; then
@@ -203,6 +207,55 @@ fi
 MACHINE="${PLATFORM}"
 printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2
 
+
+# Conda is not used on WCOSS2
+if [ "${PLATFORM}" = "wcoss2" ]; then
+    BUILD_CONDA="off"
+fi
+
+# build conda and conda environments, if requested.
+if [ "${BUILD_CONDA}" = "on" ] ; then
+  if [ ! -d "${CONDA_BUILD_DIR}" ] ; then
+    os=$(uname)
+    test $os == Darwin && os=MacOSX
+    hardware=$(uname -m)
+    installer=Miniforge3-${os}-${hardware}.sh
+    curl -L -O "https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/${installer}"
+    bash ./${installer} -bfp "${CONDA_BUILD_DIR}"
+    rm ${installer}
+  fi
+
+  source ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh
+  # Put some additional packages in the base environment on MacOS systems
+  if [ "${os}" == "MacOSX" ] ; then
+    mamba install -y bash coreutils sed
+  fi
+  conda activate
+  if ! conda env list | grep -q "^srw_app\s" ; then
+    mamba env create -n srw_app --file environment.yml
+  fi
+  if ! conda env list | grep -q "^srw_graphics\s" ; then
+    mamba env create -n srw_graphics --file graphics_environment.yml
+  fi
+  if [ "${APPLICATION}" = "ATMAQ" ]; then
+    if ! conda env list | grep -q "^srw_aqm\s" ; then
+      mamba env create -n srw_aqm --file aqm_environment.yml
+    fi
+  fi
+
+else
+  source ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh
+  conda activate
+fi
+
+# Conda environment should have linux utilities to perform these tasks on macos.
+SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
+MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh
+BUILD_DIR="${BUILD_DIR:-${SRW_DIR}/build}"
+INSTALL_DIR=${INSTALL_DIR:-$SRW_DIR}
+CONDA_BUILD_DIR="$(readlink -f "${CONDA_BUILD_DIR}")"
+echo ${CONDA_BUILD_DIR} > ${SRW_DIR}/conda_loc
+
 # choose default apps to build
 if [ "${DEFAULT_BUILD}" = true ]; then
   BUILD_UFS="on"