From 44d4e7cea5f19aeecfb3b693c7d294856e559758 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 19 Sep 2024 16:02:59 -0600
Subject: [PATCH 01/61] Update util/weekly_build/ShellSetup.sh for Nautilus and
 add first step of nightly builds Nautilus using GitHub actions

---
 .../Nightly_01_DirectorySetup/action.yaml     | 14 ++++++++
 .github/workflows/nautilus-nightly.yaml       | 32 +++++++++++++++++++
 util/weekly_build/ShellSetup.sh               |  4 +--
 3 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 .github/actions/Nightly_01_DirectorySetup/action.yaml
 create mode 100644 .github/workflows/nautilus-nightly.yaml

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
new file mode 100644
index 000000000..e62caa2d8
--- /dev/null
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -0,0 +1,14 @@
+name: Nightly 01 Directory Setup
+#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 01 Directory Setup
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      RUNDIR=${BASE_DIR}/${RUNID}
+      [ -d {RUNDIR} ] && rm -fr ${RUNDIR}
+      cd util/weekly_build
+      ./1_DirectorySetup.sh ${RUNID} ${RUNDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml
new file mode 100644
index 000000000..08341e039
--- /dev/null
+++ b/.github/workflows/nautilus-nightly.yaml
@@ -0,0 +1,32 @@
+name: nautilus heinzell auto ci
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - develop
+    types: [opened, synchronize, reopened]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BASE_DIR: /p/work2/heinzell/spack-stack-nightly
+  PLATFORM: nautilus
+
+jobs:
+
+  nightly-build:
+    runs-on: ['self-hosted','nautilus-heinzell']
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        clean: true
+    - name: 01_DirectorySetup
+      uses: ./.github/actions/Nightly_01_DirectorySetup
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 2700bdd4e..46ad0ef5a 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -86,8 +86,8 @@ case $PLATFORM in
     BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache}
     ;;
   nautilus)
-    COMPILERS=${COMPILERS:-"intel"}
-    BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache}
+    COMPILERS=${COMPILERS:-"intel oneapi gcc"}
+    BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build-cache}
     ;;
   s4)
     COMPILERS=${COMPILERS:-"intel"}

From 544f5a73e8a3bde216366b0ca7652bfdc6e49bbf Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 19 Sep 2024 16:12:15 -0600
Subject: [PATCH 02/61] Add step 2 for nightly builds on Nautilus:
 GetSpackStackDevelop

---
 .../actions/Nightly_01_DirectorySetup/action.yaml    |  8 +++++---
 .../Nightly_02_GetSpackStackDevelop/action.yaml      | 12 ++++++++++++
 .github/workflows/nautilus-nightly.yaml              |  4 +++-
 util/weekly_build/2_GetSpackStackDevelop.sh          |  3 ++-
 4 files changed, 22 insertions(+), 5 deletions(-)
 create mode 100644 .github/actions/Nightly_02_GetSpackStackDevelop/action.yaml

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
index e62caa2d8..d1ebb76c3 100644
--- a/.github/actions/Nightly_01_DirectorySetup/action.yaml
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -8,7 +8,9 @@ runs:
     shell: bash
     run: |
       RUNID=`date +"%Y%m%d"`
-      RUNDIR=${BASE_DIR}/${RUNID}
-      [ -d {RUNDIR} ] && rm -fr ${RUNDIR}
+      # DH* TODO REMOVE LATER?
+      RUNDIR=${BASEDIR}/${RUNID}
+      rm -vfr ${RUNDIR}
+      # *DH
       cd util/weekly_build
-      ./1_DirectorySetup.sh ${RUNID} ${RUNDIR} ${PLATFORM}
+      ./1_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
new file mode 100644
index 000000000..cb6141781
--- /dev/null
+++ b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
@@ -0,0 +1,12 @@
+name: Nightly 02 Get Spack-Stack Develop
+#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 02 Get Spack-Stack Develop
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      cd util/weekly_build
+      ./2_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml
index 08341e039..07bdcce09 100644
--- a/.github/workflows/nautilus-nightly.yaml
+++ b/.github/workflows/nautilus-nightly.yaml
@@ -16,7 +16,7 @@ defaults:
     shell: bash
 
 env:
-  BASE_DIR: /p/work2/heinzell/spack-stack-nightly
+  BASEDIR: /p/work2/heinzell/spack-stack-nightly
   PLATFORM: nautilus
 
 jobs:
@@ -30,3 +30,5 @@ jobs:
         clean: true
     - name: 01_DirectorySetup
       uses: ./.github/actions/Nightly_01_DirectorySetup
+    - name: 02_GetSpackStackDevelop
+      uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
\ No newline at end of file
diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh
index 67e95b136..fc61ffa46 100755
--- a/util/weekly_build/2_GetSpackStackDevelop.sh
+++ b/util/weekly_build/2_GetSpackStackDevelop.sh
@@ -9,5 +9,6 @@ if [ -d $RUNID ]; then
   cd $RUNID
   git pull
 else
-  git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID
+  #git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID
+  git clone --recurse-submodules https://github.nrlmry.navy.mil/JCSDA/spack-stack -b feature/weekly_build_nautilus $RUNID
 fi

From e6e0b772be1c598e2ae2a3030f3cf6ac536bd1b5 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 19 Sep 2024 16:26:52 -0600
Subject: [PATCH 03/61] Revert me: remove other CI workflows on self-hosted
 runners

---
 .github/workflows/macos-ci-aarch64.yaml       | 178 -------------
 .../ubuntu-ci-containers-x86_64.yaml          | 111 --------
 .github/workflows/ubuntu-ci-x86_64-gnu.yaml   | 209 ---------------
 .github/workflows/ubuntu-ci-x86_64-intel.yaml | 241 -----------------
 .../workflows/ubuntu-ci-x86_64-oneapi.yaml    | 249 ------------------
 5 files changed, 988 deletions(-)
 delete mode 100644 .github/workflows/macos-ci-aarch64.yaml
 delete mode 100644 .github/workflows/ubuntu-ci-containers-x86_64.yaml
 delete mode 100644 .github/workflows/ubuntu-ci-x86_64-gnu.yaml
 delete mode 100644 .github/workflows/ubuntu-ci-x86_64-intel.yaml
 delete mode 100644 .github/workflows/ubuntu-ci-x86_64-oneapi.yaml

diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml
deleted file mode 100644
index 3087b7f1f..000000000
--- a/.github/workflows/macos-ci-aarch64.yaml
+++ /dev/null
@@ -1,178 +0,0 @@
-name: macos-ci-aarch64-build
-on:
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  macos-ci-aarch64-build:
-    runs-on: [macos-ci-aarch64]
-
-    steps:
-      - name: cleanup
-        run: |
-          pwd
-          ls -lart
-          find ./* -type d -exec chmod u+xw {} \;
-          rm -fr *
-
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - name: prepare-directories
-        run: |
-          mkdir -p /Users/ec2-user/spack-stack/build-cache
-          mkdir -p /Users/ec2-user/spack-stack/source-cache
-
-      - name: create-env
-        run: |
-          # Get day of week to decide whether to use build caches or not
-          DOW=$(date +%u)
-          # Monday is 1 ... Sunday is 7
-          if [[ $DOW == 7 ]]; then
-            export USE_BINARY_CACHE=false
-            echo "Ignore existing binary cache for creating buildcache environment"
-          else
-            export USE_BINARY_CACHE=true
-            echo "Use existing binary cache for creating buildcache environment"
-          fi
-
-          # Set up homebrew and lmod support
-          eval "$(/opt/homebrew/bin/brew shellenv)"
-          source /opt/homebrew/opt/lmod/init/profile
-
-          # Set up spack-stack
-          source ./setup.sh
-          # Important!
-          export SPACK_PYTHON=/usr/bin/python3
-          export ENVNAME=ue-apple-clang-14.0.3
-          export ENVDIR=$PWD/envs/${ENVNAME}
-
-          spack clean -ab
-          spack bootstrap now
-
-          spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang
-          spack env activate ${ENVDIR}
-          spack add ${{ inputs.specs || '' }}
-          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
-
-          # Find external packages
-          spack external find --scope system \
-              --exclude bison --exclude openssl \
-              --exclude curl --exclude python
-          spack external find --scope system perl
-          spack external find --scope system libiconv
-          spack external find --scope system wget
-          PATH="/opt/homebrew/opt/curl/bin:$PATH" \
-              spack external find --scope system curl
-          PATH="/opt/homebrew/opt/qt5/bin:$PATH" \
-              spack external find --scope system qt
-          spack external find --scope system texlive
-          spack external find --scope system mysql
-
-          # Find compilers
-          spack compiler find --scope system
-
-          export -n SPACK_SYSTEM_CONFIG_PATH
-
-          # Set compiler and MPI specs
-          spack config add "packages:all:providers:mpi:[openmpi@5.0.3]"
-          spack config add "packages:all:compiler:[apple-clang@14.0.3]"
-
-          # Add additional variants for MET packages, different from config/common/packages.yaml
-          # DH* 20240513 - avoid hdf-eos2 until https://github.com/spack/spack/issues/44168 is resolved
-          #spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-          spack config add "packages:met:variants:+python +grib2 +graphics"
-          # *DH
-
-          # Concretize and check for duplicates
-          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl
-
-          # Add and update source cache
-          spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
-          spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/
-
-          # Add binary cache if requested
-          if [ "$USE_BINARY_CACHE" = true ] ; then
-            set +e
-            spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
-            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
-            echo "Packages in spack binary cache:"
-            spack buildcache list
-          fi
-
-          # Break installation up in pieces and create build caches in between
-          # This allows us to "spin up" builds that altogether take longer than
-          # six hours, and/or fail later in the build process.
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
-
-          # jedi-base-env
-          echo "jedi-base-env ..."
-          spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
-
-          # the rest
-          echo "unified-env ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
-
-          # Remove binary cache for next round of concretization
-          if [ "$USE_BINARY_CACHE" = true ] ; then
-            spack mirror rm local-binary
-          fi
-
-          # Create modules
-          spack clean -a
-          spack module lmod refresh -y
-          spack stack setup-meta-modules
-
-          # Next steps: synchronize source and build cache to a central/combined mirror?
-          spack env deactivate
-
-          # Test environment chaining
-          echo "Test environment chaining"
-          spack stack create env --name chaintest --template empty --site macos.default --upstream ${ENVDIR}/install --compiler apple-clang
-          # Retain config from upstream so we don't have to rebuild:
-          cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
-          spack env activate ${PWD}/envs/chaintest
-          spack add nccmp@1.9.0.1%apple-clang
-          spack concretize | tee envs/chaintest/log.concretize
-          unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l)
-          if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
-          spack env deactivate
-
-      - name: test-env
-        run: |
-          # Set up homebrew and lmod support
-          eval "$(/opt/homebrew/bin/brew shellenv)"
-          source /opt/homebrew/opt/lmod/init/profile
-
-          export ENVNAME=ue-apple-clang-14.0.3
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
-
-          module use ${ENVDIR}/install/modulefiles/Core
-          module load stack-apple-clang/14.0.3
-          module load stack-openmpi/5.0.3
-          module load stack-python/3.11.7
-          module available
-
-          module load jedi-ufs-env
-          module load ewok-env
-          module load soca-env
-          module list
diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/ubuntu-ci-containers-x86_64.yaml
deleted file mode 100644
index b9265c094..000000000
--- a/.github/workflows/ubuntu-ci-containers-x86_64.yaml
+++ /dev/null
@@ -1,111 +0,0 @@
-name: ubuntu-ci-container-x86_64-build
-on:
-  # Uncomment this to test for PRs (but do not submit)
-  #pull_request:
-  #  paths-ignore:
-  #    - 'configs/sites/**'
-  #    - 'doc/**'
-  #    - '**.md'
-  #    - '.github/ISSUE_TEMPLATE/*'
-  #    - '.gitignore'
-  # 
-  schedule:
-    - cron: '0 8 * * *'
-  workflow_dispatch:
-    inputs:
-      container:
-        description: 'Container template (input requred, no default value).'
-        required: true
-      specs:
-        description: 'Which specs to add to the template  (input requred, no default value).'
-        required: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  ubuntu-ci-container-x86_64-build:
-    runs-on: [ubuntu-ci-c6a-x86_64]
-
-    steps:
-      - name: checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-
-      # Ensure that ubuntu can run docker
-      - name: ubuntu-docker
-        run: |
-          echo "Checking if user ubuntu can run docker"
-          if id -nG ubuntu | grep -qw docker; then
-            echo "User ubuntu already belongs to group docker"
-          else
-            echo "Adding user ubuntu to group docker"
-            sudo gpasswd -a ubuntu docker
-            newgrp docker
-            docker run hello-world
-          fi
-
-      - name: create-ctr
-        run: |
-          source ./setup.sh
-
-          # Get day of week to set default container for scheduled builds
-          DOW=$(date +%u)
-          # Monday is 1 ... Sunday is 7
-          if [[ $DOW == 1 || $DOW == 4 ]]; then
-            export CONTAINER=${{ inputs.container || 'docker-ubuntu-clang-mpich' }}
-            export SPECS=${{ inputs.specs || 'jedi-ci' }}
-          elif [[ $DOW == 2 || $DOW == 5 ]]; then
-            export CONTAINER=${{ inputs.container || 'docker-ubuntu-gcc-openmpi' }}
-            export SPECS=${{ inputs.specs || 'jedi-ci' }}
-          elif [[ $DOW == 3 || $DOW == 6 ]]; then
-            export CONTAINER=${{ inputs.container || 'docker-ubuntu-intel-impi' }}
-            export SPECS=${{ inputs.specs || 'jedi-ci' }}
-          else
-            # Day 7: The Sabbath of rest (but do some house keeping later on)
-            exit 0
-          fi
-
-          export ENVDIR=$PWD/envs/${CONTAINER}
-          spack stack create ctr --container ${CONTAINER} --specs ${SPECS}
-
-          cd ${ENVDIR}
-          spack containerize > Dockerfile
-          docker build -t ${CONTAINER}-${SPECS} .
-
-      # Report status to JCSDA CI slack channel for nightly runs only
-      - name: Report Status
-        if: always()
-        uses: ravsamhq/notify-slack-action@v1
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-        with:
-          # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url
-          # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script
-          # https://github.com/marketplace/actions/notify-slack-action
-          status: ${{ job.status }}
-          notify_when: 'success,failure,warnings'
-          notification_title: '{workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>'
-          footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }}
-          # For testing: only notify user Dom
-          #mention_users: 'U02NLGXF5HV'
-          #mention_users_when: 'failure,warnings'
-          # Default: notify channel
-          mention_groups: '!channel'
-          mention_groups_when: 'failure,warnings'
-
-      # We can only clean the docker image registry *after* reporting
-      # the status to the CI slack channel, because the cleanup is
-      # deleting the docker image for ravsamhq/notify-slack-action@v1
-      - name: clean-docker-registry
-        if: always()
-        run: |
-          # Get day of week (only run this on Sunday, which is 7)
-          DOW=$(date +%u)
-          if [[ $DOW == 7 ]]; then
-            echo "Pruning all docker images"
-            docker system prune -a -f
-          fi
diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
deleted file mode 100644
index 13414615c..000000000
--- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
+++ /dev/null
@@ -1,209 +0,0 @@
-name: ubuntu-ci-c6a-x86_64-gnu-build
-on:
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  ubuntu-ci-c6a-x86_64-gnu-build:
-    runs-on: [ubuntu-ci-c6a-x86_64]
-
-    steps:
-      - name: cleanup
-        run: |
-          pwd
-          ls -lart
-          find ./* -type d -exec chmod u+xw {} \;
-          rm -fr *
-
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - name: prepare-directories
-        run: |
-          mkdir -p /home/ubuntu/spack-stack/build-cache/
-          mkdir -p /home/ubuntu/spack-stack/source-cache/
-
-      - name: create-buildcache
-        run: |
-          # Get day of week to decide whether to use build caches or not
-          DOW=$(date +%u)
-          # Monday is 1 ... Sunday is 7
-          if [[ $DOW == 7 ]]; then
-            export USE_BINARY_CACHE=false
-            echo "Ignore existing binary cache for creating buildcache environment"
-          else
-            export USE_BINARY_CACHE=true
-            echo "Use existing binary cache for creating buildcache environment"
-          fi
-
-          # Set up spack-stack
-          source ./setup.sh
-
-          declare -a TEMPLATES=("unified-dev" "skylab-dev")
-          for TEMPLATE in "${TEMPLATES[@]}"; do
-            if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then
-              export ENVNAME=ue-gcc-11.4.0-buildcache
-            elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then
-              export ENVNAME=se-gcc-11.4.0-buildcache
-            fi
-            echo "Creating environment ${ENVNAME} from template ${TEMPLATE}"
-
-            export ENVDIR=$PWD/envs/${ENVNAME}
-            spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler gcc
-            spack env activate ${ENVDIR}
-            export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
-
-            # Find external packages
-            spack external find --scope system \
-                --exclude bison --exclude openssl \
-                --exclude curl --exclude python
-            spack external find --scope system sed
-            spack external find --scope system perl
-            spack external find --scope system wget
-            spack external find --scope system texlive
-            spack external find --scope system mysql
-
-            # Find compilers
-            spack compiler find --scope system
-
-            export -n SPACK_SYSTEM_CONFIG_PATH
-
-            # For buildcaches
-            spack config add config:install_tree:padded_length:200
-
-            # Set compiler and MPI specs
-            spack config add "packages:all:providers:mpi:[openmpi@5.0.3]"
-            spack config add "packages:all:compiler:[gcc@11.4.0]"
-
-            # Add additional variants for MET packages, different from config/common/packages.yaml
-            spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-
-            # Concretize and check for duplicates
-            spack concretize 2>&1 | tee log.concretize.gnu-11.4.0-buildcache
-            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0-buildcache -i fms -i crtm -i esmf -i mapl
-
-            # Add and update source cache
-            spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
-            spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/
-
-            # Add binary cache if requested
-            if [ "$USE_BINARY_CACHE" = true ] ; then
-              set +e
-              spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-              spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-              set +e
-              echo "Packages in spack binary cache:"
-              spack buildcache list
-            fi
-
-            # Break installation up in pieces and create build caches in between
-            # This allows us to "spin up" builds that altogether take longer than
-            # six hours, and/or fail later in the build process.
-
-            # base-env
-            echo "base-env ..."
-            spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
-
-            # jedi-base-env
-            echo "jedi-base-env ..."
-            spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
-
-            # the rest
-            echo "${TEMPLATE} ..."
-            spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
-
-            # Remove binary cache for next round of concretization
-            if [ "$USE_BINARY_CACHE" = true ] ; then
-              spack mirror rm local-binary
-            fi
-
-            # Remove buildcache config settings
-            spack config remove config:install_tree:padded_length
-
-            # Next steps: synchronize source and build cache to a central/combined mirror?
-
-            # Cleanup
-            spack clean -a
-            spack env deactivate
-
-          done
-
-      - name: create-env
-        run: |
-          source ./setup.sh
-          export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache
-          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
-          export ENVNAME=ue-gcc-11.4.0
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
-          spack env activate ${ENVDIR}
-
-          # Concretize
-          spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl
-
-          # Add binary cache back in and reindex it
-          spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-          echo "Packages in combined spack build caches:"
-          spack buildcache list
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env
-
-          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
-          spack clean -a
-          spack module tcl refresh -y
-          spack stack setup-meta-modules
-          spack env deactivate
-
-          # Test environment chaining
-          echo "Test environment chaining"
-          spack stack create env --name chaintest --template empty --site linux.default --compiler gcc --upstream ${ENVDIR}/install
-          # Retain config from upstream so we don't have to rebuild:
-          cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
-          spack env activate ${PWD}/envs/chaintest
-          spack add nccmp@1.9.0.1%gcc
-          spack concretize | tee envs/chaintest/log.concretize
-          unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l)
-          if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
-          spack env deactivate
-          echo "Verify 'create env' warnings"
-          echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/site/packages.yaml
-          echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml
-          spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt
-          cnt=$(grep -c "WARNING.*do not match" stderr.txt || true)
-          if [ $cnt -ne 2 ]; then echo "Missing 'create env' warnings"; exit 1; fi
-
-      - name: test-env
-        run: |
-          source /etc/profile.d/modules.sh
-          module use /home/ubuntu/spack-stack/modulefiles
-
-          export ENVNAME=ue-gcc-11.4.0
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
-
-          module use ${ENVDIR}/install/modulefiles/Core
-          module load stack-gcc/11.4.0
-          module load stack-openmpi/5.0.3
-          module load stack-python/3.11.7
-          module available
-
-          module load jedi-ufs-env
-          module load ewok-env
-          module load soca-env
-          module list
diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
deleted file mode 100644
index d1ed33d65..000000000
--- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml
+++ /dev/null
@@ -1,241 +0,0 @@
-name: ubuntu-ci-c6a-x86_64-intel-build
-on:
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  ubuntu-ci-c6a-x86_64-intel-build:
-    runs-on: [ubuntu-ci-c6a-x86_64]
-
-    steps:
-      - name: cleanup
-        run: |
-          pwd
-          ls -lart
-          find ./* -type d -exec chmod u+xw {} \;
-          rm -fr *
-
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - name: prepare-directories
-        run: |
-          mkdir -p /home/ubuntu/spack-stack/build-cache/
-          mkdir -p /home/ubuntu/spack-stack/source-cache/
-
-      - name: create-buildcache
-        run: |
-          # Get day of week to decide whether to use build caches or not
-          DOW=$(date +%u)
-          # Monday is 1 ... Sunday is 7
-          if [[ $DOW == 7 ]]; then
-            export USE_BINARY_CACHE=false
-            echo "Ignore existing binary cache for creating buildcache environment"
-          else
-            export USE_BINARY_CACHE=true
-            echo "Use existing binary cache for creating buildcache environment"
-          fi
-
-          # Set up spack-stack
-          source ./setup.sh
-          export ENVNAME=ue-intel-2021.10.0-buildcache
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler intel
-          spack env activate ${ENVDIR}
-          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
-
-          # Find external packages
-          spack external find --scope system \
-              --exclude bison --exclude openssl \
-              --exclude curl --exclude python
-          spack external find --scope system sed
-          spack external find --scope system perl
-          spack external find --scope system wget
-          spack external find --scope system texlive
-          spack external find --scope system mysql
-
-          # Find compilers
-          spack compiler find --scope system
-
-          # For Intel
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    spec: intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    environment:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      prepend_path:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "        LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-
-          # Need to find external Intel MPI and annotate with the
-          # correct compiler, no way to do that with spack commands.
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-
-          # Add external Intel MKL and oneAPI runtime
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          #echo "  intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          #echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          #echo "    - spec: intel-oneapi-runtime@2023.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          #echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-
-          # Add external qt@5 to build ecflow
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  qt::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    buildable: False" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: qt@5.15.3" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      prefix: /usr" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-
-          export -n SPACK_SYSTEM_CONFIG_PATH
-
-          # For buildcaches
-          spack config add config:install_tree:padded_length:200
-
-          # Set compiler and MPI specs
-          spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]"
-          spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]"
-
-          # Add additional variants for MET packages, different from config/common/packages.yaml
-          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-
-          # Don't generate qt module when using external package
-          spack config add "modules:default:tcl:exclude:[qt]"
-
-          # Concretize and check for duplicates
-          spack concretize 2>&1 | tee log.concretize.intel-2021.10.0-buildcache
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0-buildcache -i fms -i crtm -i esmf -i mapl
-
-          # Add and update source cache
-          spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
-          spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/
-
-          # Add binary cache if requested
-          if [ "$USE_BINARY_CACHE" = true ] ; then
-            set +e
-            spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
-            echo "Packages in spack binary cache:"
-            spack buildcache list
-          fi
-
-          # Break installation up in pieces and create build caches in between
-          # This allows us to "spin up" builds that altogether take longer than
-          # six hours, and/or fail later in the build process.
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
-
-          # jedi-base-env
-          echo "jedi-base-env ..."
-          spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
-
-          # the rest
-          echo "unified-env ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
-
-          # Remove binary cache for next round of concretization
-          if [ "$USE_BINARY_CACHE" = true ] ; then
-            spack mirror rm local-binary
-          fi
-
-          # Remove buildcache config settings
-          spack config remove config:install_tree:padded_length
-
-          # Next steps: synchronize source and build cache to a central/combined mirror?
-
-          # Cleanup
-          spack clean -a
-          spack env deactivate
-
-      - name: create-env
-        run: |
-          source ./setup.sh
-          export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache
-          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
-          export ENVNAME=ue-intel-2021.10.0
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
-          spack env activate ${ENVDIR}
-
-          # Concretize
-          spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl
-
-          # Add binary cache back in and reindex it
-          spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-          echo "Packages in combined spack build caches:"
-          spack buildcache list
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env
-
-          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
-          spack clean -a
-          spack module tcl refresh -y
-          spack stack setup-meta-modules
-          spack env deactivate
-
-          # Test environment chaining
-          echo "Test environment chaining"
-          spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel
-          # Retain config from upstream so we don't have to rebuild:
-          cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
-          spack env activate ${PWD}/envs/chaintest
-          spack add nccmp@1.9.0.1%intel
-          spack concretize | tee envs/chaintest/log.concretize
-          unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l)
-          if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
-          spack env deactivate
-
-      - name: test-env
-        run: |
-          source /etc/profile.d/modules.sh
-          module use /home/ubuntu/spack-stack/modulefiles
-
-          export ENVNAME=ue-intel-2021.10.0
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
-
-          module use ${ENVDIR}/install/modulefiles/Core
-          module load stack-intel/2021.10.0
-          module load stack-intel-oneapi-mpi/2021.10.0
-          module load stack-python/3.11.7
-          module available
-
-          module load jedi-ufs-env
-          module load ewok-env
-          module load soca-env
-          module list
diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
deleted file mode 100644
index e2bf9e604..000000000
--- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
+++ /dev/null
@@ -1,249 +0,0 @@
-name: ubuntu-ci-c6a-x86_64-oneapi-build
-on:
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  ubuntu-ci-c6a-x86_64-oneapi-build:
-    runs-on: [ubuntu-ci-c6a-x86_64]
-
-    steps:
-      - name: cleanup
-        run: |
-          pwd
-          ls -lart
-          find ./* -type d -exec chmod u+xw {} \;
-          rm -fr *
-
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - name: prepare-directories
-        run: |
-          mkdir -p /home/ubuntu/spack-stack/build-cache/
-          mkdir -p /home/ubuntu/spack-stack/source-cache/
-
-      - name: create-buildcache
-        run: |
-          # Get day of week to decide whether to use build caches or not
-          DOW=$(date +%u)
-          # Monday is 1 ... Sunday is 7
-          if [[ $DOW == 7 ]]; then
-            export USE_BINARY_CACHE=false
-            echo "Ignore existing binary cache for creating buildcache environment"
-          else
-            export USE_BINARY_CACHE=true
-            echo "Use existing binary cache for creating buildcache environment"
-          fi
-
-          # Set up spack-stack
-          source ./setup.sh
-          export ENVNAME=ue-oneapi-2024.2.0-buildcache
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi
-          spack env activate ${ENVDIR}
-          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
-
-          # Find external packages
-          spack external find --scope system \
-              --exclude bison --exclude openssl \
-              --exclude curl --exclude python \
-              --exclude gmake
-          spack external find --scope system sed
-          spack external find --scope system perl
-          spack external find --scope system wget
-          spack external find --scope system texlive
-          spack external find --scope system mysql
-
-          # Find compilers
-          spack compiler find --scope system
-
-          # For Intel oneAPI
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    spec: oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "    extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-
-          # Need to find external Intel MPI and annotate with the
-          # correct compiler, no way to do that with spack commands.
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-
-          # Add external Intel MKL and oneAPI runtime
-          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "  intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-
-          export -n SPACK_SYSTEM_CONFIG_PATH
-
-          # For buildcaches
-          spack config add config:install_tree:padded_length:200
-
-          # Set compiler and MPI specs
-          spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]"
-          spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]"
-
-          # Add additional variants for MET packages, different from config/common/packages.yaml
-          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-
-          # Switch providers for blas, lapack, fftw-api
-          spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]"
-          spack config add "packages:all:providers:blas:[intel-oneapi-mkl]"
-          spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]"
-          spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]"
-          spack config add "packages:ectrans:require:'+mkl ~fftw'"
-          spack config add "packages:gsibec:require:'+mkl'"
-          spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']"
-
-          # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above)
-          spack config add "packages:gmake:require:'@:4.2'"
-
-          # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI
-          sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/`
-
-          # Don't generate ecflow module when using external package
-          spack config add "modules:default:tcl:exclude:[ecflow]"
-
-          # Concretize and check for duplicates
-          spack concretize 2>&1 | tee log.concretize.oneapi-2024.2.0-buildcache
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl
-
-          # Add and update source cache
-          spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
-          spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/
-
-          # Add binary cache if requested
-          if [ "$USE_BINARY_CACHE" = true ] ; then
-            set +e
-            spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
-            echo "Packages in spack binary cache:"
-            spack buildcache list
-          fi
-
-          # Break installation up in pieces and create build caches in between
-          # This allows us to "spin up" builds that altogether take longer than
-          # six hours, and/or fail later in the build process.
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
-
-          # jedi-base-env
-          echo "jedi-base-env ..."
-          spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
-
-          # the rest
-          echo "unified-env ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
-
-          # Remove binary cache for next round of concretization
-          if [ "$USE_BINARY_CACHE" = true ] ; then
-            spack mirror rm local-binary
-          fi
-
-          # Remove buildcache config settings
-          spack config remove config:install_tree:padded_length
-
-          # Next steps: synchronize source and build cache to a central/combined mirror?
-
-          # Cleanup
-          spack clean -a
-          spack env deactivate
-
-      - name: create-env
-        run: |
-          source ./setup.sh
-          export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache
-          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
-          export ENVNAME=ue-oneapi-2024.2.0
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
-          spack env activate ${ENVDIR}
-
-          # Concretize
-          spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl
-
-          # Add binary cache back in and reindex it
-          spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-          echo "Packages in combined spack build caches:"
-          spack buildcache list
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env
-
-          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
-          spack clean -a
-          spack module tcl refresh -y
-          spack stack setup-meta-modules
-          spack env deactivate
-
-          ### # Test environment chaining
-          ### echo "Test environment chaining"
-          ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel
-          ### # Retain config from upstream so we don't have to rebuild:
-          ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
-          ### spack env activate ${PWD}/envs/chaintest
-          ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above)
-          ### spack config add "packages:gmake:require:'@:4.2'"
-          ### spack add nccmp@1.9.0.1%oneapi
-          ### spack concretize | tee envs/chaintest/log.concretize
-          ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l)
-          ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
-          ### spack env deactivate
-
-      - name: test-env
-        run: |
-          source /etc/profile.d/modules.sh
-          module use /home/ubuntu/spack-stack/modulefiles
-
-          export ENVNAME=ue-oneapi-2024.2.0
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
-
-          module use ${ENVDIR}/install/modulefiles/Core
-          module load stack-oneapi/2024.2.0
-          module load stack-intel-oneapi-mpi/2021.13
-          module load stack-python/3.11.7
-          module available
-
-          module load jedi-ufs-env
-          module load ewok-env
-          module load soca-env
-          module list

From 3a979065a899bd64e6220d9706af2d6ea9af6800 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 19 Sep 2024 16:34:07 -0600
Subject: [PATCH 04/61] Add .github/actions/Nightly_03_SetupUnifiedEnv

---
 .github/actions/Nightly_01_DirectorySetup/action.yaml |  1 -
 .../Nightly_02_GetSpackStackDevelop/action.yaml       |  1 -
 .../actions/Nightly_03_SetupUnifiedEnv/action.yaml    | 11 +++++++++++
 .github/workflows/nautilus-nightly.yaml               |  4 +++-
 4 files changed, 14 insertions(+), 3 deletions(-)
 create mode 100644 .github/actions/Nightly_03_SetupUnifiedEnv/action.yaml

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
index d1ebb76c3..9773eecc1 100644
--- a/.github/actions/Nightly_01_DirectorySetup/action.yaml
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -1,5 +1,4 @@
 name: Nightly 01 Directory Setup
-#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller
 
 runs:
   using: "composite"
diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
index cb6141781..246b17484 100644
--- a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
+++ b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
@@ -1,5 +1,4 @@
 name: Nightly 02 Get Spack-Stack Develop
-#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller
 
 runs:
   using: "composite"
diff --git a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
new file mode 100644
index 000000000..2b4bb64ae
--- /dev/null
+++ b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 03 Setup Unified Env
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 03 Setup Unified Env
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      cd util/weekly_build
+      ./3_SetupUnifiedEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml
index 07bdcce09..69ad4a85e 100644
--- a/.github/workflows/nautilus-nightly.yaml
+++ b/.github/workflows/nautilus-nightly.yaml
@@ -31,4 +31,6 @@ jobs:
     - name: 01_DirectorySetup
       uses: ./.github/actions/Nightly_01_DirectorySetup
     - name: 02_GetSpackStackDevelop
-      uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
\ No newline at end of file
+      uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
+    - name: 03_SetupUnifiedEnv
+      uses: ./.github/actions/Nightly_02_SetupUnifiedEnv

From 5ab19610595661a6d248f3549ca6e0da61c94478 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 19 Sep 2024 16:39:43 -0600
Subject: [PATCH 05/61] Add .github/actions/Nightly_04_SpackInstall

---
 .github/actions/Nightly_04_SpackInstall/action.yaml | 11 +++++++++++
 .github/workflows/nautilus-nightly.yaml             |  4 +++-
 2 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 .github/actions/Nightly_04_SpackInstall/action.yaml

diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
new file mode 100644
index 000000000..ea90e5755
--- /dev/null
+++ b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 04 Spack Install
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 04 Spack Install
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      cd util/weekly_build
+      ./4_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml
index 69ad4a85e..06ebdb6e3 100644
--- a/.github/workflows/nautilus-nightly.yaml
+++ b/.github/workflows/nautilus-nightly.yaml
@@ -33,4 +33,6 @@ jobs:
     - name: 02_GetSpackStackDevelop
       uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
     - name: 03_SetupUnifiedEnv
-      uses: ./.github/actions/Nightly_02_SetupUnifiedEnv
+      uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
+    - name: 04_SpackInstall
+      uses: ./.github/actions/Nightly_04_SpackInstall

From 3fa47c5795928cbd60b9e4fc9686d4ee922c9755 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 20 Sep 2024 05:52:55 -0600
Subject: [PATCH 06/61] Update .github/* for nautilus nightly oneapi

---
 .../actions/Nightly_01_DirectorySetup/action.yaml  |  2 +-
 .../Nightly_02_GetSpackStackDevelop/action.yaml    |  2 +-
 .../actions/Nightly_03_SetupUnifiedEnv/action.yaml |  2 +-
 .../actions/Nightly_04_SpackInstall/action.yaml    |  2 +-
 ...s-nightly.yaml => nautilus-nightly-oneapi.yaml} |  5 ++++-
 .../{1_DirectorySetup.sh => 01_DirectorySetup.sh}  |  0
 ...kStackDevelop.sh => 02_GetSpackStackDevelop.sh} |  0
 .../{3_SetupUnifiedEnv.sh => 03_SetupEnv.sh}       |  0
 .../{4_SpackInstall.sh => 04_SpackInstall.sh}      |  0
 .../{5_BuildCache.sh => 05_BuildCache.sh}          |  0
 .../weekly_build/{6_AppTests.sh => 06_AppTests.sh} |  0
 util/weekly_build/{7_Cleanup.sh => 07_Cleanup.sh}  |  0
 util/weekly_build/ShellSetup.sh                    |  2 ++
 util/weekly_build/SpackStackBuildCache_AllSteps.sh | 14 +++++++-------
 14 files changed, 17 insertions(+), 12 deletions(-)
 rename .github/workflows/{nautilus-nightly.yaml => nautilus-nightly-oneapi.yaml} (90%)
 rename util/weekly_build/{1_DirectorySetup.sh => 01_DirectorySetup.sh} (100%)
 rename util/weekly_build/{2_GetSpackStackDevelop.sh => 02_GetSpackStackDevelop.sh} (100%)
 rename util/weekly_build/{3_SetupUnifiedEnv.sh => 03_SetupEnv.sh} (100%)
 rename util/weekly_build/{4_SpackInstall.sh => 04_SpackInstall.sh} (100%)
 rename util/weekly_build/{5_BuildCache.sh => 05_BuildCache.sh} (100%)
 rename util/weekly_build/{6_AppTests.sh => 06_AppTests.sh} (100%)
 rename util/weekly_build/{7_Cleanup.sh => 07_Cleanup.sh} (100%)

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
index 9773eecc1..e58d305ca 100644
--- a/.github/actions/Nightly_01_DirectorySetup/action.yaml
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -12,4 +12,4 @@ runs:
       rm -vfr ${RUNDIR}
       # *DH
       cd util/weekly_build
-      ./1_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
+      ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
index 246b17484..7565046dd 100644
--- a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
+++ b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
@@ -8,4 +8,4 @@ runs:
     run: |
       RUNID=`date +"%Y%m%d"`
       cd util/weekly_build
-      ./2_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM}
+      ./02_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
index 2b4bb64ae..0f598cb88 100644
--- a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
+++ b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
@@ -8,4 +8,4 @@ runs:
     run: |
       RUNID=`date +"%Y%m%d"`
       cd util/weekly_build
-      ./3_SetupUnifiedEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM}
+      ./03_SetupEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
index ea90e5755..0954971b1 100644
--- a/.github/actions/Nightly_04_SpackInstall/action.yaml
+++ b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -8,4 +8,4 @@ runs:
     run: |
       RUNID=`date +"%Y%m%d"`
       cd util/weekly_build
-      ./4_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
+      ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
similarity index 90%
rename from .github/workflows/nautilus-nightly.yaml
rename to .github/workflows/nautilus-nightly-oneapi.yaml
index 06ebdb6e3..5c8b33f9f 100644
--- a/.github/workflows/nautilus-nightly.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -18,11 +18,14 @@ defaults:
 env:
   BASEDIR: /p/work2/heinzell/spack-stack-nightly
   PLATFORM: nautilus
+  COMPILER: oneapi
 
 jobs:
 
-  nightly-build:
+  nightly-build-oneapi:
     runs-on: ['self-hosted','nautilus-heinzell']
+    # Set timeout to 12hrs
+    #timeout-minutes: 720
     steps:
     - uses: actions/checkout@v4
       with:
diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/01_DirectorySetup.sh
similarity index 100%
rename from util/weekly_build/1_DirectorySetup.sh
rename to util/weekly_build/01_DirectorySetup.sh
diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/02_GetSpackStackDevelop.sh
similarity index 100%
rename from util/weekly_build/2_GetSpackStackDevelop.sh
rename to util/weekly_build/02_GetSpackStackDevelop.sh
diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupEnv.sh
similarity index 100%
rename from util/weekly_build/3_SetupUnifiedEnv.sh
rename to util/weekly_build/03_SetupEnv.sh
diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
similarity index 100%
rename from util/weekly_build/4_SpackInstall.sh
rename to util/weekly_build/04_SpackInstall.sh
diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/05_BuildCache.sh
similarity index 100%
rename from util/weekly_build/5_BuildCache.sh
rename to util/weekly_build/05_BuildCache.sh
diff --git a/util/weekly_build/6_AppTests.sh b/util/weekly_build/06_AppTests.sh
similarity index 100%
rename from util/weekly_build/6_AppTests.sh
rename to util/weekly_build/06_AppTests.sh
diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/07_Cleanup.sh
similarity index 100%
rename from util/weekly_build/7_Cleanup.sh
rename to util/weekly_build/07_Cleanup.sh
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 46ad0ef5a..24767ea78 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -86,6 +86,8 @@ case $PLATFORM in
     BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache}
     ;;
   nautilus)
+    module purge
+    umask 0022
     COMPILERS=${COMPILERS:-"intel oneapi gcc"}
     BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build-cache}
     ;;
diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
index 8f6e2cb0a..f9b85eae6 100755
--- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh
+++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
@@ -16,10 +16,10 @@ function trap_and_run {
   $*
 }
 
-trap_and_run ./1_DirectorySetup.sh $*
-trap_and_run ./2_GetSpackStackDevelop.sh $*
-trap_and_run ./3_SetupUnifiedEnv.sh $*
-trap_and_run ./4_SpackInstall.sh $*
-trap_and_run ./5_BuildCache.sh $*
-trap_and_run ./6_AppTests.sh $*
-trap_and_run ./7_Cleanup.sh $*
+trap_and_run ./01_DirectorySetup.sh $*
+trap_and_run ./02_GetSpackStackDevelop.sh $*
+trap_and_run ./03_SetupEnv.sh $*
+trap_and_run ./04_SpackInstall.sh $*
+trap_and_run ./05_BuildCache.sh $*
+trap_and_run ./06_AppTests.sh $*
+trap_and_run ./07_Cleanup.sh $*

From 6c61957ea37475ac8d75e6d18e6ed28893e977aa Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 24 Sep 2024 07:45:02 -0600
Subject: [PATCH 07/61] Increase timeout of
 .github/workflows/nautilus-nightly-oneapi.yaml to 720 minutes

---
 .github/workflows/nautilus-nightly-oneapi.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
index 5c8b33f9f..a7062f6f1 100644
--- a/.github/workflows/nautilus-nightly-oneapi.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -25,7 +25,7 @@ jobs:
   nightly-build-oneapi:
     runs-on: ['self-hosted','nautilus-heinzell']
     # Set timeout to 12hrs
-    #timeout-minutes: 720
+    timeout-minutes: 720
     steps:
     - uses: actions/checkout@v4
       with:

From 71e5a2c65c12c47a65c0e3123e6cfe28ad23a32d Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 24 Sep 2024 09:05:10 -0600
Subject: [PATCH 08/61] Add .github/workflows/nautilus-nightly-intel.yaml and
 build only base-env in unified-dev for testing

---
 .github/workflows/nautilus-nightly-intel.yaml | 41 +++++++++++++
 .../workflows/nautilus-nightly-oneapi.yaml    |  8 +--
 configs/templates/unified-dev/spack.yaml      | 57 ++++++++++---------
 util/weekly_build/03_SetupEnv.sh              |  2 +
 4 files changed, 76 insertions(+), 32 deletions(-)
 create mode 100644 .github/workflows/nautilus-nightly-intel.yaml

diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml
new file mode 100644
index 000000000..e9a55b191
--- /dev/null
+++ b/.github/workflows/nautilus-nightly-intel.yaml
@@ -0,0 +1,41 @@
+name: nautilus unified-env intel
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - develop
+    types: [opened, synchronize, reopened]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BASEDIR: /p/work2/heinzell/spack-stack-nightly
+  PLATFORM: nautilus
+  COMPILERS: intel
+
+jobs:
+
+  nightly-build-intel:
+    runs-on: ['self-hosted','nautilus-heinzell']
+    # Set timeout to 12hrs
+    timeout-minutes: 720
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        clean: true
+    - name: 01_DirectorySetup
+      uses: ./.github/actions/Nightly_01_DirectorySetup
+    - name: 02_GetSpackStackDevelop
+      uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
+    - name: 03_SetupUnifiedEnv
+      uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
+    #- name: 04_SpackInstall
+    #  uses: ./.github/actions/Nightly_04_SpackInstall
diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
index a7062f6f1..671c7555d 100644
--- a/.github/workflows/nautilus-nightly-oneapi.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -1,4 +1,4 @@
-name: nautilus heinzell auto ci
+name: nautilus unified-env oneapi
 
 on:
   workflow_dispatch:
@@ -18,7 +18,7 @@ defaults:
 env:
   BASEDIR: /p/work2/heinzell/spack-stack-nightly
   PLATFORM: nautilus
-  COMPILER: oneapi
+  COMPILERS: oneapi
 
 jobs:
 
@@ -37,5 +37,5 @@ jobs:
       uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
     - name: 03_SetupUnifiedEnv
       uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
-    - name: 04_SpackInstall
-      uses: ./.github/actions/Nightly_04_SpackInstall
+    #- name: 04_SpackInstall
+    #  uses: ./.github/actions/Nightly_04_SpackInstall
diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml
index a05ed4a75..6420943f4 100644
--- a/configs/templates/unified-dev/spack.yaml
+++ b/configs/templates/unified-dev/spack.yaml
@@ -8,34 +8,35 @@ spack:
   definitions:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
   - packages:
-    - ewok-env +ecflow ~cylc
-    - ai-env
-    - geos-gcm-env          ^esmf@=8.6.1
-    - global-workflow-env   ^esmf@=8.6.1
-    - gmao-swell-env
-    - gsi-env               ^esmf@=8.6.1
-    - jedi-fv3-env
-    - jedi-geos-env         ^esmf@=8.6.1
-    - jedi-mpas-env
-    - jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
-    - jedi-tools-env
-    - jedi-ufs-env          ^esmf@=8.6.1
-    - jedi-um-env
-    - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11
-    - soca-env
-    - ufs-srw-app-env       ^esmf@=8.6.1
-    - ufs-weather-model-env ^esmf@=8.6.1
-
-    # Various crtm tags (list all to avoid duplicate packages)
-    - crtm@2.4.0.1
-    - crtm@v2.4.1-jedi
-
-    # Various esmf tags (list all to avoid duplicate packages)
-    - esmf@=8.6.1 snapshot=none
-    - esmf@=8.7.0b11 snapshot=b11
-
-    # MADIS for WCOSS2 decoders.
-    - madis@4.5
+    - base-env
+    #- ewok-env +ecflow ~cylc
+    #- ai-env
+    #- geos-gcm-env          ^esmf@=8.6.1
+    #- global-workflow-env   ^esmf@=8.6.1
+    #- gmao-swell-env
+    #- gsi-env               ^esmf@=8.6.1
+    #- jedi-fv3-env
+    #- jedi-geos-env         ^esmf@=8.6.1
+    #- jedi-mpas-env
+    #- jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
+    #- jedi-tools-env
+    #- jedi-ufs-env          ^esmf@=8.6.1
+    #- jedi-um-env
+    #- neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11
+    #- soca-env
+    #- ufs-srw-app-env       ^esmf@=8.6.1
+    #- ufs-weather-model-env ^esmf@=8.6.1
+    #
+    ## Various crtm tags (list all to avoid duplicate packages)
+    #- crtm@2.4.0.1
+    #- crtm@v2.4.1-jedi
+    #
+    ## Various esmf tags (list all to avoid duplicate packages)
+    #- esmf@=8.6.1 snapshot=none
+    #- esmf@=8.7.0b11 snapshot=b11
+    #
+    ## MADIS for WCOSS2 decoders.
+    #- madis@4.5
 
   specs:
   - matrix:
diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 3479ed7ec..303240dc4 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -11,6 +11,8 @@ set +x
 set -x
 
 for compiler in $COMPILERS; do
+  echo "Setting up environment build-${compiler/@/-} using template unified-dev"
+  #echo "Setting up environment build-${compiler/@/-} using template unified-dev"
   rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-}
   spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler
   cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}

From 18c71335ceb135f9bf155587134ac746e440ed25 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 24 Sep 2024 11:20:14 -0600
Subject: [PATCH 09/61] Enable step 4 of nightly tests on Nautilus with Intel
 and oneAPI

---
 .github/workflows/nautilus-nightly-intel.yaml  | 4 ++--
 .github/workflows/nautilus-nightly-oneapi.yaml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml
index e9a55b191..7dbf11bec 100644
--- a/.github/workflows/nautilus-nightly-intel.yaml
+++ b/.github/workflows/nautilus-nightly-intel.yaml
@@ -37,5 +37,5 @@ jobs:
       uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
     - name: 03_SetupUnifiedEnv
       uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
-    #- name: 04_SpackInstall
-    #  uses: ./.github/actions/Nightly_04_SpackInstall
+    - name: 04_SpackInstall
+      uses: ./.github/actions/Nightly_04_SpackInstall
diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
index 671c7555d..df7b708a5 100644
--- a/.github/workflows/nautilus-nightly-oneapi.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -37,5 +37,5 @@ jobs:
       uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
     - name: 03_SetupUnifiedEnv
       uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
-    #- name: 04_SpackInstall
-    #  uses: ./.github/actions/Nightly_04_SpackInstall
+    - name: 04_SpackInstall
+      uses: ./.github/actions/Nightly_04_SpackInstall

From f5a575b7d61eb1485b6c9798edbdc3f21e1abfb2 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 24 Sep 2024 16:06:37 -0600
Subject: [PATCH 10/61] Remove piping to log files in
 util/weekly_build/03_SetupEnv.sh and util/weekly_build/04_SpackInstall.sh

---
 util/weekly_build/03_SetupEnv.sh     | 3 ++-
 util/weekly_build/04_SpackInstall.sh | 8 ++++----
 util/weekly_build/ShellSetup.sh      | 3 ++-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 303240dc4..ccaf562fe 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -18,9 +18,10 @@ for compiler in $COMPILERS; do
   cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
   spack env activate .
   spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
-  # Check for duplicates and fail before doing the "real" concretization with test deps:
+  # Check for duplicates and fail before doing the "real" concretization:
   spack concretize --fresh 2>&1 | tee log.concretize
   ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
+  spack concretize --fresh --force
 # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
 #  spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
 done
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index 7c98a2c6b..34711fcc8 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -16,12 +16,12 @@ for compiler in $COMPILERS; do
   cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
   spack env activate .
   if [ -z "$PACKAGES_TO_INSTALL" ]; then
-    spack fetch --missing 2>&1 | tee log.fetch
+    spack fetch --missing # 2>&1 | tee log.fetch
   else
-    spack fetch --missing --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch
+    spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch
   fi
   # Just install the packages we're testing (+dependencies):
-  spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.test.install
+  spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # 2>&1 | tee log.test.install
   # Install the rest of the stack as usual:
-  spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL 2>&1 | tee log.install
+  spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL # 2>&1 | tee log.install
 done
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 24767ea78..61243d933 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -18,7 +18,8 @@ function alert_cmd {
 }
 
 function spack_install_exe {
-  spack $* | tee -a log.install 2>&1
+  #spack $* | tee -a log.install 2>&1
+  spack $*
 }
 
 case $PLATFORM in

From 979607d0cdd6f7edb2fa113d090cb03280f23070 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 24 Sep 2024 21:07:29 -0600
Subject: [PATCH 11/61] Comment out default definition of PACKAGES_TO_TEST

---
 util/weekly_build/ShellSetup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 61243d933..c12f4cee7 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -11,7 +11,7 @@ if [ ${RUNDIR::1} != "/" ]; then
   exit 1
 fi
 
-PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"}
+#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"}
 
 function alert_cmd {
   echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system."

From 4709452ccc91ad43d29ab6bb8da91b5f6cf9bde7 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 25 Sep 2024 15:36:41 -0600
Subject: [PATCH 12/61] Try spack fetch without --missing

---
 util/weekly_build/04_SpackInstall.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index 34711fcc8..ac396de1d 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -15,11 +15,12 @@ INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS"
 for compiler in $COMPILERS; do
   cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
   spack env activate .
-  if [ -z "$PACKAGES_TO_INSTALL" ]; then
-    spack fetch --missing # 2>&1 | tee log.fetch
-  else
-    spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch
-  fi
+  spack fetch # 2>&1 | tee log.fetch
+  #if [ -z "$PACKAGES_TO_INSTALL" ]; then
+  #  spack fetch --missing # 2>&1 | tee log.fetch
+  #else
+  #  spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch
+  #fi
   # Just install the packages we're testing (+dependencies):
   spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # 2>&1 | tee log.test.install
   # Install the rest of the stack as usual:

From b4bcf8fa70f4e7a750481b003f248751a9bb6a08 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 26 Sep 2024 16:00:20 -0600
Subject: [PATCH 13/61] Add logic to extract path to local source mirror and
 comment out 'spack fetch'

---
 util/weekly_build/03_SetupEnv.sh     | 5 +++++
 util/weekly_build/04_SpackInstall.sh | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index ccaf562fe..a6d360ed5 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -24,4 +24,9 @@ for compiler in $COMPILERS; do
   spack concretize --fresh --force
 # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
 #  spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
+  # Get path to local-source mirror from spack site config
+  SOURCE_MIRROR=`spack mirror list | grep local-source`
+  SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//}
+  # Download all source codes needed to build the environment
+  spack mirror create -a -d ${SOURCE_MIRROR}
 done
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index ac396de1d..f6edac28e 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -15,7 +15,7 @@ INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS"
 for compiler in $COMPILERS; do
   cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
   spack env activate .
-  spack fetch # 2>&1 | tee log.fetch
+  #spack fetch # 2>&1 | tee log.fetch
   #if [ -z "$PACKAGES_TO_INSTALL" ]; then
   #  spack fetch --missing # 2>&1 | tee log.fetch
   #else

From ff17223cee098ff9c87d09ca2a6321b46af03e7f Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 27 Sep 2024 10:27:56 -0600
Subject: [PATCH 14/61] Configure oneapi@2024.2.1 for Narwhal

---
 configs/sites/tier1/narwhal/compilers.yaml    | 32 +++++++++++++++++++
 .../sites/tier1/narwhal/packages_oneapi.yaml  | 32 +++++++++++++++++++
 doc/source/PreConfiguredSites.rst             | 30 ++++++++++++++++-
 3 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 configs/sites/tier1/narwhal/packages_oneapi.yaml

diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml
index 1b7c23dcf..5e2e17c11 100644
--- a/configs/sites/tier1/narwhal/compilers.yaml
+++ b/configs/sites/tier1/narwhal/compilers.yaml
@@ -21,6 +21,38 @@ compilers::
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
+  - compiler:
+      spec: oneapi@2024.2.1
+      paths:
+        #cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx
+        #cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx
+        #f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort
+        #fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      modules:
+      - PrgEnv-intel/8.3.3
+      - tbb/2021.13
+      - compiler-rt/2024.2.1
+      - compiler/2024.2.1
+      - ifort/2024.2.1
+      - cray-libsci/23.05.1.4
+      - libfabric/1.12.1.2.2.1
+      environment:
+        prepend_path:
+          PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
+          CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
+          MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles'
+        append_path:
+          CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64'
+        set:
+          CRAYPE_LINK_TYPE: 'dynamic'
+      extra_rpaths: []
   - compiler:
       spec: gcc@10.3.0
       paths:
diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml
new file mode 100644
index 000000000..9b032dcf1
--- /dev/null
+++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml
@@ -0,0 +1,32 @@
+packages:
+  all:
+    compiler:: [oneapi@2024.2.1,gcc@10.3.0]
+    providers:
+      mpi:: [cray-mpich@8.1.21]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers
+      modules:
+      - cray-mpich-ucx/8.1.21
+      - craype-network-ucx
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
+      modules:
+      - mkl/2024.2
+  intel-oneapi-tbb:
+    externals:
+    - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
+      modules:
+      - tbb/2021.13
+  intel-oneapi-runtime:
+    externals:
+    - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
+      modules:
+      - tbb/2021.13
+      - compiler-rt/2024.2.1
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index 97b532daa..0ea8e3a7b 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -63,7 +63,7 @@ Pre-configured sites (tier 1)
 +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+
 |                     | Narwhal               | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/``               | NRL             |
 | U.S. Navy (HPCMP)   +-----------------------+--------------------+--------------------------------------------------------+-----------------+
-|                     | Nautilus              | Intel              | ``/p/app/projects/NEPTUNE/spack-stack/``               | NRL             |
+|                     | Nautilus              | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/``               | NRL             |
 +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+
 | Univ. of Wisconsin  | S4                    | Intel              | ``/data/prod/jedi/spack-stack/``                       | JCSDA           |
 +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+
@@ -169,6 +169,34 @@ The following is required for building new spack environments with Intel on this
    module unload cray-libsci
    module load cray-libsci/23.05.1.4
 
+The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal!
+
+.. code-block:: console
+
+   umask 0022
+   module unload PrgEnv-cray
+   module load PrgEnv-intel/8.3.3
+   module unload intel
+   module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles
+   module load tbb/2021.13
+   module load compiler-rt/2024.2.1
+   module load compiler/2024.2.1
+   module load ifort/2024.2.1
+   module unload cray-mpich
+   module unload craype-network-ofi
+   # Warning. Do not load craype-network-ucx
+   # or cray-mpich-ucx/8.1.21!
+   # There is a bug in the modulefile that prevents
+   # spack from setting the environment for its
+   # build steps when the module is already
+   # loaded. Instead, let spack load it when the
+   # package requires it.
+   #module load craype-network-ucx
+   #module load cray-mpich-ucx/8.1.21
+   module load libfabric/1.12.1.2.2.1
+   module unload cray-libsci
+   module load cray-libsci/23.05.1.4
+
 The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal!
 
 .. code-block:: console

From e0576ca5139c54de2ef65d52a99938b1ebffb9eb Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 30 Sep 2024 16:07:52 -0600
Subject: [PATCH 15/61] Update doc/source/PreConfiguredSites.rst: add note that
 oneAPI Narwhal setup currently does not compile Python packages

---
 doc/source/PreConfiguredSites.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index 0ea8e3a7b..09ddec8d1 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -171,6 +171,8 @@ The following is required for building new spack environments with Intel on this
 
 The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal!
 
+.. note:: This particular setup of the oneAPI compilers is outside the Cray-blessed environment and there are problems installing Python packages with spack.
+
 .. code-block:: console
 
    umask 0022

From 76361b06745500f2b1521589ec79ff291a7a5964 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 3 Oct 2024 15:46:18 -0600
Subject: [PATCH 16/61] Add configs/sites/tier1/atlantis/mirrors.yaml

---
 configs/sites/tier1/atlantis/mirrors.yaml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 configs/sites/tier1/atlantis/mirrors.yaml

diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml
new file mode 100644
index 000000000..94e16cfaf
--- /dev/null
+++ b/configs/sites/tier1/atlantis/mirrors.yaml
@@ -0,0 +1,20 @@
+mirrors:
+  local-source:
+    fetch:
+      # TODO UPDATE?
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null
+    push:
+      # TODO UPDATE?
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
+      access_pair:
+      - null
+      - null
+      access_token: null
+      profile: null
+      endpoint_url: null

From 87067e943e5b51167ae139845f171b005bd31577 Mon Sep 17 00:00:00 2001
From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com>
Date: Thu, 3 Oct 2024 22:45:20 -0400
Subject: [PATCH 17/61] Update site/packages.yaml and site/modules.yaml (#1322)

Latest changes in site/packages.yaml and site/modules.yaml config files regarding ecFlow and QT. Removed exclude for creating ecflow modulefile, added path for qt and removed ecflow from package list.
---
 configs/sites/tier1/derecho/modules.yaml      |  2 --
 configs/sites/tier1/gaea-c5/modules.yaml      |  2 --
 configs/sites/tier1/gaea-c5/packages.yaml     |  8 +-------
 configs/sites/tier1/gaea-c6/packages.yaml     |  7 +------
 configs/sites/tier1/hera/modules.yaml         |  2 --
 configs/sites/tier1/hera/packages.yaml        | 11 ++++-------
 configs/sites/tier1/hercules/modules.yaml     |  2 --
 configs/sites/tier1/hercules/packages.yaml    | 13 +++----------
 configs/sites/tier1/jet/modules.yaml          |  2 --
 configs/sites/tier1/jet/packages.yaml         | 12 +++++-------
 configs/sites/tier1/noaa-aws/modules.yaml     |  2 --
 configs/sites/tier1/noaa-aws/packages.yaml    | 12 +++++-------
 configs/sites/tier1/noaa-azure/modules.yaml   |  2 --
 configs/sites/tier1/noaa-azure/packages.yaml  | 12 +++++-------
 configs/sites/tier1/noaa-gcloud/modules.yaml  |  2 --
 configs/sites/tier1/noaa-gcloud/packages.yaml | 12 +++++-------
 configs/sites/tier1/orion/modules.yaml        |  2 --
 configs/sites/tier1/orion/packages.yaml       |  7 -------
 18 files changed, 29 insertions(+), 83 deletions(-)

diff --git a/configs/sites/tier1/derecho/modules.yaml b/configs/sites/tier1/derecho/modules.yaml
index b9d5b4703..960198ae7 100644
--- a/configs/sites/tier1/derecho/modules.yaml
+++ b/configs/sites/tier1/derecho/modules.yaml
@@ -3,8 +3,6 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - cray-mpich
       - python
diff --git a/configs/sites/tier1/gaea-c5/modules.yaml b/configs/sites/tier1/gaea-c5/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/gaea-c5/modules.yaml
+++ b/configs/sites/tier1/gaea-c5/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml
index 6e13f710e..e9f4fc586 100644
--- a/configs/sites/tier1/gaea-c5/packages.yaml
+++ b/configs/sites/tier1/gaea-c5/packages.yaml
@@ -81,12 +81,6 @@ packages:
     externals:
     - spec: dos2unix@7.4.0
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4
-      modules: [ecflow/5.8.4]
   file:
     externals:
     - spec: file@5.32
@@ -197,7 +191,7 @@ packages:
   qt:
     externals:
     - spec: qt@5.15.2
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2/5.15.2/gcc_64
+      prefix: /ncrc/proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64
   rdma-core:
     externals:
     - spec: rdma-core@37.0
diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml
index 75473846c..7e9663d31 100644
--- a/configs/sites/tier1/gaea-c6/packages.yaml
+++ b/configs/sites/tier1/gaea-c6/packages.yaml
@@ -70,11 +70,6 @@ packages:
     externals:
     - spec: dos2unix@7.4.0
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /autofs/ncrc-svm1_proj/epic/spack-stack/ecflow-5.8.4
   file:
     externals:
     - spec: file@5.32
@@ -184,7 +179,7 @@ packages:
   qt:
     externals:
     - spec: qt@5.15.2
-      prefix: /autofs/ncrc-svm1_proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64
+      prefix: /ncrc/proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64
   rdma-core:
     externals:
     - spec: rdma-core@42.0
diff --git a/configs/sites/tier1/hera/modules.yaml b/configs/sites/tier1/hera/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/hera/modules.yaml
+++ b/configs/sites/tier1/hera/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/hera/packages.yaml b/configs/sites/tier1/hera/packages.yaml
index 894af0f5e..36a0a3f68 100644
--- a/configs/sites/tier1/hera/packages.yaml
+++ b/configs/sites/tier1/hera/packages.yaml
@@ -34,13 +34,6 @@ packages:
     externals:
     - spec: doxygen@1.8.5+graphviz~mscgen
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.5.3+ui+static_boost
-      prefix: /apps/ecflow/5.5.3
-      modules:
-      - ecflow/5.5.3
   file:
     externals:
     - spec: file@5.11
@@ -120,6 +113,10 @@ packages:
     externals:
     - spec: pkg-config@0.27.1
       prefix: /usr
+  qt:
+    externals:
+    - spec: qt@5.12.6
+      prefix: /scratch1/NCEPDEV/nems/role.epic/installs/qt-5.12.6/qtbase
   rsync:
     externals:
     - spec: rsync@3.1.2
diff --git a/configs/sites/tier1/hercules/modules.yaml b/configs/sites/tier1/hercules/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/hercules/modules.yaml
+++ b/configs/sites/tier1/hercules/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml
index 9c2c95209..0128505f1 100644
--- a/configs/sites/tier1/hercules/packages.yaml
+++ b/configs/sites/tier1/hercules/packages.yaml
@@ -19,13 +19,6 @@ packages:
     externals:
     - spec: diffutils@3.7
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4
-      modules:
-      - ecflow/5.8.4
   findutils:
     externals:
     - spec: findutils@4.8.0
@@ -73,10 +66,10 @@ packages:
       prefix: /usr
   qt:
     externals:
-    - spec: qt@5.15.8
-      prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd
+    - spec: qt@5.15.14
+      prefix: /apps/contrib/spack-stack-1.1/gcc-11.3.1/qt-5.15.14-mfeuvcidmyqoi2m5i2tfrk6yd7xtk6pt
       modules:
-      - qt/5.15.8
+      - qt/5.15.14
   sed:
     externals:
     - spec: sed@4.8
diff --git a/configs/sites/tier1/jet/modules.yaml b/configs/sites/tier1/jet/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/jet/modules.yaml
+++ b/configs/sites/tier1/jet/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/jet/packages.yaml b/configs/sites/tier1/jet/packages.yaml
index 04633817d..fa6a0d123 100644
--- a/configs/sites/tier1/jet/packages.yaml
+++ b/configs/sites/tier1/jet/packages.yaml
@@ -34,13 +34,6 @@ packages:
     externals:
     - spec: doxygen@1.8.5+graphviz~mscgen
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.5.3+ui+static_boost
-      prefix: /apps/ecflow/5.5.3
-      modules:
-      - ecflow/5.5.3
   file:
     externals:
     - spec: file@5.11
@@ -124,6 +117,11 @@ packages:
     externals:
     - spec: pkg-config@0.27.1
       prefix: /usr
+  qt:
+    buildable: false
+    externals:
+    - spec: qt@5.12.6
+      prefix: /contrib/spack-stack/installs/qt-5.12.6/qtbase
   rsync:
     externals:
     - spec: rsync@3.1.2
diff --git a/configs/sites/tier1/noaa-aws/modules.yaml b/configs/sites/tier1/noaa-aws/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/noaa-aws/modules.yaml
+++ b/configs/sites/tier1/noaa-aws/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/noaa-aws/packages.yaml b/configs/sites/tier1/noaa-aws/packages.yaml
index ee153e8a5..dda1f369b 100644
--- a/configs/sites/tier1/noaa-aws/packages.yaml
+++ b/configs/sites/tier1/noaa-aws/packages.yaml
@@ -15,13 +15,6 @@ packages:
     externals:
     - spec: diffutils@3.6
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4
-      modules:
-      - ecflow/5.8.4
   file:
     externals:
     - spec: file@5.33
@@ -82,6 +75,11 @@ packages:
     externals:
     - spec: pkg-config@1.4.2
       prefix: /usr
+  qt:
+    buildable: false
+    externals:
+    - spec: qt@5.15.3
+      prefix: /usr/lib64/qt5
   rsync:
     externals:
     - spec: rsync@3.1.3
diff --git a/configs/sites/tier1/noaa-azure/modules.yaml b/configs/sites/tier1/noaa-azure/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/noaa-azure/modules.yaml
+++ b/configs/sites/tier1/noaa-azure/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/noaa-azure/packages.yaml b/configs/sites/tier1/noaa-azure/packages.yaml
index 004aea213..397bc780b 100644
--- a/configs/sites/tier1/noaa-azure/packages.yaml
+++ b/configs/sites/tier1/noaa-azure/packages.yaml
@@ -15,13 +15,6 @@ packages:
     externals:
     - spec: diffutils@3.6
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4
-      modules:
-      - ecflow/5.8.4
   file:
     externals:
     - spec: file@5.33
@@ -78,6 +71,11 @@ packages:
     externals:
     - spec: pkg-config@1.4.2
       prefix: /usr
+  qt:
+    buildable: false
+    externals:
+    - spec: qt@5.15.3
+      prefix: /usr/lib64/qt5
   rsync:
     externals:
     - spec: rsync@3.1.3
diff --git a/configs/sites/tier1/noaa-gcloud/modules.yaml b/configs/sites/tier1/noaa-gcloud/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/noaa-gcloud/modules.yaml
+++ b/configs/sites/tier1/noaa-gcloud/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/noaa-gcloud/packages.yaml b/configs/sites/tier1/noaa-gcloud/packages.yaml
index 004aea213..397bc780b 100644
--- a/configs/sites/tier1/noaa-gcloud/packages.yaml
+++ b/configs/sites/tier1/noaa-gcloud/packages.yaml
@@ -15,13 +15,6 @@ packages:
     externals:
     - spec: diffutils@3.6
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4
-      modules:
-      - ecflow/5.8.4
   file:
     externals:
     - spec: file@5.33
@@ -78,6 +71,11 @@ packages:
     externals:
     - spec: pkg-config@1.4.2
       prefix: /usr
+  qt:
+    buildable: false
+    externals:
+    - spec: qt@5.15.3
+      prefix: /usr/lib64/qt5
   rsync:
     externals:
     - spec: rsync@3.1.3
diff --git a/configs/sites/tier1/orion/modules.yaml b/configs/sites/tier1/orion/modules.yaml
index 4c530d42d..65801706f 100644
--- a/configs/sites/tier1/orion/modules.yaml
+++ b/configs/sites/tier1/orion/modules.yaml
@@ -3,7 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      exclude:
-      - ecflow
       include:
       - python
diff --git a/configs/sites/tier1/orion/packages.yaml b/configs/sites/tier1/orion/packages.yaml
index 3fcecb71c..bef7849a9 100644
--- a/configs/sites/tier1/orion/packages.yaml
+++ b/configs/sites/tier1/orion/packages.yaml
@@ -31,13 +31,6 @@ packages:
     externals:
     - spec: diffutils@3.7
       prefix: /usr
-  ecflow::
-    buildable: False
-    externals:
-    - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /work/noaa/epic/role-epic/spack-stack/orion/ecflow-5.8.4
-      modules:
-      - ecflow/5.8.4
   findutils:
     externals:
     - spec: findutils@4.8.0

From 91fe139a3d8fce9393f84d9435a0732266ae1ed2 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 4 Oct 2024 05:29:09 -0600
Subject: [PATCH 18/61] Add configs/sites/tier1/atlantis/packages_gcc.yaml

---
 .../sites/tier1/atlantis/packages_gcc.yaml    | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 configs/sites/tier1/atlantis/packages_gcc.yaml

diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml
new file mode 100644
index 000000000..7d36ed0b4
--- /dev/null
+++ b/configs/sites/tier1/atlantis/packages_gcc.yaml
@@ -0,0 +1,23 @@
+packages:
+  all:
+    compiler:: [gcc@11.2.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+      #blas:: [intel-oneapi-mkl]
+      #fftw-api:: [intel-oneapi-mkl]
+      #lapack:: [intel-oneapi-mkl]
+  slurm:
+    buildable: False
+    externals:
+    - spec: slurm@21.08.8 +pmix
+      modules:
+      - slurm/slurm/21.08.8
+  openmpi:
+    require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers'
+modules:
+  default:
+    lmod:
+      include:
+      - openmpi
+      exclude:
+      - slurm

From 5469c560ce72ac2d27317389a00a04b3d53a9c1a Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 4 Oct 2024 05:37:30 -0600
Subject: [PATCH 19/61] Remove site sandy

---
 configs/sites/tier1/sandy/compilers.yaml |  14 ---
 configs/sites/tier1/sandy/config.yaml    |   2 -
 configs/sites/tier1/sandy/mirrors.yaml   |  18 ----
 configs/sites/tier1/sandy/modules.yaml   |   8 --
 configs/sites/tier1/sandy/packages.yaml  | 113 -----------------------
 5 files changed, 155 deletions(-)
 delete mode 100644 configs/sites/tier1/sandy/compilers.yaml
 delete mode 100644 configs/sites/tier1/sandy/config.yaml
 delete mode 100644 configs/sites/tier1/sandy/mirrors.yaml
 delete mode 100644 configs/sites/tier1/sandy/modules.yaml
 delete mode 100644 configs/sites/tier1/sandy/packages.yaml

diff --git a/configs/sites/tier1/sandy/compilers.yaml b/configs/sites/tier1/sandy/compilers.yaml
deleted file mode 100644
index b86ff0a33..000000000
--- a/configs/sites/tier1/sandy/compilers.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-compilers:
-- compiler:
-    spec: gcc@=9.3.0
-    paths:
-      cc: /software/depot/gcc-9.3.0/bin/gcc
-      cxx: /software/depot/gcc-9.3.0/bin/g++
-      f77: /software/depot/gcc-9.3.0/bin/gfortran
-      fc: /software/depot/gcc-9.3.0/bin/gfortran
-    flags: {}
-    operating_system: centos7
-    target: x86_64
-    modules: []
-    environment: {}
-    extra_rpaths: []
diff --git a/configs/sites/tier1/sandy/config.yaml b/configs/sites/tier1/sandy/config.yaml
deleted file mode 100644
index 84fe05ab9..000000000
--- a/configs/sites/tier1/sandy/config.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-config:
-  build_jobs: 8
diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/sandy/mirrors.yaml
deleted file mode 100644
index c48f33bf1..000000000
--- a/configs/sites/tier1/sandy/mirrors.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-mirrors:
-  local-source:
-    fetch:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml
deleted file mode 100644
index aeb254c2e..000000000
--- a/configs/sites/tier1/sandy/modules.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-modules:
-  default:
-    enable::
-    - tcl
-    tcl:
-      include:
-      # List of packages for which we need modules that are blacklisted by default
-      - python
diff --git a/configs/sites/tier1/sandy/packages.yaml b/configs/sites/tier1/sandy/packages.yaml
deleted file mode 100644
index 1af1ab029..000000000
--- a/configs/sites/tier1/sandy/packages.yaml
+++ /dev/null
@@ -1,113 +0,0 @@
-packages:
-  all:
-    compiler:: [gcc@9.3.0]
-    providers: 
-      mpi:: [openmpi@4.0.5]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  openmpi:
-    externals:
-    - spec: openmpi@4.0.5%gcc@9.3.0
-      prefix: /software7/depot/openmpi-4.0.5
-
-### All other external packages listed alphabetically
-  autoconf:
-    externals:
-    - spec: autoconf@2.69
-      prefix: /usr
-  automake:
-    externals:
-    - spec: automake@1.16.1
-      prefix: /usr
-  binutils:
-    externals:
-    - spec: binutils@2.30.117
-      prefix: /usr
-  bison:
-    externals:
-    - spec: bison@3.0.4
-      prefix: /usr
-  coreutils:
-    externals:
-    - spec: coreutils@8.30
-      prefix: /usr
-  curl:
-    externals:
-    - spec: curl@7.61.1
-      prefix: /usr
-  diffutils:
-    externals:
-    - spec: diffutils@3.6
-      prefix: /usr
-  findutils:
-    externals:
-    - spec: findutils@4.6.0
-      prefix: /usr
-  flex:
-    externals:
-    - spec: flex@2.6.1+lex
-      prefix: /usr
-  gawk:
-    externals:
-    - spec: gawk@4.2.1
-      prefix: /usr
-  git:
-    externals:
-    - spec: git@2.39.2+tcltk
-      prefix: /software8/depot/git-2.39.2
-  git-lfs:
-    externals:
-    - spec: git-lfs@3.3.0
-      prefix: /software8/depot/git-2.39.2
-  gmake:
-    externals:
-    - spec: gmake@4.2.1
-      prefix: /usr
-  groff:
-    externals:
-    - spec: groff@1.22.3
-      prefix: /usr
-  libtool:
-    externals:
-    - spec: libtool@2.4.6
-      prefix: /usr
-  m4:
-    externals:
-    - spec: m4@1.4.18
-      prefix: /usr
-  openssh:
-    externals:
-    - spec: openssh@8.0p1
-      prefix: /usr
-  # Don't use external openssl, too old
-  #openssl:
-  #  externals:
-  #  - spec: openssl@1.1.1k
-  #    prefix: /usr
-  # Can't use with py-xnrl
-  #perl:
-  #  externals:
-  #  - spec: perl@5.26.3~cpanm+shared+threads
-  #    prefix: /usr
-  pkgconf:
-    externals:
-    - spec: pkgconf@1.4.2
-      prefix: /usr
-  subversion:
-    externals:
-    - spec: subversion@1.10.2
-      prefix: /usr
-  tar:
-    externals:
-    - spec: tar@1.30
-      prefix: /usr
-  texinfo:
-    externals:
-    - spec: texinfo@6.5
-      prefix: /usr
-  wget:
-    externals:
-    - spec: wget@1.19.5
-      prefix: /usr

From ab722bb3cfa31a761aa78f25c6c2800e363467b1 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 4 Oct 2024 10:21:35 -0600
Subject: [PATCH 20/61] More updates to atlantis site config for GNU

---
 configs/sites/tier1/atlantis/modules.yaml      |  3 +++
 configs/sites/tier1/atlantis/packages_gcc.yaml | 12 +++---------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml
index 11c5514e2..a3a386a9c 100644
--- a/configs/sites/tier1/atlantis/modules.yaml
+++ b/configs/sites/tier1/atlantis/modules.yaml
@@ -6,3 +6,6 @@ modules:
       include:
       # List of packages for which we need modules that are blacklisted by default
       - python
+      - openmpi
+      exclude:
+      - slurm
diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml
index 7d36ed0b4..c5e189964 100644
--- a/configs/sites/tier1/atlantis/packages_gcc.yaml
+++ b/configs/sites/tier1/atlantis/packages_gcc.yaml
@@ -10,14 +10,8 @@ packages:
     buildable: False
     externals:
     - spec: slurm@21.08.8 +pmix
-      modules:
-      - slurm/slurm/21.08.8
+      prefix: /cm/shared/apps/slurm/21.08.8
+      #modules:
+      #- slurm/slurm/21.08.8
   openmpi:
     require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers'
-modules:
-  default:
-    lmod:
-      include:
-      - openmpi
-      exclude:
-      - slurm

From 46234a70f0c72f5e94c177773219eb8977a3ba9f Mon Sep 17 00:00:00 2001
From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com>
Date: Mon, 7 Oct 2024 07:14:23 -0400
Subject: [PATCH 21/61] fix openmpi version on Hercules (#1326)

---
 configs/sites/tier1/hercules/packages_gcc.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/configs/sites/tier1/hercules/packages_gcc.yaml b/configs/sites/tier1/hercules/packages_gcc.yaml
index 5781224d8..37c496c45 100644
--- a/configs/sites/tier1/hercules/packages_gcc.yaml
+++ b/configs/sites/tier1/hercules/packages_gcc.yaml
@@ -2,13 +2,13 @@ packages:
   all:
     compiler:: [gcc@12.2.0]
     providers:
-      mpi:: [openmpi@4.1.6]
+      mpi:: [openmpi@4.1.4]
   mpi:
     buildable: False
   openmpi:
     externals:
-    - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
+    - spec: openmpi@4.1.4%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
         schedulers=slurm
       modules:
       - gcc/12.2.0
-      - openmpi/4.1.6-gcc-12.2.0-spack
+      - openmpi/4.1.4

From 5101d55845ad79c591874367ec34262d80b22229 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 7 Oct 2024 15:50:04 -0600
Subject: [PATCH 22/61] For release/1.8.0: update S4 site config and
 documentation (#1328)

* Update site config and documentation for S4 after Rocky8 update (Intel only so far with gcc@13 backend)
---
 configs/sites/tier1/s4/compilers.yaml |  48 +++++----
 configs/sites/tier1/s4/packages.yaml  | 141 +++++++++++++-------------
 doc/source/PreConfiguredSites.rst     |   5 -
 3 files changed, 102 insertions(+), 92 deletions(-)

diff --git a/configs/sites/tier1/s4/compilers.yaml b/configs/sites/tier1/s4/compilers.yaml
index d364dd768..dff2729fb 100644
--- a/configs/sites/tier1/s4/compilers.yaml
+++ b/configs/sites/tier1/s4/compilers.yaml
@@ -1,34 +1,46 @@
 compilers:
 - compiler:
-    spec: intel@2021.5.0
+    spec: intel@=2021.10.0
     paths:
-      cc: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/icc
-      cxx: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/icpc
-      f77: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/ifort
-      fc: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/ifort
+      cc: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/icc
+      cxx: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/icpc
+      f77: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/ifort
+      fc: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/ifort
     flags: {}
-    operating_system: centos7
+    operating_system: rocky8
     target: x86_64
     modules:
-    - license_intel/S4
-    - intel/2022.1
+    - intel/2023.2
     environment:
       prepend_path:
-        PATH: '/data/prod/hpc-stack/gnu/9.3.0/bin'
-        LD_LIBRARY_PATH: '/home/opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/compiler/lib/intel64_lin:/data/prod/hpc-stack/gnu/9.3.0/lib64'
-        CPATH: '/data/prod/hpc-stack/gnu/9.3.0/include'
+        PATH: /opt/gcc/13.3/bin
+        LD_LIBRARY_PATH: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/opt/gcc/13.3/lib64:/opt/gcc/13.3/lib
+        CPATH: /opt/gcc/13.3/include
     extra_rpaths: []
 - compiler:
-    spec: gcc@9.3.0
+    spec: gcc@=13.3.0
     paths:
-      cc: /data/prod/hpc-stack/gnu/9.3.0/bin/gcc
-      cxx: /data/prod/hpc-stack/gnu/9.3.0/bin/g++
-      f77: /data/prod/hpc-stack/gnu/9.3.0/bin/gfortran
-      fc: /data/prod/hpc-stack/gnu/9.3.0/bin/gfortran
+      cc: /opt/gcc/13.3/bin/gcc
+      cxx: /opt/gcc/13.3/bin/g++
+      f77: /opt/gcc/13.3/bin/gfortran
+      fc: /opt/gcc/13.3/bin/gfortran
     flags: {}
-    operating_system: centos7
+    operating_system: rocky8
     target: x86_64
     modules:
-    - gnu/9.3.0
+    - gcc/13.3
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: gcc@=8.5.0
+    paths:
+      cc: /usr/bin/gcc
+      cxx: /usr/bin/g++
+      f77: /usr/bin/gfortran
+      fc: /usr/bin/gfortran
+    flags: {}
+    operating_system: rocky8
+    target: x86_64
+    modules: []
     environment: {}
     extra_rpaths: []
diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml
index e0312b799..a2457222e 100644
--- a/configs/sites/tier1/s4/packages.yaml
+++ b/configs/sites/tier1/s4/packages.yaml
@@ -1,10 +1,8 @@
 packages:
   all:
-    compiler:: [intel@2021.5.0,gcc@9.3.0]
+    compiler:: [intel@2021.10.0,gcc@13.3.0]
     providers:
-      mpi:: [intel-oneapi-mpi@2021.5.0]
-      # https://github.com/JCSDA/spack-stack/issues/1055
-      zlib-api:: [zlib]
+      mpi:: [intel-oneapi-mpi@2021.10.0]
       # Remove the next three lines to switch to intel-oneapi-mkl
       blas:: [openblas]
       fftw-api:: [fftw]
@@ -15,15 +13,18 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0
-      prefix: /opt/intel/oneapi/2022.1
+    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0
+      prefix: /opt/intel/oneapi/2023.2
+      modules:
+      - intel/2023.2
   intel-oneapi-mkl:
     # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl
     buildable: False
     #externals:
-    #- spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0
-    #  prefix: /opt/intel/oneapi/2022.1
-
+    #- spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0
+    #  prefix: /opt/intel/oneapi/2023.2
+    #  modules:
+    #  - intel/2023.2
   # Remove this section to switch to intel-oneapi-mkl
   ectrans:
     require::
@@ -42,47 +43,63 @@ packages:
       prefix: /usr
   automake:
     externals:
-    - spec: automake@1.15
+    - spec: automake@1.16.1
       prefix: /usr
   bash:
     externals:
-    - spec: bash@4.2.46
+    - spec: bash@4.4.20
       prefix: /usr
   berkeley-db:
     externals:
-    - spec: berkeley-db@5.3.21
+    - spec: berkeley-db@5.3.28
+      prefix: /usr
+  binutils:
+    externals:
+    - spec: binutils@2.30.123
+      prefix: /usr
+  bison:
+    externals:
+    - spec: bison@3.0.4
       prefix: /usr
   bzip2:
     externals:
     - spec: bzip2@1.0.6
       prefix: /usr
+  cmake:
+    externals:
+    - spec: cmake@3.26.5
+      prefix: /usr
+  coreutils:
+    externals:
+    - spec: coreutils@8.30
+      prefix: /usr
   cpio:
     externals:
-    - spec: cpio@2.11
+    - spec: cpio@2.12
       prefix: /usr
-  diffutils:
+  curl:
     externals:
-    - spec: diffutils@3.3
+    - spec: curl@7.61.1+gssapi+ldap+nghttp2
       prefix: /usr
-  doxygen:
+  diffutils:
     externals:
-    - spec: doxygen@1.8.5+graphviz~mscgen
+    - spec: diffutils@3.6
       prefix: /usr
   file:
     externals:
-    - spec: file@5.11
+    - spec: file@5.33
       prefix: /usr
   findutils:
     externals:
-    - spec: findutils@4.5.11
+    - spec: findutils@4.6.0
       prefix: /usr
   flex:
     externals:
-    - spec: flex@2.5.37+lex
+    - spec: flex@2.6.1+lex
       prefix: /usr
   gawk:
     externals:
-    - spec: gawk@4.0.2
+    - spec: gawk@4.2.1
       prefix: /usr
   gettext:
     externals:
@@ -90,105 +107,91 @@ packages:
       prefix: /usr
   ghostscript:
     externals:
-    - spec: ghostscript@9.25
+    - spec: ghostscript@9.27
       prefix: /usr
   git:
     externals:
-    - spec: git@2.30.0+tcltk
-      prefix: /opt/git/2.30.0
-      modules:
-      - git/2.30.0
+    - spec: git@2.43.5+tcltk
+      prefix: /usr
   git-lfs:
     externals:
-    - spec: git-lfs@2.10.0
+    - spec: git-lfs@3.5.1
       prefix: /usr
   gmake:
     externals:
-    - spec: gmake@3.82
+    - spec: gmake@4.2.1
       prefix: /usr
   go:
     externals:
-    - spec: go@1.11.5
+    - spec: go@1.21.13
       prefix: /usr
   groff:
     externals:
-    - spec: groff@1.22.2
-      prefix: /usr
-  openjdk:
-    externals:
-    - spec: openjdk@1.8.0_262-b10
+    - spec: groff@1.22.3
       prefix: /usr
   krb5:
     externals:
-    - spec: krb5@1.15.1
-      prefix: /usr
-  libfuse:
-    externals:
-    - spec: libfuse@2.9.2
+    - spec: krb5@1.18.2
       prefix: /usr
   libtool:
     externals:
-    - spec: libtool@2.4.2
+    - spec: libtool@2.4.6
       prefix: /usr
-  libxpm:
+  m4:
     externals:
-    - spec: libxpm@4.11.0
+    - spec: m4@1.4.18
       prefix: /usr
-  lustre:
+  openssh:
     externals:
-    - spec: lustre@2.10.5
+    - spec: openssh@8.0p1
       prefix: /usr
-  m4:
+  openssl:
     externals:
-    - spec: m4@1.4.16
+    - spec: openssl@1.1.1k
       prefix: /usr
-  mysql:
-    buildable: False
-    externals:
-    - spec: mysql@8.0.31
-      prefix: /data/prod/jedi/spack-stack/mysql-8.0.31
-      modules:
-      - mysql/8.0.31
-  pkg-config:
+  perl:
     externals:
-    - spec: pkg-config@0.27.1
+    - spec: perl@5.26.3~cpanm+opcode+open+shared+threads
       prefix: /usr
-  qt:
-    buildable: False
+  pkgconf:
     externals:
-    - spec: qt@5.9.7
+    - spec: pkgconf@1.4.2
       prefix: /usr
+  # TODO - installing via spack-stack works, for example for
+  # spack-stack-1.8.0, but would be good if we didn't have to.
+  # https://github.com/JCSDA/spack-stack/issues/1329
+  #qt:
+  #  buildable: False
+  #  externals:
+  #  - spec: qt@5.9.7
+  #    prefix: /usr
   rsync:
     externals:
-    - spec: rsync@3.1.2
+    - spec: rsync@3.1.3
       prefix: /usr
   ruby:
     externals:
-    - spec: ruby@2.0.0
+    - spec: ruby@2.5.9
       prefix: /usr
   sed:
     externals:
-    - spec: sed@4.2.2
+    - spec: sed@4.5
       prefix: /usr
   tar:
     externals:
-    - spec: tar@1.26
+    - spec: tar@1.30
       prefix: /usr
   texinfo:
     externals:
-    - spec: texinfo@5.1
-      prefix: /usr
-  texlive:
-    externals:
-    - spec: texlive@20130530
+    - spec: texinfo@6.5
       prefix: /usr
   wget:
     externals:
-    - spec: wget@1.14
+    - spec: wget@1.19.5
       prefix: /usr
   xz:
     externals:
-    - spec: xz@5.2.2
+    - spec: xz@5.2.4
       prefix: /usr
   zip:
     externals:
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index 2ea670ac2..bfcd808d3 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -343,14 +343,9 @@ UW (Univ. of Wisconsin) S4
 
 The following is required for building new spack environments with any supported compiler on this platform.
 
-**NEEDS UPDATING**
-
 .. code-block:: console
 
    module purge
-   module use /data/prod/jedi/spack-stack/modulefiles
-   module load miniconda/3.9.12
-   module load ecflow/5.8.4
 
 
 .. _Preconfigured_Sites_AWS_Parallelcluster:

From cbf2bd3f66da87278b97b45af974d2a9339e4c1e Mon Sep 17 00:00:00 2001
From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com>
Date: Mon, 7 Oct 2024 21:55:36 -0400
Subject: [PATCH 23/61] Release 1.8.0 additional fixes for Jet, Hercules and
 Orion (after running ufs-weather-model) (#1331)

* Update Jet packages_intel.yaml
* Update Orion packages_intel.yaml
* Update Orion packages_gcc.yaml
* Update Hercules packages.yaml
---
 configs/sites/tier1/hercules/packages.yaml    | 12 ++++++++++++
 configs/sites/tier1/jet/packages_intel.yaml   |  2 ++
 configs/sites/tier1/orion/packages_gcc.yaml   |  2 --
 configs/sites/tier1/orion/packages_intel.yaml |  2 --
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml
index 0128505f1..bb6e6222c 100644
--- a/configs/sites/tier1/hercules/packages.yaml
+++ b/configs/sites/tier1/hercules/packages.yaml
@@ -15,6 +15,10 @@ packages:
     externals:
     - spec: coreutils@8.32
       prefix: /usr
+  curl:
+    externals:
+    - spec: curl@7.76.1+gssapi+ldap+nghttp2
+      prefix: /usr
   diffutils:
     externals:
     - spec: diffutils@3.7
@@ -60,6 +64,14 @@ packages:
       prefix: /work/noaa/epic/role-epic/spack-stack/hercules/mysql-8.0.31
       modules:
       - mysql/8.0.31
+  openssh:
+    externals:
+    - spec: openssh@8.7p1
+      prefix: /usr
+  openssl:
+    externals:
+    - spec: openssl@3.0.1
+      prefix: /usr
   pkgconf:
     externals:
     - spec: pkgconf@1.7.3
diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml
index cc0cac045..5e0b82aaf 100644
--- a/configs/sites/tier1/jet/packages_intel.yaml
+++ b/configs/sites/tier1/jet/packages_intel.yaml
@@ -8,6 +8,8 @@ packages:
   intel-oneapi-mpi:
     externals:
     - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0
+      modules:
+      - impi/2022.1.2
       prefix: /apps/oneapi
   intel-oneapi-mkl:
     externals:
diff --git a/configs/sites/tier1/orion/packages_gcc.yaml b/configs/sites/tier1/orion/packages_gcc.yaml
index 8a8f8d641..aadfeb18c 100644
--- a/configs/sites/tier1/orion/packages_gcc.yaml
+++ b/configs/sites/tier1/orion/packages_gcc.yaml
@@ -3,8 +3,6 @@ packages:
     compiler:: [gcc@12.2.0]
     providers:
       mpi:: [openmpi@4.1.4]
-      # https://github.com/JCSDA/spack-stack/issues/1055
-      zlib-api:: [zlib]
   mpi:
     buildable: False
   openmpi:
diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml
index 8c1e33385..eb4a22dc7 100644
--- a/configs/sites/tier1/orion/packages_intel.yaml
+++ b/configs/sites/tier1/orion/packages_intel.yaml
@@ -3,8 +3,6 @@ packages:
     compiler:: [intel@2021.9.0,gcc@12.2.0]
     providers:
       mpi:: [intel-oneapi-mpi@2021.9.0]
-      # https://github.com/JCSDA/spack-stack/issues/1055
-      zlib-api:: [zlib]
       # Remove the next three lines to switch to intel-oneapi-mkl
       blas:: [openblas]
       fftw-api:: [fftw]

From 4ecbdae943a6709d3221ad13bf3a8a1ba50ded71 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 11 Oct 2024 15:09:42 -0600
Subject: [PATCH 24/61] Next round of updates for Atlantis: use Bright OpenMPI

---
 configs/sites/tier1/atlantis/modules.yaml      |  3 ---
 configs/sites/tier1/atlantis/packages_gcc.yaml | 18 +++++++-----------
 .../sites/tier1/atlantis/packages_intel.yaml   |  2 +-
 3 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml
index a3a386a9c..11c5514e2 100644
--- a/configs/sites/tier1/atlantis/modules.yaml
+++ b/configs/sites/tier1/atlantis/modules.yaml
@@ -6,6 +6,3 @@ modules:
       include:
       # List of packages for which we need modules that are blacklisted by default
       - python
-      - openmpi
-      exclude:
-      - slurm
diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml
index c5e189964..bc10f8b75 100644
--- a/configs/sites/tier1/atlantis/packages_gcc.yaml
+++ b/configs/sites/tier1/atlantis/packages_gcc.yaml
@@ -2,16 +2,12 @@ packages:
   all:
     compiler:: [gcc@11.2.0]
     providers:
-      mpi:: [openmpi@4.1.6]
-      #blas:: [intel-oneapi-mkl]
-      #fftw-api:: [intel-oneapi-mkl]
-      #lapack:: [intel-oneapi-mkl]
-  slurm:
+      mpi:: [openmpi@4.1.5]
+  mpi:
     buildable: False
-    externals:
-    - spec: slurm@21.08.8 +pmix
-      prefix: /cm/shared/apps/slurm/21.08.8
-      #modules:
-      #- slurm/slurm/21.08.8
   openmpi:
-    require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers'
+    buildable: False
+    externals:
+    - spec: openmpi@4.1.5%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      modules:
+      - openmpi/mlnx/gcc/64/4.1.5a1
diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml
index ea2f4de32..4271f3f10 100644
--- a/configs/sites/tier1/atlantis/packages_intel.yaml
+++ b/configs/sites/tier1/atlantis/packages_intel.yaml
@@ -15,6 +15,6 @@ packages:
   intel-oneapi-mkl:
     externals:
     - spec: intel-oneapi-mkl@2022.1.0
+      prefix: /cm/shared/apps/intel/oneapi
       modules:
       -  mkl/2022.1.0
-      prefix: /cm/shared/apps/intel/oneapi

From d6a41f55b18961db57997ef8bec38b1142b943db Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 11 Oct 2024 15:10:38 -0600
Subject: [PATCH 25/61] Update configs/common/packages.yaml: do not require,
 but prefer, the Python variant for ESMF, set accordingly in
 spack-ext/repos/spack-stack/packages/neptune-env/package.py

---
 configs/common/packages.yaml                  | 24 ++++++++++---------
 .../packages/neptune-env/package.py           |  3 ++-
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
index fc7089741..1b48f823b 100644
--- a/configs/common/packages.yaml
+++ b/configs/common/packages.yaml
@@ -54,17 +54,19 @@ packages:
   # Also, check the acorn and derecho site configs which have esmf modifications.
   esmf:
     require:
-      - '~xerces ~pnetcdf +shared +external-parallelio +python'
-      - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
-      - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
-        when: "%intel"
-        message: "Extra ESMF compile options for Intel"
-      #- any_of: ['']
-      #  when: "%gcc"
-      #  message: "Extra ESMF compile options for GCC"
-      #- any_of: ['']
-      #  when: "%apple-clang"
-      #  message: "Extra ESMF compile options for GCC"
+    - '~xerces ~pnetcdf +shared +external-parallelio'
+    - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
+    - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+      when: "%intel"
+      message: "Extra ESMF compile options for Intel"
+    #- any_of: ['']
+    #  when: "%gcc"
+    #  message: "Extra ESMF compile options for GCC"
+    #- any_of: ['']
+    #  when: "%apple-clang"
+    #  message: "Extra ESMF compile options for GCC"
+    prefer:
+    - '+python'
   fckit:
     require: '@0.11.0 +eckit'
   fftw:
diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py
index f79968eac..94c7baac7 100644
--- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py
@@ -39,7 +39,8 @@ class NeptuneEnv(BundlePackage):
     depends_on("ip@5:", type="run", when="%gcc")
     depends_on("ip@5:", type="run", when="%intel")
     depends_on("ip@5:", type="run", when="%oneapi")
-    depends_on("esmf", type="run")
+    depends_on("esmf +python", when="+python", type="run")
+    depends_on("esmf ~python", when="~python", type="run")
     depends_on("nco", type="run")
     depends_on("mct", type="run")
 

From 28ea3f72cf0dbd1c15e0dba67fd3b3dac1ff32da Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 14 Oct 2024 10:00:50 -0600
Subject: [PATCH 26/61] Remove legacy environment variables for hdf5, p4est,
 libyaml from configs/common/modules_{lmod,tcl}.yaml

---
 configs/common/modules_lmod.yaml | 16 ----------------
 configs/common/modules_tcl.yaml  | 16 ----------------
 2 files changed, 32 deletions(-)

diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml
index 41bfa24df..7c994ce45 100644
--- a/configs/common/modules_lmod.yaml
+++ b/configs/common/modules_lmod.yaml
@@ -112,20 +112,10 @@ modules:
         environment:
           set:
             'ESMFMKFILE': '{prefix}/lib/esmf.mk'
-      hdf5:
-        environment:
-          set:
-            'HDF5_DIR': '{prefix}'
       libpng:
         environment:
           set:
             'PNG_ROOT': '{prefix}'
-      libyaml:
-        environment:
-          set:
-            'YAML_DIR': '{prefix}'
-            'YAML_LIB': '{prefix}/lib'
-            'YAML_INC': '{prefix}/include'
       madis:
         environment:
           set:
@@ -151,12 +141,6 @@ modules:
           set:
             'OMPI_MCA_rmaps_base_oversubscribe': '1'
             'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe'
-      p4est:
-        environment:
-          set:
-            'P4EST_API_DIR': '{prefix}'
-            'P4EST_API_LIB': '{prefix}/lib'
-            'P4EST_API_INC': '{prefix}/include'
       bacio:
         environment:
           set:
diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml
index 5ce83e8b4..551c12e65 100644
--- a/configs/common/modules_tcl.yaml
+++ b/configs/common/modules_tcl.yaml
@@ -114,20 +114,10 @@ modules:
         environment:
           set:
             'ESMFMKFILE': '{prefix}/lib/esmf.mk'
-      hdf5:
-        environment:
-          set:
-            'HDF5_DIR': '{prefix}'
       libpng:
         environment:
           set:
             'PNG_ROOT': '{prefix}'
-      libyaml:
-        environment:
-          set:
-            'YAML_DIR': '{prefix}'
-            'YAML_LIB': '{prefix}/lib'
-            'YAML_INC': '{prefix}/include'
       madis:
         environment:
           set:
@@ -153,12 +143,6 @@ modules:
           set:
             'OMPI_MCA_rmaps_base_oversubscribe': '1'
             'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe'
-      p4est:
-        environment:
-          set:
-            'P4EST_API_DIR': '{prefix}'
-            'P4EST_API_LIB': '{prefix}/lib'
-            'P4EST_API_INC': '{prefix}/include'
       bacio:
         environment:
           set:

From cd7c3208a04fc42339f03a4e7f004de4b47510ff Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 14 Oct 2024 10:02:09 -0600
Subject: [PATCH 27/61] Split neptune-env into neptune-env (with default espc
 false) and neptune-python-env

---
 configs/templates/neptune-dev/spack.yaml      |  3 +-
 .../packages/neptune-env/package.py           | 33 ++++-----------
 .../packages/neptune-python-env/package.py    | 41 +++++++++++++++++++
 3 files changed, 50 insertions(+), 27 deletions(-)
 create mode 100644 spack-ext/repos/spack-stack/packages/neptune-python-env/package.py

diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml
index 5f3332f79..e0219fa0d 100644
--- a/configs/templates/neptune-dev/spack.yaml
+++ b/configs/templates/neptune-dev/spack.yaml
@@ -8,7 +8,8 @@ spack:
   definitions:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
   - packages:
-      - neptune-env +espc +python +xnrl ^esmf@8.7.0b11 snapshot=b11
+      - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11
+      - neptune-python-env +xnrl
 
   specs:
     - matrix:
diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py
index f79968eac..ddd732692 100644
--- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,7 +9,7 @@
 
 
 class NeptuneEnv(BundlePackage):
-    """Development environment for neptune standalone"""
+    """Development environment for NEPTUNE standalone"""
 
     # Fake URL
     homepage = "https://github.com/notavalidaccount/neptune"
@@ -17,11 +17,9 @@ class NeptuneEnv(BundlePackage):
 
     maintainers("climbfuji", "areinecke")
 
-    version("1.4.0")
+    version("1.5.0")
 
-    variant("python", default=True, description="Build Python dependencies")
-    variant("espc", default=True, description="Build ESPC dependencies")
-    variant("xnrl", default=True, description="Build XNRL and its extra Python dependencies")
+    variant("espc", default=False, description="Build ESPC dependencies")
 
     depends_on("base-env", type="run")
 
@@ -33,7 +31,6 @@ class NeptuneEnv(BundlePackage):
     depends_on("libyaml", type="run")
     depends_on("p4est", type="run")
     depends_on("w3emc", type="run")
-    depends_on("w3nco", type="run")
     depends_on("sp", type="run", when="%aocc")
     depends_on("ip@5:", type="run", when="%apple-clang")
     depends_on("ip@5:", type="run", when="%gcc")
@@ -43,28 +40,12 @@ class NeptuneEnv(BundlePackage):
     depends_on("nco", type="run")
     depends_on("mct", type="run")
 
-    conflicts("+xnrl", when="~python", msg="Variant xnrl requires variant python")
-
     with when("+espc"):
         depends_on("fftw", type="build")
         depends_on("netlib-lapack", type="build")
 
-    with when("+python"):
-        depends_on("py-f90nml", type="run")
-        depends_on("py-h5py", type="run")
-        depends_on("py-netcdf4", type="run")
-        depends_on("py-pandas", type="run")
-        depends_on("py-pycodestyle", type="run")
-        depends_on("py-pybind11", type="run")
-        depends_on("py-pyhdf", type="run")
-        depends_on("py-python-dateutil", type="run")
-        depends_on("py-pyyaml", type="run")
-        depends_on("py-scipy", type="run")
-        depends_on("py-xarray", type="run")
-        depends_on("py-pytest", type="run")
-        depends_on("py-fortranformat", type="run")
-
-    with when("+xnrl"):
-        depends_on("py-xnrl", type="run")
+    # Basic Python dependencies that are always needed
+    depends_on("py-f90nml", type="run")
+    depends_on("py-python-dateutil", type="run")
 
     # There is no need for install() since there is no code.
diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py
new file mode 100644
index 000000000..5b6f243b8
--- /dev/null
+++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py
@@ -0,0 +1,41 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+
+from spack.package import *
+
+
+class NeptunePythonEnv(BundlePackage):
+    """Development environment for NEPTUNE standalone with all Python dependencies"""
+
+    # Fake URL
+    homepage = "https://github.com/notavalidaccount/neptune"
+    git = "https://github.com/notavalidaccount/neptune.git"
+
+    maintainers("climbfuji", "areinecke")
+
+    version("1.5.0")
+
+    variant("xnrl", default=False, description="Build non-pulic XNRL")
+
+    depends_on("neptune-env", type="run")
+
+    depends_on("py-h5py", type="run")
+    depends_on("py-netcdf4", type="run")
+    depends_on("py-pandas", type="run")
+    depends_on("py-pycodestyle", type="run")
+    depends_on("py-pybind11", type="run")
+    depends_on("py-pyhdf", type="run")
+    depends_on("py-pyyaml", type="run")
+    depends_on("py-scipy", type="run")
+    depends_on("py-xarray", type="run")
+    depends_on("py-pytest", type="run")
+    depends_on("py-fortranformat", type="run")
+
+    with when("+xnrl"):
+        depends_on("py-xnrl", type="run")
+
+    # There is no need for install() since there is no code.

From 886cf903d9116e2b05a47240a05efcde0e47b193 Mon Sep 17 00:00:00 2001
From: Nate Crossette <ncrossette@ucar.edu>
Date: Mon, 14 Oct 2024 06:22:32 -0600
Subject: [PATCH 28/61] Add note to docs for selecting branch when cloning
 (#1341)

---
 doc/source/NewSiteConfigs.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst
index 036fa41f9..65b59b6b0 100644
--- a/doc/source/NewSiteConfigs.rst
+++ b/doc/source/NewSiteConfigs.rst
@@ -211,11 +211,11 @@ Creating a new environment
 
 Remember to activate the ``lua`` module environment and have MacTeX in your search path, if applicable. It is also recommended to increase the stacksize limit to 65Kb using ``ulimit -S -s unlimited``.
 
-1. You will need to clone spack-stack and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use.
+1. You will need to clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use.
 
 .. code-block:: console
 
-   git clone --recurse-submodules https://github.com/jcsda/spack-stack.git
+   git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git
    cd spack-stack
 
    # Sources Spack from submodule and sets ${SPACK_STACK_DIR}
@@ -500,11 +500,11 @@ Creating a new environment
 
 It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimited``, and to test if the module environment functions correctly (``module available``).
 
-1. You will need to clone spack-stack and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use.
+1. You will need to clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use.
 
 .. code-block:: console
 
-   git clone --recurse-submodules https://github.com/jcsda/spack-stack.git
+   git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git
    cd spack-stack
 
    # Sources Spack from submodule and sets ${SPACK_STACK_DIR}
@@ -675,11 +675,11 @@ With all of that in mind, the following instructions were used on an Amazon Web
    module use /opt/nvidia/hpc_sdk/modulefiles
    module load nvhpc-openmpi3/24.3
 
-4. Clone spack-stack and its dependencies and activate the spack-stack tool.
+4. Clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool.
 
 .. code-block:: console
 
-   git clone --recurse-submodules https://github.com/jcsda/spack-stack.git
+   git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git
    cd spack-stack
 
    # Sources Spack from submodule and sets ${SPACK_STACK_DIR}

From 3d25bfb129406711d6a1067602116eb65dc9570b Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 14 Oct 2024 10:55:30 -0600
Subject: [PATCH 29/61] Update
 configs/templates/{neptune-dev,skylab-dev,unified-dev}

---
 configs/templates/neptune-dev/spack.yaml | 4 ++--
 configs/templates/skylab-dev/spack.yaml  | 3 ++-
 configs/templates/unified-dev/spack.yaml | 3 ++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml
index e0219fa0d..564cf6d86 100644
--- a/configs/templates/neptune-dev/spack.yaml
+++ b/configs/templates/neptune-dev/spack.yaml
@@ -8,8 +8,8 @@ spack:
   definitions:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
   - packages:
-      - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11
-      - neptune-python-env +xnrl
+      - neptune-env +espc        ^esmf@8.7.0b11 snapshot=b11
+      - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11
 
   specs:
     - matrix:
diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml
index 1ba272a63..4616138ca 100644
--- a/configs/templates/skylab-dev/spack.yaml
+++ b/configs/templates/skylab-dev/spack.yaml
@@ -17,7 +17,8 @@ spack:
     - jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
     - jedi-ufs-env          ^esmf@=8.6.1
     - jedi-um-env
-    - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-env           ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-python-env    ^esmf@=8.7.0b11 snapshot=b11
     - soca-env
 
     # Various crtm tags (list all to avoid duplicate packages)
diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml
index a05ed4a75..608b4462a 100644
--- a/configs/templates/unified-dev/spack.yaml
+++ b/configs/templates/unified-dev/spack.yaml
@@ -21,7 +21,8 @@ spack:
     - jedi-tools-env
     - jedi-ufs-env          ^esmf@=8.6.1
     - jedi-um-env
-    - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-env           ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-python-env    ^esmf@=8.7.0b11 snapshot=b11
     - soca-env
     - ufs-srw-app-env       ^esmf@=8.6.1
     - ufs-weather-model-env ^esmf@=8.6.1

From d6641e4968207de1b83791fdd518e76660c488d8 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 16 Oct 2024 10:40:34 -0600
Subject: [PATCH 30/61] Add py-pyyaml to neptune-env, remove invalid Python
 variant from neptune-env and add esmf+python to neptune-python-env

---
 spack-ext/repos/spack-stack/packages/neptune-env/package.py   | 4 ++--
 .../repos/spack-stack/packages/neptune-python-env/package.py  | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py
index a8d3d5807..ff7aef9d4 100644
--- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py
@@ -36,8 +36,7 @@ class NeptuneEnv(BundlePackage):
     depends_on("ip@5:", type="run", when="%gcc")
     depends_on("ip@5:", type="run", when="%intel")
     depends_on("ip@5:", type="run", when="%oneapi")
-    depends_on("esmf +python", when="+python", type="run")
-    depends_on("esmf ~python", when="~python", type="run")
+    depends_on("esmf", type="run")
     depends_on("nco", type="run")
     depends_on("mct", type="run")
 
@@ -48,5 +47,6 @@ class NeptuneEnv(BundlePackage):
     # Basic Python dependencies that are always needed
     depends_on("py-f90nml", type="run")
     depends_on("py-python-dateutil", type="run")
+    depends_on("py-pyyaml", type="run")
 
     # There is no need for install() since there is no code.
diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py
index 5b6f243b8..2e20105b8 100644
--- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py
@@ -22,6 +22,8 @@ class NeptunePythonEnv(BundlePackage):
     variant("xnrl", default=False, description="Build non-pulic XNRL")
 
     depends_on("neptune-env", type="run")
+    # Enable the Python variant for ESMF
+    depends_on("esmf +python", type="run")
 
     depends_on("py-h5py", type="run")
     depends_on("py-netcdf4", type="run")

From c6a067f7730e4554fd75f9685d603d10f60c55de Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 16 Oct 2024 14:33:29 -0600
Subject: [PATCH 31/61] Update site config and documentation for Narwhal to use
 new, official oneapi@2024.2 installation

---
 configs/sites/tier1/narwhal/compilers.yaml    | 16 ++++------------
 .../sites/tier1/narwhal/packages_oneapi.yaml  | 19 +++++++------------
 doc/source/PreConfiguredSites.rst             | 10 ++--------
 3 files changed, 13 insertions(+), 32 deletions(-)

diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml
index 5e2e17c11..838a8a1dd 100644
--- a/configs/sites/tier1/narwhal/compilers.yaml
+++ b/configs/sites/tier1/narwhal/compilers.yaml
@@ -22,12 +22,8 @@ compilers::
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
   - compiler:
-      spec: oneapi@2024.2.1
+      spec: oneapi@2024.2.0
       paths:
-        #cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx
-        #cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx
-        #f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort
-        #fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort
         cc: cc
         cxx: CC
         f77: ftn
@@ -35,11 +31,8 @@ compilers::
       flags: {}
       operating_system: sles15
       modules:
-      - PrgEnv-intel/8.3.3
-      - tbb/2021.13
-      - compiler-rt/2024.2.1
-      - compiler/2024.2.1
-      - ifort/2024.2.1
+      - PrgEnv-intel/8.4.0
+      - intel/2024.2
       - cray-libsci/23.05.1.4
       - libfabric/1.12.1.2.2.1
       environment:
@@ -47,9 +40,8 @@ compilers::
           PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
           CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
           LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
-          MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles'
         append_path:
-          CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64'
+          CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64'
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml
index 9b032dcf1..4724e25a3 100644
--- a/configs/sites/tier1/narwhal/packages_oneapi.yaml
+++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml
@@ -7,26 +7,21 @@ packages:
     buildable: False
   cray-mpich:
     externals:
-    - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers
+    - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers
       modules:
       - cray-mpich-ucx/8.1.21
       - craype-network-ucx
   intel-oneapi-mkl:
     externals:
-    - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1
-      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
-      modules:
-      - mkl/2024.2
+    - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0
+      prefix: /opt/intel/oneapi_2024.2.0.634
   intel-oneapi-tbb:
     externals:
-    - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1
-      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
+    - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0
+      prefix: /opt/intel/oneapi_2024.2.0.634
       modules:
       - tbb/2021.13
   intel-oneapi-runtime:
     externals:
-    - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1
-      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1
-      modules:
-      - tbb/2021.13
-      - compiler-rt/2024.2.1
+    - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0
+      prefix: /opt/intel/oneapi_2024.2.0.634
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index 79567859e..cd3b8e3bb 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -166,19 +166,13 @@ The following is required for building new spack environments with Intel on this
 
 The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal!
 
-.. note:: This particular setup of the oneAPI compilers is outside the Cray-blessed environment and there are problems installing Python packages with spack.
-
 .. code-block:: console
 
    umask 0022
    module unload PrgEnv-cray
-   module load PrgEnv-intel/8.3.3
+   module load PrgEnv-intel/8.4.0
    module unload intel
-   module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles
-   module load tbb/2021.13
-   module load compiler-rt/2024.2.1
-   module load compiler/2024.2.1
-   module load ifort/2024.2.1
+   module load intel/2024.2
    module unload cray-mpich
    module unload craype-network-ofi
    # Warning. Do not load craype-network-ucx

From 716932c9dc5e4597c60fc61c50877e13694fa742 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 18 Oct 2024 07:59:46 -0600
Subject: [PATCH 32/61] Fix typo in
 configs/sites/tier1/narwhal/packages_oneapi.yaml

---
 configs/sites/tier1/narwhal/packages_oneapi.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml
index 4724e25a3..3bcf99f38 100644
--- a/configs/sites/tier1/narwhal/packages_oneapi.yaml
+++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml
@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [oneapi@2024.2.1,gcc@10.3.0]
+    compiler:: [oneapi@2024.2.0,gcc@10.3.0]
     providers:
       mpi:: [cray-mpich@8.1.21]
   mpi:

From 41b59e35d7358e2418f1bbad04a2208928ef7be5 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 18 Oct 2024 18:37:17 -0600
Subject: [PATCH 33/61] Update submodule pointer for spack (esmf@8.7.0) and
 replace esmf@8.7.0b11 with esmf@8.7.0 in all configs

---
 configs/common/modules_lmod.yaml         | 2 ++
 configs/common/modules_tcl.yaml          | 2 ++
 configs/common/packages                  | 0
 configs/common/packages.yaml             | 2 +-
 configs/templates/neptune-dev/spack.yaml | 4 ++--
 configs/templates/skylab-dev/spack.yaml  | 8 ++++----
 configs/templates/unified-dev/spack.yaml | 8 ++++----
 spack                                    | 2 +-
 8 files changed, 16 insertions(+), 12 deletions(-)
 create mode 100644 configs/common/packages

diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml
index 7c994ce45..8fe637fb6 100644
--- a/configs/common/modules_lmod.yaml
+++ b/configs/common/modules_lmod.yaml
@@ -136,6 +136,8 @@ modules:
           ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug'
           ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11'
           ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug'
+          ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0'
+          ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug'
       openmpi:
         environment:
           set:
diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml
index 551c12e65..3275aee44 100644
--- a/configs/common/modules_tcl.yaml
+++ b/configs/common/modules_tcl.yaml
@@ -138,6 +138,8 @@ modules:
           ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug'
           ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11'
           ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug'
+          ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0'
+          ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug'
       openmpi:
         environment:
           set:
diff --git a/configs/common/packages b/configs/common/packages
new file mode 100644
index 000000000..e69de29bb
diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
index 1b48f823b..99f17b060 100644
--- a/configs/common/packages.yaml
+++ b/configs/common/packages.yaml
@@ -55,7 +55,7 @@ packages:
   esmf:
     require:
     - '~xerces ~pnetcdf +shared +external-parallelio'
-    - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
+    - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none']
     - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
       when: "%intel"
       message: "Extra ESMF compile options for Intel"
diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml
index 564cf6d86..9a9e3dfec 100644
--- a/configs/templates/neptune-dev/spack.yaml
+++ b/configs/templates/neptune-dev/spack.yaml
@@ -8,8 +8,8 @@ spack:
   definitions:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
   - packages:
-      - neptune-env +espc        ^esmf@8.7.0b11 snapshot=b11
-      - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11
+      - neptune-env +espc        ^esmf@8.7.0
+      - neptune-python-env +xnrl ^esmf@8.7.0
 
   specs:
     - matrix:
diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml
index 4616138ca..be901ec7e 100644
--- a/configs/templates/skylab-dev/spack.yaml
+++ b/configs/templates/skylab-dev/spack.yaml
@@ -14,11 +14,11 @@ spack:
     - jedi-fv3-env
     - jedi-geos-env         ^esmf@=8.6.1
     - jedi-mpas-env
-    - jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
+    - jedi-neptune-env      ^esmf@=8.7.0
     - jedi-ufs-env          ^esmf@=8.6.1
     - jedi-um-env
-    - neptune-env           ^esmf@=8.7.0b11 snapshot=b11
-    - neptune-python-env    ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-env           ^esmf@=8.7.0
+    - neptune-python-env    ^esmf@=8.7.0
     - soca-env
 
     # Various crtm tags (list all to avoid duplicate packages)
@@ -27,7 +27,7 @@ spack:
 
     # Various esmf tags (list all to avoid duplicate packages)
     - esmf@=8.6.1 snapshot=none
-    - esmf@=8.7.0b11 snapshot=b11
+    - esmf@=8.7.0 snapshot=none
 
   specs:
   - matrix:
diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml
index 608b4462a..370a03e7c 100644
--- a/configs/templates/unified-dev/spack.yaml
+++ b/configs/templates/unified-dev/spack.yaml
@@ -17,12 +17,12 @@ spack:
     - jedi-fv3-env
     - jedi-geos-env         ^esmf@=8.6.1
     - jedi-mpas-env
-    - jedi-neptune-env      ^esmf@=8.7.0b11 snapshot=b11
+    - jedi-neptune-env      ^esmf@=8.7.0
     - jedi-tools-env
     - jedi-ufs-env          ^esmf@=8.6.1
     - jedi-um-env
-    - neptune-env           ^esmf@=8.7.0b11 snapshot=b11
-    - neptune-python-env    ^esmf@=8.7.0b11 snapshot=b11
+    - neptune-env           ^esmf@=8.7.0
+    - neptune-python-env    ^esmf@=8.7.0
     - soca-env
     - ufs-srw-app-env       ^esmf@=8.6.1
     - ufs-weather-model-env ^esmf@=8.6.1
@@ -33,7 +33,7 @@ spack:
 
     # Various esmf tags (list all to avoid duplicate packages)
     - esmf@=8.6.1 snapshot=none
-    - esmf@=8.7.0b11 snapshot=b11
+    - esmf@=8.7.0 snapshot=none
 
     # MADIS for WCOSS2 decoders.
     - madis@4.5
diff --git a/spack b/spack
index 6f9a6ba4e..b9f2dd225 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9
+Subproject commit b9f2dd2254cfb1086040f1a100746bd22cb12702

From 4b2afed39127320af04f3b1d80d2e98dc17db101 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 09:44:20 -0600
Subject: [PATCH 34/61] Change spack source-cache locations on atlantis,
 narwhal, nautilus

---
 configs/sites/tier1/atlantis/mirrors.yaml | 2 --
 configs/sites/tier1/narwhal/mirrors.yaml  | 4 ++--
 configs/sites/tier1/nautilus/mirrors.yaml | 4 ++--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml
index 94e16cfaf..4d363e009 100644
--- a/configs/sites/tier1/atlantis/mirrors.yaml
+++ b/configs/sites/tier1/atlantis/mirrors.yaml
@@ -1,7 +1,6 @@
 mirrors:
   local-source:
     fetch:
-      # TODO UPDATE?
       url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
@@ -10,7 +9,6 @@ mirrors:
       profile: null
       endpoint_url: null
     push:
-      # TODO UPDATE?
       url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml
index 709f9096c..ec3db4a00 100644
--- a/configs/sites/tier1/narwhal/mirrors.yaml
+++ b/configs/sites/tier1/narwhal/mirrors.yaml
@@ -1,7 +1,7 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null
@@ -9,7 +9,7 @@ mirrors:
       profile: null
       endpoint_url: null
     push:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null
diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml
index 709f9096c..ec3db4a00 100644
--- a/configs/sites/tier1/nautilus/mirrors.yaml
+++ b/configs/sites/tier1/nautilus/mirrors.yaml
@@ -1,7 +1,7 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null
@@ -9,7 +9,7 @@ mirrors:
       profile: null
       endpoint_url: null
     push:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null

From 84d8f78692067cafd159fb153d0ff7bedc57e540 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 10:36:44 -0600
Subject: [PATCH 35/61] Add TEMPLATES list and loop over templates; rename
 util/weekly_build/03_SetupUnifiedEnv.sh to util/weekly_build/03_SetupEnv.sh

---
 util/weekly_build/03_SetupEnv.sh              | 33 +++++++++++++++++++
 util/weekly_build/03_SetupUnifiedEnv.sh       | 32 ------------------
 .../SpackStackBuildCache_AllSteps.sh          |  3 +-
 util/weekly_build/sites/acorn.sh              |  1 +
 util/weekly_build/sites/nautilus.sh           |  2 +-
 5 files changed, 36 insertions(+), 35 deletions(-)
 create mode 100755 util/weekly_build/03_SetupEnv.sh
 delete mode 100755 util/weekly_build/03_SetupUnifiedEnv.sh

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
new file mode 100755
index 000000000..40704086d
--- /dev/null
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+set -ex
+
+cd $RUNDIR/$RUNID
+
+set +x
+. setup.sh
+set -x
+
+for compiler in $COMPILERS; do
+  for template in $TEMPLATES; do
+    echo "Setting up environment build-${compiler/@/-} using template $template"
+    rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-}
+    spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler
+    cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
+    spack env activate .
+    spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
+    # Check for duplicates and fail before doing the "real" concretization:
+    spack concretize --fresh 2>&1 | tee log.concretize
+    ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
+    spack concretize --fresh --force
+#   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
+#    spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
+    # Get path to local-source mirror from spack site config
+    SOURCE_MIRROR=`spack mirror list | grep local-source`
+    SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//}
+    # Download all source codes needed to build the environment
+    spack mirror create -a -d ${SOURCE_MIRROR}
+  done
+done
diff --git a/util/weekly_build/03_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupUnifiedEnv.sh
deleted file mode 100755
index a6d360ed5..000000000
--- a/util/weekly_build/03_SetupUnifiedEnv.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
-
-set -ex
-
-cd $RUNDIR/$RUNID
-
-set +x
-. setup.sh
-set -x
-
-for compiler in $COMPILERS; do
-  echo "Setting up environment build-${compiler/@/-} using template unified-dev"
-  #echo "Setting up environment build-${compiler/@/-} using template unified-dev"
-  rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler
-  cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack env activate .
-  spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
-  # Check for duplicates and fail before doing the "real" concretization:
-  spack concretize --fresh 2>&1 | tee log.concretize
-  ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
-  spack concretize --fresh --force
-# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
-#  spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
-  # Get path to local-source mirror from spack site config
-  SOURCE_MIRROR=`spack mirror list | grep local-source`
-  SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//}
-  # Download all source codes needed to build the environment
-  spack mirror create -a -d ${SOURCE_MIRROR}
-done
diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
index 0f8bb3870..73a7931c4 100755
--- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh
+++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
@@ -18,8 +18,7 @@ function trap_and_run {
 
 trap_and_run ./01_DirectorySetup.sh $*
 trap_and_run ./02_GetSpackStack.sh $*
-#trap_and_run ./03_SetupEnv.sh $*
-trap_and_run ./03_SetupUnifiedEnv.sh $*
+trap_and_run ./03_SetupEnv.sh $*
 trap_and_run ./04_SpackInstall.sh $*
 trap_and_run ./05_BuildCache.sh $*
 trap_and_run ./06_AppTests.sh $*
diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh
index ef19a4962..d84094f7b 100644
--- a/util/weekly_build/sites/acorn.sh
+++ b/util/weekly_build/sites/acorn.sh
@@ -1,5 +1,6 @@
 module load gcc/11.2.0 python/3.11.7
 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"}
+TEMPLATES=${TEMPLATES:-"unified-dev"}
 function spack_install_exe {
 #  set +e
 #  ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) &
diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh
index 20c2497ea..916b799a7 100644
--- a/util/weekly_build/sites/nautilus.sh
+++ b/util/weekly_build/sites/nautilus.sh
@@ -1,6 +1,6 @@
 COMPILERS=${COMPILERS:-"intel oneapi gcc"}
+TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"}
 module purge
 umask 0022
 SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack
 SPACK_STACK_BRANCH=feature/weekly_build_nautilus
-### NEEDED ? BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build-cache}
\ No newline at end of file

From 0cdcb52bbc790d1b192c6ca272b8dd1372f5c096 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 10:38:59 -0600
Subject: [PATCH 36/61] Update names of weekly build scripts in Github actions

---
 .github/actions/Nightly_02_GetSpackStack/action.yaml  | 11 +++++++++++
 .../Nightly_02_GetSpackStackDevelop/action.yaml       | 11 -----------
 .../action.yaml                                       |  4 ++--
 .github/workflows/nautilus-nightly-intel.yaml         |  8 ++++----
 .github/workflows/nautilus-nightly-oneapi.yaml        |  8 ++++----
 5 files changed, 21 insertions(+), 21 deletions(-)
 create mode 100644 .github/actions/Nightly_02_GetSpackStack/action.yaml
 delete mode 100644 .github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
 rename .github/actions/{Nightly_03_SetupUnifiedEnv => Nightly_03_SetupEnv}/action.yaml (70%)

diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml
new file mode 100644
index 000000000..19a558bcb
--- /dev/null
+++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 02 Get Spack-Stack
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 02 Get Spack-Stack
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      cd util/weekly_build
+      ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
deleted file mode 100644
index 7565046dd..000000000
--- a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-name: Nightly 02 Get Spack-Stack Develop
-
-runs:
-  using: "composite"
-  steps:
-  - name: Nightly 02 Get Spack-Stack Develop
-    shell: bash
-    run: |
-      RUNID=`date +"%Y%m%d"`
-      cd util/weekly_build
-      ./02_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml b/.github/actions/Nightly_03_SetupEnv/action.yaml
similarity index 70%
rename from .github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
rename to .github/actions/Nightly_03_SetupEnv/action.yaml
index 0f598cb88..48256bdc2 100644
--- a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml
+++ b/.github/actions/Nightly_03_SetupEnv/action.yaml
@@ -1,9 +1,9 @@
-name: Nightly 03 Setup Unified Env
+name: Nightly 03 Setup Env
 
 runs:
   using: "composite"
   steps:
-  - name: Nightly 03 Setup Unified Env
+  - name: Nightly 03 Setup Env
     shell: bash
     run: |
       RUNID=`date +"%Y%m%d"`
diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml
index 7dbf11bec..90767bb18 100644
--- a/.github/workflows/nautilus-nightly-intel.yaml
+++ b/.github/workflows/nautilus-nightly-intel.yaml
@@ -33,9 +33,9 @@ jobs:
         clean: true
     - name: 01_DirectorySetup
       uses: ./.github/actions/Nightly_01_DirectorySetup
-    - name: 02_GetSpackStackDevelop
-      uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
-    - name: 03_SetupUnifiedEnv
-      uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
+    - name: 02_GetSpackStack
+      uses: ./.github/actions/Nightly_02_GetSpackStack
+    - name: 03_SetupEnv
+      uses: ./.github/actions/Nightly_03_SetupEnv
     - name: 04_SpackInstall
       uses: ./.github/actions/Nightly_04_SpackInstall
diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
index df7b708a5..cd8823cb0 100644
--- a/.github/workflows/nautilus-nightly-oneapi.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -33,9 +33,9 @@ jobs:
         clean: true
     - name: 01_DirectorySetup
       uses: ./.github/actions/Nightly_01_DirectorySetup
-    - name: 02_GetSpackStackDevelop
-      uses: ./.github/actions/Nightly_02_GetSpackStackDevelop
-    - name: 03_SetupUnifiedEnv
-      uses: ./.github/actions/Nightly_03_SetupUnifiedEnv
+    - name: 02_GetSpackStack
+      uses: ./.github/actions/Nightly_02_GetSpackStack
+    - name: 03_SetupEnv
+      uses: ./.github/actions/Nightly_03_SetupEnv
     - name: 04_SpackInstall
       uses: ./.github/actions/Nightly_04_SpackInstall

From 30aa75825bf256b0137bc99e7c0af5f8c6cb8047 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 10:52:03 -0600
Subject: [PATCH 37/61] Set envname and envdir, only perform packages_to_test
 install step if variable defined

---
 util/weekly_build/03_SetupEnv.sh     | 18 ++++++-----
 util/weekly_build/04_SpackInstall.sh | 48 +++++++++++-----------------
 2 files changed, 29 insertions(+), 37 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 40704086d..d9fe5a13c 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -12,10 +12,12 @@ set -x
 
 for compiler in $COMPILERS; do
   for template in $TEMPLATES; do
-    echo "Setting up environment build-${compiler/@/-} using template $template"
-    rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-}
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/build-$envname
+    echo "Setting up environment $envname in $envdir"
+    rm -rf $envdir
     spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler
-    cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
+    cd $envdir
     spack env activate .
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
@@ -24,10 +26,10 @@ for compiler in $COMPILERS; do
     spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
 #    spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
-    # Get path to local-source mirror from spack site config
-    SOURCE_MIRROR=`spack mirror list | grep local-source`
-    SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//}
-    # Download all source codes needed to build the environment
-    spack mirror create -a -d ${SOURCE_MIRROR}
+###    # Get path to local-source mirror from spack site config
+###    SOURCE_MIRROR=`spack mirror list | grep local-source`
+###    SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//}
+###    # Download all source codes needed to build the environment
+###    spack mirror create -a -d ${SOURCE_MIRROR}
   done
 done
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index bcd2750ae..857d7f472 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -10,9 +10,6 @@ set +x
 . setup.sh
 set -x
 
-<<<<<<< HEAD
-INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS"
-=======
 if [ "$REUSE_BUILD_CACHE" == YES ]; then
   cache_flag="--no-check-signature"
 else
@@ -20,32 +17,25 @@ else
 fi
 
 INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS"
->>>>>>> 5068f19444eaa1564c66a6e9ff751bf7fac16be6
 
 for compiler in $COMPILERS; do
-  cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack env activate .
-<<<<<<< HEAD
-  #spack fetch # 2>&1 | tee log.fetch
-  #if [ -z "$PACKAGES_TO_INSTALL" ]; then
-  #  spack fetch --missing # 2>&1 | tee log.fetch
-  #else
-  #  spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch
-  #fi
-  # Just install the packages we're testing (+dependencies):
-  spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # 2>&1 | tee log.test.install
-  # Install the rest of the stack as usual:
-  spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL # 2>&1 | tee log.install
-=======
-  if [ "${SOURCE_CACHE::7}" == "file://" ]; then
-    mirrorpath=${SOURCE_CACHE}
-  else
-    mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
-  fi
-  spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
-  # Just install the packages we're testing (+dependencies):
-  spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
-  # Install the rest of the stack as usual:
-  spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL
->>>>>>> 5068f19444eaa1564c66a6e9ff751bf7fac16be6
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/build-$envname
+    echo "Building environment $envname in $envdir"
+    cd $envdir
+    spack env activate .
+    if [ "${SOURCE_CACHE::7}" == "file://" ]; then
+      mirrorpath=${SOURCE_CACHE}
+    else
+      mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
+    fi
+    spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
+    # Just install the packages we're testing (+dependencies):
+    if [[ -z "${PACKAGES_TO_TEST}" ]]; then
+      spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
+    fi
+    # Install the rest of the stack as usual:
+    spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL
+  done
 done

From d73425069d3616e36270ce434a369333f30863a3 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 10:59:14 -0600
Subject: [PATCH 38/61] Fix bug in env name definition in
 uti/weekly_build/{03,04}*

---
 util/weekly_build/03_SetupEnv.sh     | 2 +-
 util/weekly_build/04_SpackInstall.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index d9fe5a13c..9365a9b81 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -13,7 +13,7 @@ set -x
 for compiler in $COMPILERS; do
   for template in $TEMPLATES; do
     envname=build-$template-${compiler/@/-}
-    envdir=$RUNDIR/$RUNID/envs/build-$envname
+    envdir=$RUNDIR/$RUNID/envs/$envname
     echo "Setting up environment $envname in $envdir"
     rm -rf $envdir
     spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index 857d7f472..d83387453 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -21,7 +21,7 @@ INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS"
 for compiler in $COMPILERS; do
   for template in $TEMPLATES; do
     envname=build-$template-${compiler/@/-}
-    envdir=$RUNDIR/$RUNID/envs/build-$envname
+    envdir=$RUNDIR/$RUNID/envs/$envname
     echo "Building environment $envname in $envdir"
     cd $envdir
     spack env activate .

From a474cb74052717ef562c8d40e482abb8eed1da5d Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 11:29:36 -0600
Subject: [PATCH 39/61] Strip down CI testing to neptune-dev w/o Python for
 faster development and debugging

---
 configs/templates/neptune-dev/spack.yaml | 2 +-
 util/weekly_build/sites/nautilus.sh      | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml
index 9a9e3dfec..75dfeddb4 100644
--- a/configs/templates/neptune-dev/spack.yaml
+++ b/configs/templates/neptune-dev/spack.yaml
@@ -9,7 +9,7 @@ spack:
   - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
   - packages:
       - neptune-env +espc        ^esmf@8.7.0
-      - neptune-python-env +xnrl ^esmf@8.7.0
+      #- neptune-python-env +xnrl ^esmf@8.7.0
 
   specs:
     - matrix:
diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh
index 916b799a7..9cd0dabd7 100644
--- a/util/weekly_build/sites/nautilus.sh
+++ b/util/weekly_build/sites/nautilus.sh
@@ -1,5 +1,6 @@
 COMPILERS=${COMPILERS:-"intel oneapi gcc"}
-TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"}
+#TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"}
+TEMPLATES=${TEMPLATES:-"neptune-dev"}
 module purge
 umask 0022
 SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack

From a803a546784c27684c7753dbcbf47edc927641e2 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 16:04:18 -0600
Subject: [PATCH 40/61] Add steps 05_BuildCache and 07_Cleanup to Github
 actions, run only for OneAPI for testing

---
 .github/actions/Nightly_05_BuildCache/action.yaml    | 11 +++++++++++
 .github/actions/Nightly_07_Cleanup/action.yaml       | 11 +++++++++++
 ...ly-intel.yaml => nautilus-nightly-intel.yaml.tmp} |  2 +-
 .github/workflows/nautilus-nightly-oneapi.yaml       |  7 ++++++-
 util/weekly_build/05_BuildCache.sh                   | 12 ++++++++----
 util/weekly_build/sites/nautilus.sh                  |  2 ++
 6 files changed, 39 insertions(+), 6 deletions(-)
 create mode 100644 .github/actions/Nightly_05_BuildCache/action.yaml
 create mode 100644 .github/actions/Nightly_07_Cleanup/action.yaml
 rename .github/workflows/{nautilus-nightly-intel.yaml => nautilus-nightly-intel.yaml.tmp} (94%)

diff --git a/.github/actions/Nightly_05_BuildCache/action.yaml b/.github/actions/Nightly_05_BuildCache/action.yaml
new file mode 100644
index 000000000..af064d659
--- /dev/null
+++ b/.github/actions/Nightly_05_BuildCache/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 05 Build Cache
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 05 Build Cache
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      cd util/weekly_build
+      ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_07_Cleanup/action.yaml b/.github/actions/Nightly_07_Cleanup/action.yaml
new file mode 100644
index 000000000..70f27c4f1
--- /dev/null
+++ b/.github/actions/Nightly_07_Cleanup/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 07 Cleanup
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 07 Cleanup
+    shell: bash
+    run: |
+      RUNID=`date +"%Y%m%d"`
+      cd util/weekly_build
+      ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml.tmp
similarity index 94%
rename from .github/workflows/nautilus-nightly-intel.yaml
rename to .github/workflows/nautilus-nightly-intel.yaml.tmp
index 90767bb18..b6ffd15fa 100644
--- a/.github/workflows/nautilus-nightly-intel.yaml
+++ b/.github/workflows/nautilus-nightly-intel.yaml.tmp
@@ -16,7 +16,7 @@ defaults:
     shell: bash
 
 env:
-  BASEDIR: /p/work2/heinzell/spack-stack-nightly
+  BASEDIR: /p/work2/heinzell/spack-stack-nightly/intel
   PLATFORM: nautilus
   COMPILERS: intel
 
diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
index cd8823cb0..c5d689444 100644
--- a/.github/workflows/nautilus-nightly-oneapi.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -16,7 +16,7 @@ defaults:
     shell: bash
 
 env:
-  BASEDIR: /p/work2/heinzell/spack-stack-nightly
+  BASEDIR: /p/work2/heinzell/spack-stack-nightly/oneapi
   PLATFORM: nautilus
   COMPILERS: oneapi
 
@@ -39,3 +39,8 @@ jobs:
       uses: ./.github/actions/Nightly_03_SetupEnv
     - name: 04_SpackInstall
       uses: ./.github/actions/Nightly_04_SpackInstall
+    - name: 05_BuildCache
+      uses: ./.github/actions/Nightly_05_BuildCache
+    # Skip step 6 - app test - for the moment
+    - name: 07_Cleanup
+      uses: ./.github/actions/Nightly_07_Cleanup
diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh
index 4ce53cba7..2799c0464 100755
--- a/util/weekly_build/05_BuildCache.sh
+++ b/util/weekly_build/05_BuildCache.sh
@@ -11,8 +11,12 @@ set +x
 set -x
 
 for compiler in $COMPILERS; do
-  cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack env activate .
-  spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL
-  spack buildcache rebuild-index ${BUILD_CACHE}
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    cd $envdir
+    spack env activate .
+    spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL
+    spack buildcache rebuild-index ${BUILD_CACHE}
+  done
 done
diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh
index 9cd0dabd7..5f76087d1 100644
--- a/util/weekly_build/sites/nautilus.sh
+++ b/util/weekly_build/sites/nautilus.sh
@@ -3,5 +3,7 @@ COMPILERS=${COMPILERS:-"intel oneapi gcc"}
 TEMPLATES=${TEMPLATES:-"neptune-dev"}
 module purge
 umask 0022
+
 SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack
 SPACK_STACK_BRANCH=feature/weekly_build_nautilus
+KEEP_WEEKLY_BUILD_DIR="YES"
\ No newline at end of file

From 4b127283dfc12411259d69eb25e76499e4fba9ad Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 16:04:57 -0600
Subject: [PATCH 41/61] Bug fix for wrong source cache for Narwhal and
 Nautilus, addition of binary cache for Atlantis, Narwhal, Nautilus

---
 configs/sites/tier1/atlantis/mirrors.yaml | 19 ++-----------------
 configs/sites/tier1/narwhal/mirrors.yaml  | 19 ++-----------------
 configs/sites/tier1/nautilus/mirrors.yaml | 19 ++-----------------
 3 files changed, 6 insertions(+), 51 deletions(-)

diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml
index 4d363e009..d2371749e 100644
--- a/configs/sites/tier1/atlantis/mirrors.yaml
+++ b/configs/sites/tier1/atlantis/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///neptune_diagnostics/spack-stack/source-cache/
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///neptune_diagnostics/spack-stack/source-cache/
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///neptune_diagnostics/spack-stack/source-cache
+  local-binary: file:///neptune_diagnostics/spack-stack/build-cache
diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml
index ec3db4a00..5b06fc3fb 100644
--- a/configs/sites/tier1/narwhal/mirrors.yaml
+++ b/configs/sites/tier1/narwhal/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
+  local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml
index ec3db4a00..5b06fc3fb 100644
--- a/configs/sites/tier1/nautilus/mirrors.yaml
+++ b/configs/sites/tier1/nautilus/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
+  local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache

From 7b17952994f8d4b0c2a2328249ea0e153edf4c24 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 16:51:21 -0600
Subject: [PATCH 42/61] In
 util/weekly_build/{03_SetupEnv.sh,04_SpackInstall.sh} pipe spack concretizer
 and installer output to logfile, but don't redirect stderr

---
 util/weekly_build/03_SetupEnv.sh     | 9 ++-------
 util/weekly_build/04_SpackInstall.sh | 4 ++--
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 9365a9b81..5ad8a69c6 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -21,15 +21,10 @@ for compiler in $COMPILERS; do
     spack env activate .
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
-    spack concretize --fresh 2>&1 | tee log.concretize
+    spack concretize --fresh | tee log.concretize
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
     spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
-#    spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
-###    # Get path to local-source mirror from spack site config
-###    SOURCE_MIRROR=`spack mirror list | grep local-source`
-###    SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//}
-###    # Download all source codes needed to build the environment
-###    spack mirror create -a -d ${SOURCE_MIRROR}
+#    spack concretize --force --fresh --test all | tee log.concretize_test
   done
 done
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index d83387453..c2d698d95 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -33,9 +33,9 @@ for compiler in $COMPILERS; do
     spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
     # Just install the packages we're testing (+dependencies):
     if [[ -z "${PACKAGES_TO_TEST}" ]]; then
-      spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
+      spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test
     fi
     # Install the rest of the stack as usual:
-    spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL
+    spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install
   done
 done

From 9ca24073e05e33e99a116abf790b69af6f001b15 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 21 Oct 2024 16:51:54 -0600
Subject: [PATCH 43/61] Add logic to
 .github/actions/Nightly_04_SpackInstall/action.yaml to reuse existing binary
 cache for spack-stack installs on all days except Sundays

---
 .github/actions/Nightly_04_SpackInstall/action.yaml | 9 +++++++++
 .github/workflows/nautilus-nightly-oneapi.yaml      | 3 +++
 2 files changed, 12 insertions(+)

diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
index 0954971b1..bd2ab560d 100644
--- a/.github/actions/Nightly_04_SpackInstall/action.yaml
+++ b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -6,6 +6,15 @@ runs:
   - name: Nightly 04 Spack Install
     shell: bash
     run: |
+      # Get day of week - build everything from scratch on Sundays,
+      # otherwise use build cache to speed up the installations
+      DOW=$(date +%u)
+      # Monday is 1 ... Sunday is 7
+      if [[ $DOW == 7 ]]; then
+        export REUSE_BUILD_CACHE="NO"
+      else
+        export REUSE_BUILD_CACHE="YES"
+      fi
       RUNID=`date +"%Y%m%d"`
       cd util/weekly_build
       ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
index c5d689444..4459ceb4d 100644
--- a/.github/workflows/nautilus-nightly-oneapi.yaml
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -6,6 +6,9 @@ on:
     branches:
       - develop
     types: [opened, synchronize, reopened]
+  # Is this UTC?
+  #schedule:
+  #  - cron: '0 8 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

From 4c906cf767992b42431802913acb270f35b55825 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 05:49:35 -0600
Subject: [PATCH 44/61] Write RUNID and DOW to file in step
 Nightly_01_DirectorySetup, retrieve from files in other steps

---
 .../actions/Nightly_01_DirectorySetup/action.yaml   | 13 ++++++++-----
 .../actions/Nightly_02_GetSpackStack/action.yaml    |  4 +---
 .github/actions/Nightly_03_SetupEnv/action.yaml     |  2 +-
 .github/actions/Nightly_04_SpackInstall/action.yaml |  8 ++++----
 .github/actions/Nightly_05_BuildCache/action.yaml   |  2 +-
 .github/actions/Nightly_07_Cleanup/action.yaml      |  2 +-
 6 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
index 4d1a614e9..2660a3745 100644
--- a/.github/actions/Nightly_01_DirectorySetup/action.yaml
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -6,13 +6,16 @@ runs:
   - name: Nightly 01 Directory Setup
     shell: bash
     run: |
-      RUNID=`date +"%Y%m%d"`
       # Write RUNID to file so that following steps use the
       # same date and don't pick up the next day's date
+      RUNID=`date +"%Y%m%d"`
       echo "${RUNID}" > RUNID_SAVE.log
-      # DH* TODO REMOVE LATER?
-      RUNDIR=${BASEDIR}/${RUNID}
-      rm -vfr ${RUNDIR}
-      # *DH
+      # Get day of week for later use and write to file
+      DOW=$(date +%u)
+      echo "${DOW}" > DOW_SAVE.log
+      ## DH* TODO REMOVE LATER?
+      #RUNDIR=${BASEDIR}/${RUNID}
+      #rm -vfr ${RUNDIR}
+      ## *DH
       cd util/weekly_build
       ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml
index 49fbf2512..7377bfb88 100644
--- a/.github/actions/Nightly_02_GetSpackStack/action.yaml
+++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml
@@ -6,9 +6,7 @@ runs:
   - name: Nightly 02 Get Spack-Stack
     shell: bash
     run: |
-      #RUNID=`date +"%Y%m%d"`
       RUNID=$(<RUNID_SAVE.log)
-      echo "RUNID? '${RUNID}'"
-      exit 1
+      echo "Got RUNID from file: '${RUNID}'"
       cd util/weekly_build
       ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_03_SetupEnv/action.yaml b/.github/actions/Nightly_03_SetupEnv/action.yaml
index 48256bdc2..ef44b9e80 100644
--- a/.github/actions/Nightly_03_SetupEnv/action.yaml
+++ b/.github/actions/Nightly_03_SetupEnv/action.yaml
@@ -6,6 +6,6 @@ runs:
   - name: Nightly 03 Setup Env
     shell: bash
     run: |
-      RUNID=`date +"%Y%m%d"`
+      RUNID=$(<RUNID_SAVE.log)
       cd util/weekly_build
       ./03_SetupEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
index bd2ab560d..3d86123e1 100644
--- a/.github/actions/Nightly_04_SpackInstall/action.yaml
+++ b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -6,15 +6,15 @@ runs:
   - name: Nightly 04 Spack Install
     shell: bash
     run: |
-      # Get day of week - build everything from scratch on Sundays,
-      # otherwise use build cache to speed up the installations
-      DOW=$(date +%u)
+      RUNID=$(<RUNID_SAVE.log)
+      DOW=$(<DOW_SAVE.log)
+      # Bbuild everything from scratch on Sundays, otherwise
+      # use build cache to speed up the installations.
       # Monday is 1 ... Sunday is 7
       if [[ $DOW == 7 ]]; then
         export REUSE_BUILD_CACHE="NO"
       else
         export REUSE_BUILD_CACHE="YES"
       fi
-      RUNID=`date +"%Y%m%d"`
       cd util/weekly_build
       ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_05_BuildCache/action.yaml b/.github/actions/Nightly_05_BuildCache/action.yaml
index af064d659..b73505c37 100644
--- a/.github/actions/Nightly_05_BuildCache/action.yaml
+++ b/.github/actions/Nightly_05_BuildCache/action.yaml
@@ -6,6 +6,6 @@ runs:
   - name: Nightly 05 Build Cache
     shell: bash
     run: |
-      RUNID=`date +"%Y%m%d"`
+      RUNID=$(<RUNID_SAVE.log)
       cd util/weekly_build
       ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_07_Cleanup/action.yaml b/.github/actions/Nightly_07_Cleanup/action.yaml
index 70f27c4f1..53fee8000 100644
--- a/.github/actions/Nightly_07_Cleanup/action.yaml
+++ b/.github/actions/Nightly_07_Cleanup/action.yaml
@@ -6,6 +6,6 @@ runs:
   - name: Nightly 07 Cleanup
     shell: bash
     run: |
-      RUNID=`date +"%Y%m%d"`
+      RUNID=$(<RUNID_SAVE.log)
       cd util/weekly_build
       ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}

From fe40fafe82229a74c724af992cd93feecd87957e Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 07:09:47 -0600
Subject: [PATCH 45/61] Bug fix in util/weekly_build/04_SpackInstall.sh, add
 missing NOT (!) in if test for PACKAGES_TO_TEST

---
 util/weekly_build/04_SpackInstall.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index c2d698d95..eaac57dfd 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -32,7 +32,7 @@ for compiler in $COMPILERS; do
     fi
     spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
     # Just install the packages we're testing (+dependencies):
-    if [[ -z "${PACKAGES_TO_TEST}" ]]; then
+    if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then
       spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test
     fi
     # Install the rest of the stack as usual:

From b59c4d3895c18d4342b80e700e92dce4eeaaf4dc Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 09:23:04 -0600
Subject: [PATCH 46/61] [skip ci] Delete empty configs/common/packages (#1354)

This file got merged mistakenly, it doesn't have a `.yaml` suffix and is empty.
---
 configs/common/packages | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 configs/common/packages

diff --git a/configs/common/packages b/configs/common/packages
deleted file mode 100644
index e69de29bb..000000000

From c32cb03c370d3727f11571d289cbf67ec5f5ba6f Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 09:30:34 -0600
Subject: [PATCH 47/61] Update util/weekly_build/sites/{acorn.sh,nautilus.sh}
 and delete currently unused sites

---
 util/weekly_build/sites/acorn.sh         | 1 +
 util/weekly_build/sites/derecho.sh       | 1 -
 util/weekly_build/sites/discover16.sh    | 1 -
 util/weekly_build/sites/discover17.sh    | 1 -
 util/weekly_build/sites/gaea.sh          | 1 -
 util/weekly_build/sites/hera.sh          | 1 -
 util/weekly_build/sites/hercules.sh      | 1 -
 util/weekly_build/sites/jet.sh           | 1 -
 util/weekly_build/sites/linux.default.sh | 1 -
 util/weekly_build/sites/narwhal.sh       | 1 -
 util/weekly_build/sites/nautilus.sh      | 9 ++++++++-
 util/weekly_build/sites/orion.sh         | 1 -
 util/weekly_build/sites/s4.sh            | 1 -
 13 files changed, 9 insertions(+), 12 deletions(-)
 delete mode 100644 util/weekly_build/sites/derecho.sh
 delete mode 100644 util/weekly_build/sites/discover16.sh
 delete mode 100644 util/weekly_build/sites/discover17.sh
 delete mode 100644 util/weekly_build/sites/gaea.sh
 delete mode 100644 util/weekly_build/sites/hera.sh
 delete mode 100644 util/weekly_build/sites/hercules.sh
 delete mode 100644 util/weekly_build/sites/jet.sh
 delete mode 100644 util/weekly_build/sites/linux.default.sh
 delete mode 100644 util/weekly_build/sites/narwhal.sh
 delete mode 100644 util/weekly_build/sites/orion.sh
 delete mode 100644 util/weekly_build/sites/s4.sh

diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh
index ef19a4962..d84094f7b 100644
--- a/util/weekly_build/sites/acorn.sh
+++ b/util/weekly_build/sites/acorn.sh
@@ -1,5 +1,6 @@
 module load gcc/11.2.0 python/3.11.7
 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"}
+TEMPLATES=${TEMPLATES:-"unified-dev"}
 function spack_install_exe {
 #  set +e
 #  ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) &
diff --git a/util/weekly_build/sites/derecho.sh b/util/weekly_build/sites/derecho.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/derecho.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/discover16.sh b/util/weekly_build/sites/discover16.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/discover16.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/discover17.sh b/util/weekly_build/sites/discover17.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/discover17.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/gaea.sh b/util/weekly_build/sites/gaea.sh
deleted file mode 100644
index fe7e69664..000000000
--- a/util/weekly_build/sites/gaea.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel"}
diff --git a/util/weekly_build/sites/hera.sh b/util/weekly_build/sites/hera.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/hera.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/hercules.sh b/util/weekly_build/sites/hercules.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/hercules.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/jet.sh b/util/weekly_build/sites/jet.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/jet.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/linux.default.sh b/util/weekly_build/sites/linux.default.sh
deleted file mode 100644
index c044afced..000000000
--- a/util/weekly_build/sites/linux.default.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"gcc"}
diff --git a/util/weekly_build/sites/narwhal.sh b/util/weekly_build/sites/narwhal.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/narwhal.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh
index fe7e69664..fddcfd62f 100644
--- a/util/weekly_build/sites/nautilus.sh
+++ b/util/weekly_build/sites/nautilus.sh
@@ -1 +1,8 @@
-COMPILERS=${COMPILERS:-"intel"}
+COMPILERS=${COMPILERS:-"intel oneapi gcc"}
+TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"}
+module purge
+umask 0022
+
+SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack
+SPACK_STACK_BRANCH=feature/weekly_build_nautilus
+KEEP_WEEKLY_BUILD_DIR="YES"
diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/orion.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/s4.sh b/util/weekly_build/sites/s4.sh
deleted file mode 100644
index fe7e69664..000000000
--- a/util/weekly_build/sites/s4.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel"}

From 31d378d71a15fc9f87b26df33c5498d056090ccc Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 09:30:51 -0600
Subject: [PATCH 48/61] Update mirrors for NRL machines in configs/sites/tier1

---
 configs/sites/tier1/atlantis/mirrors.yaml | 19 ++-----------------
 configs/sites/tier1/narwhal/mirrors.yaml  | 19 ++-----------------
 configs/sites/tier1/nautilus/mirrors.yaml | 19 ++-----------------
 3 files changed, 6 insertions(+), 51 deletions(-)

diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml
index 4d363e009..d2371749e 100644
--- a/configs/sites/tier1/atlantis/mirrors.yaml
+++ b/configs/sites/tier1/atlantis/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///neptune_diagnostics/spack-stack/source-cache/
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///neptune_diagnostics/spack-stack/source-cache/
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///neptune_diagnostics/spack-stack/source-cache
+  local-binary: file:///neptune_diagnostics/spack-stack/build-cache
diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml
index 2b67bea03..5b06fc3fb 100644
--- a/configs/sites/tier1/narwhal/mirrors.yaml
+++ b/configs/sites/tier1/narwhal/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
+  local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml
index 2b67bea03..5b06fc3fb 100644
--- a/configs/sites/tier1/nautilus/mirrors.yaml
+++ b/configs/sites/tier1/nautilus/mirrors.yaml
@@ -1,18 +1,3 @@
 mirrors:
-  local-source:
-    fetch:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
-    push:
-      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
-      access_pair:
-      - null
-      - null
-      access_token: null
-      profile: null
-      endpoint_url: null
+  local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache
+  local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache

From 43ab602ab05d06c91d143f3666bf76f741db8364 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 09:31:42 -0600
Subject: [PATCH 49/61] Update util/weekly_build scripts to handle multiple
 environments plus other updates

---
 util/weekly_build/03_SetupEnv.sh              | 30 ++++++++++++++++++
 util/weekly_build/03_SetupUnifiedEnv.sh       | 24 --------------
 util/weekly_build/04_SpackInstall.sh          | 31 ++++++++++++-------
 util/weekly_build/05_BuildCache.sh            | 12 ++++---
 util/weekly_build/ShellSetup.sh               |  6 ++--
 .../SpackStackBuildCache_AllSteps.sh          |  2 +-
 6 files changed, 62 insertions(+), 43 deletions(-)
 create mode 100755 util/weekly_build/03_SetupEnv.sh
 delete mode 100755 util/weekly_build/03_SetupUnifiedEnv.sh

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
new file mode 100755
index 000000000..5ad8a69c6
--- /dev/null
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
+
+set -ex
+
+cd $RUNDIR/$RUNID
+
+set +x
+. setup.sh
+set -x
+
+for compiler in $COMPILERS; do
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    echo "Setting up environment $envname in $envdir"
+    rm -rf $envdir
+    spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler
+    cd $envdir
+    spack env activate .
+    spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
+    # Check for duplicates and fail before doing the "real" concretization:
+    spack concretize --fresh | tee log.concretize
+    ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
+    spack concretize --fresh --force
+#   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
+#    spack concretize --force --fresh --test all | tee log.concretize_test
+  done
+done
diff --git a/util/weekly_build/03_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupUnifiedEnv.sh
deleted file mode 100755
index 3479ed7ec..000000000
--- a/util/weekly_build/03_SetupUnifiedEnv.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi
-
-set -ex
-
-cd $RUNDIR/$RUNID
-
-set +x
-. setup.sh
-set -x
-
-for compiler in $COMPILERS; do
-  rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler
-  cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack env activate .
-  spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
-  # Check for duplicates and fail before doing the "real" concretization with test deps:
-  spack concretize --fresh 2>&1 | tee log.concretize
-  ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
-# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
-#  spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test
-done
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index a5bb8cc70..eaac57dfd 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -19,16 +19,23 @@ fi
 INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS"
 
 for compiler in $COMPILERS; do
-  cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack env activate .
-  if [ "${SOURCE_CACHE::7}" == "file://" ]; then
-    mirrorpath=${SOURCE_CACHE}
-  else
-    mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
-  fi
-  spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
-  # Just install the packages we're testing (+dependencies):
-  spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
-  # Install the rest of the stack as usual:
-  spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    echo "Building environment $envname in $envdir"
+    cd $envdir
+    spack env activate .
+    if [ "${SOURCE_CACHE::7}" == "file://" ]; then
+      mirrorpath=${SOURCE_CACHE}
+    else
+      mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
+    fi
+    spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
+    # Just install the packages we're testing (+dependencies):
+    if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then
+      spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test
+    fi
+    # Install the rest of the stack as usual:
+    spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install
+  done
 done
diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh
index 4ce53cba7..2799c0464 100755
--- a/util/weekly_build/05_BuildCache.sh
+++ b/util/weekly_build/05_BuildCache.sh
@@ -11,8 +11,12 @@ set +x
 set -x
 
 for compiler in $COMPILERS; do
-  cd $RUNDIR/$RUNID/envs/build-${compiler/@/-}
-  spack env activate .
-  spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL
-  spack buildcache rebuild-index ${BUILD_CACHE}
+  for template in $TEMPLATES; do
+    envname=build-$template-${compiler/@/-}
+    envdir=$RUNDIR/$RUNID/envs/$envname
+    cd $envdir
+    spack env activate .
+    spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL
+    spack buildcache rebuild-index ${BUILD_CACHE}
+  done
 done
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 31e7f2e8e..e5f25608a 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -11,16 +11,18 @@ if [ ${RUNDIR::1} != "/" ]; then
   exit 1
 fi
 
-PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"}
+#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"}
 
 function alert_cmd {
   echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system."
 }
 
 function spack_install_exe {
-  spack $* | tee -a log.install 2>&1
+  #spack $* | tee -a log.install 2>&1
+  spack $*
 }
 
+# Include platform-dependent configuration
 . $(dirname $0)/sites/${PLATFORM}.sh
 
 SOURCE_CACHE=${SOURCE_CACHE:-local-source}
diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
index 4d26df07d..73a7931c4 100755
--- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh
+++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh
@@ -18,7 +18,7 @@ function trap_and_run {
 
 trap_and_run ./01_DirectorySetup.sh $*
 trap_and_run ./02_GetSpackStack.sh $*
-trap_and_run ./03_SetupUnifiedEnv.sh $*
+trap_and_run ./03_SetupEnv.sh $*
 trap_and_run ./04_SpackInstall.sh $*
 trap_and_run ./05_BuildCache.sh $*
 trap_and_run ./06_AppTests.sh $*

From f09b991b3a4148b08cd916cde1d03aef74578a39 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 09:32:13 -0600
Subject: [PATCH 50/61] Add .github/actions/Nightly_* and
 .github/workflows/nautilus-nightly-oneapi.yaml

---
 .../Nightly_01_DirectorySetup/action.yaml     | 21 ++++++++
 .../Nightly_02_GetSpackStack/action.yaml      | 12 +++++
 .../actions/Nightly_03_SetupEnv/action.yaml   | 11 +++++
 .../Nightly_04_SpackInstall/action.yaml       | 20 ++++++++
 .../actions/Nightly_05_BuildCache/action.yaml | 11 +++++
 .../actions/Nightly_07_Cleanup/action.yaml    | 11 +++++
 .../workflows/nautilus-nightly-oneapi.yaml    | 49 +++++++++++++++++++
 7 files changed, 135 insertions(+)
 create mode 100644 .github/actions/Nightly_01_DirectorySetup/action.yaml
 create mode 100644 .github/actions/Nightly_02_GetSpackStack/action.yaml
 create mode 100644 .github/actions/Nightly_03_SetupEnv/action.yaml
 create mode 100644 .github/actions/Nightly_04_SpackInstall/action.yaml
 create mode 100644 .github/actions/Nightly_05_BuildCache/action.yaml
 create mode 100644 .github/actions/Nightly_07_Cleanup/action.yaml
 create mode 100644 .github/workflows/nautilus-nightly-oneapi.yaml

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
new file mode 100644
index 000000000..2660a3745
--- /dev/null
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -0,0 +1,21 @@
+name: Nightly 01 Directory Setup
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 01 Directory Setup
+    shell: bash
+    run: |
+      # Write RUNID to file so that following steps use the
+      # same date and don't pick up the next day's date
+      RUNID=`date +"%Y%m%d"`
+      echo "${RUNID}" > RUNID_SAVE.log
+      # Get day of week for later use and write to file
+      DOW=$(date +%u)
+      echo "${DOW}" > DOW_SAVE.log
+      ## DH* TODO REMOVE LATER?
+      #RUNDIR=${BASEDIR}/${RUNID}
+      #rm -vfr ${RUNDIR}
+      ## *DH
+      cd util/weekly_build
+      ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml
new file mode 100644
index 000000000..7377bfb88
--- /dev/null
+++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml
@@ -0,0 +1,12 @@
+name: Nightly 02 Get Spack-Stack
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 02 Get Spack-Stack
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      echo "Got RUNID from file: '${RUNID}'"
+      cd util/weekly_build
+      ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_03_SetupEnv/action.yaml b/.github/actions/Nightly_03_SetupEnv/action.yaml
new file mode 100644
index 000000000..ef44b9e80
--- /dev/null
+++ b/.github/actions/Nightly_03_SetupEnv/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 03 Setup Env
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 03 Setup Env
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./03_SetupEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
new file mode 100644
index 000000000..3d86123e1
--- /dev/null
+++ b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -0,0 +1,20 @@
+name: Nightly 04 Spack Install
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 04 Spack Install
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      DOW=$(<DOW_SAVE.log)
+      # Bbuild everything from scratch on Sundays, otherwise
+      # use build cache to speed up the installations.
+      # Monday is 1 ... Sunday is 7
+      if [[ $DOW == 7 ]]; then
+        export REUSE_BUILD_CACHE="NO"
+      else
+        export REUSE_BUILD_CACHE="YES"
+      fi
+      cd util/weekly_build
+      ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_05_BuildCache/action.yaml b/.github/actions/Nightly_05_BuildCache/action.yaml
new file mode 100644
index 000000000..b73505c37
--- /dev/null
+++ b/.github/actions/Nightly_05_BuildCache/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 05 Build Cache
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 05 Build Cache
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_07_Cleanup/action.yaml b/.github/actions/Nightly_07_Cleanup/action.yaml
new file mode 100644
index 000000000..53fee8000
--- /dev/null
+++ b/.github/actions/Nightly_07_Cleanup/action.yaml
@@ -0,0 +1,11 @@
+name: Nightly 07 Cleanup
+
+runs:
+  using: "composite"
+  steps:
+  - name: Nightly 07 Cleanup
+    shell: bash
+    run: |
+      RUNID=$(<RUNID_SAVE.log)
+      cd util/weekly_build
+      ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml
new file mode 100644
index 000000000..4459ceb4d
--- /dev/null
+++ b/.github/workflows/nautilus-nightly-oneapi.yaml
@@ -0,0 +1,49 @@
+name: nautilus unified-env oneapi
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - develop
+    types: [opened, synchronize, reopened]
+  # Is this UTC?
+  #schedule:
+  #  - cron: '0 8 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BASEDIR: /p/work2/heinzell/spack-stack-nightly/oneapi
+  PLATFORM: nautilus
+  COMPILERS: oneapi
+
+jobs:
+
+  nightly-build-oneapi:
+    runs-on: ['self-hosted','nautilus-heinzell']
+    # Set timeout to 12hrs
+    timeout-minutes: 720
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        clean: true
+    - name: 01_DirectorySetup
+      uses: ./.github/actions/Nightly_01_DirectorySetup
+    - name: 02_GetSpackStack
+      uses: ./.github/actions/Nightly_02_GetSpackStack
+    - name: 03_SetupEnv
+      uses: ./.github/actions/Nightly_03_SetupEnv
+    - name: 04_SpackInstall
+      uses: ./.github/actions/Nightly_04_SpackInstall
+    - name: 05_BuildCache
+      uses: ./.github/actions/Nightly_05_BuildCache
+    # Skip step 6 - app test - for the moment
+    - name: 07_Cleanup
+      uses: ./.github/actions/Nightly_07_Cleanup

From e4ded648b04a66d23e30d88fc91c5789d2eadfb6 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 14:55:59 -0600
Subject: [PATCH 51/61] Clean up .github/actions/**

---
 .github/actions/Nightly_01_DirectorySetup/action.yaml | 5 +----
 .github/actions/Nightly_02_GetSpackStack/action.yaml  | 1 -
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
index 2660a3745..5bcf5051c 100644
--- a/.github/actions/Nightly_01_DirectorySetup/action.yaml
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -13,9 +13,6 @@ runs:
       # Get day of week for later use and write to file
       DOW=$(date +%u)
       echo "${DOW}" > DOW_SAVE.log
-      ## DH* TODO REMOVE LATER?
-      #RUNDIR=${BASEDIR}/${RUNID}
-      #rm -vfr ${RUNDIR}
-      ## *DH
+      #
       cd util/weekly_build
       ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml
index 7377bfb88..268adaf29 100644
--- a/.github/actions/Nightly_02_GetSpackStack/action.yaml
+++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml
@@ -7,6 +7,5 @@ runs:
     shell: bash
     run: |
       RUNID=$(<RUNID_SAVE.log)
-      echo "Got RUNID from file: '${RUNID}'"
       cd util/weekly_build
       ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM}

From 3da12df16dd84d651ab6479de75142388a052e59 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 14:57:28 -0600
Subject: [PATCH 52/61] Remove sites that are not in use from
 util/weekly_build/sites/

---
 configs/common/packages                  | 0
 util/weekly_build/sites/derecho.sh       | 1 -
 util/weekly_build/sites/discover16.sh    | 1 -
 util/weekly_build/sites/discover17.sh    | 1 -
 util/weekly_build/sites/gaea.sh          | 1 -
 util/weekly_build/sites/hera.sh          | 1 -
 util/weekly_build/sites/hercules.sh      | 1 -
 util/weekly_build/sites/jet.sh           | 1 -
 util/weekly_build/sites/linux.default.sh | 1 -
 util/weekly_build/sites/narwhal.sh       | 1 -
 util/weekly_build/sites/orion.sh         | 1 -
 util/weekly_build/sites/s4.sh            | 1 -
 12 files changed, 11 deletions(-)
 delete mode 100644 configs/common/packages
 delete mode 100644 util/weekly_build/sites/derecho.sh
 delete mode 100644 util/weekly_build/sites/discover16.sh
 delete mode 100644 util/weekly_build/sites/discover17.sh
 delete mode 100644 util/weekly_build/sites/gaea.sh
 delete mode 100644 util/weekly_build/sites/hera.sh
 delete mode 100644 util/weekly_build/sites/hercules.sh
 delete mode 100644 util/weekly_build/sites/jet.sh
 delete mode 100644 util/weekly_build/sites/linux.default.sh
 delete mode 100644 util/weekly_build/sites/narwhal.sh
 delete mode 100644 util/weekly_build/sites/orion.sh
 delete mode 100644 util/weekly_build/sites/s4.sh

diff --git a/configs/common/packages b/configs/common/packages
deleted file mode 100644
index e69de29bb..000000000
diff --git a/util/weekly_build/sites/derecho.sh b/util/weekly_build/sites/derecho.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/derecho.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/discover16.sh b/util/weekly_build/sites/discover16.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/discover16.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/discover17.sh b/util/weekly_build/sites/discover17.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/discover17.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/gaea.sh b/util/weekly_build/sites/gaea.sh
deleted file mode 100644
index fe7e69664..000000000
--- a/util/weekly_build/sites/gaea.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel"}
diff --git a/util/weekly_build/sites/hera.sh b/util/weekly_build/sites/hera.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/hera.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/hercules.sh b/util/weekly_build/sites/hercules.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/hercules.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/jet.sh b/util/weekly_build/sites/jet.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/jet.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/linux.default.sh b/util/weekly_build/sites/linux.default.sh
deleted file mode 100644
index c044afced..000000000
--- a/util/weekly_build/sites/linux.default.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"gcc"}
diff --git a/util/weekly_build/sites/narwhal.sh b/util/weekly_build/sites/narwhal.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/narwhal.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh
deleted file mode 100644
index 1e7dc8450..000000000
--- a/util/weekly_build/sites/orion.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel gcc"}
diff --git a/util/weekly_build/sites/s4.sh b/util/weekly_build/sites/s4.sh
deleted file mode 100644
index fe7e69664..000000000
--- a/util/weekly_build/sites/s4.sh
+++ /dev/null
@@ -1 +0,0 @@
-COMPILERS=${COMPILERS:-"intel"}

From edab0f563178a0c4342e91bc3b3d42013db647e8 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 14:57:53 -0600
Subject: [PATCH 53/61] Add rogue cmd line argument for spack concretize for
 debugging in util/weekly_build/03_SetupEnv.sh

---
 util/weekly_build/03_SetupEnv.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 5ad8a69c6..9a1991046 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -21,7 +21,7 @@ for compiler in $COMPILERS; do
     spack env activate .
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
-    spack concretize --fresh | tee log.concretize
+    spack concretize --fresh --somethingnotsupported > log.concretize 2>&1
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
     spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.

From 8c895401c7946c3bf5168df83d11a609aa19f56a Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 15:58:27 -0600
Subject: [PATCH 54/61] Try spack_wrapper

---
 util/weekly_build/03_SetupEnv.sh     | 6 +++++-
 util/weekly_build/04_SpackInstall.sh | 8 +++++---
 util/weekly_build/ShellSetup.sh      | 7 +++++++
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 9a1991046..b8810cb7c 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -21,8 +21,12 @@ for compiler in $COMPILERS; do
     spack env activate .
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
-    spack concretize --fresh --somethingnotsupported > log.concretize 2>&1
+    #spack concretize --fresh --somethingnotsupported > log.concretize 2>&1
+    spack_wrapper log.concretize concretize --fresh --somethingnotsupported
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
+    
+    echo "EXIT HERE FOR TESTING"
+    exit 1
     spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
 #    spack concretize --force --fresh --test all | tee log.concretize_test
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index eaac57dfd..faeca8456 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -30,12 +30,14 @@ for compiler in $COMPILERS; do
     else
       mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
     fi
-    spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
+    spack_wrapper log.fetch mirror create --dependencies \
+        --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \
+        ${PACKAGES_TO_INSTALL:---all}
     # Just install the packages we're testing (+dependencies):
     if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then
-      spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test
+      spack_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
     fi
     # Install the rest of the stack as usual:
-    spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install
+    spack_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL
   done
 done
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index e5f25608a..106965dee 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -22,6 +22,13 @@ function spack_install_exe {
   spack $*
 }
 
+function spack_wrapper {
+  logfile=$1
+  shift
+  set -o pipefail
+  spack $* 2>&1 | tee -a $logfile
+}
+
 # Include platform-dependent configuration
 . $(dirname $0)/sites/${PLATFORM}.sh
 

From c9d0fb2827455ac5865c7db05474791143f2f21c Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 22 Oct 2024 20:42:34 -0600
Subject: [PATCH 55/61] Remove debugging code

---
 util/weekly_build/03_SetupEnv.sh | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index b8810cb7c..6dc944d09 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -22,11 +22,8 @@ for compiler in $COMPILERS; do
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
     #spack concretize --fresh --somethingnotsupported > log.concretize 2>&1
-    spack_wrapper log.concretize concretize --fresh --somethingnotsupported
+    spack_wrapper log.concretize concretize --fresh
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
-    
-    echo "EXIT HERE FOR TESTING"
-    exit 1
     spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
 #    spack concretize --force --fresh --test all | tee log.concretize_test

From 5105cf03de445bf1e9185450d1a1fb6f42bd7ebe Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 23 Oct 2024 05:36:24 -0600
Subject: [PATCH 56/61] Clean up util/weekly_build/03_SetupEnv.sh,
 util/weekly_build/ShellSetup.sh, and re-enable
 .github/workflows/nautilus-nightly-intel.yaml

---
 ...nightly-intel.yaml.tmp => nautilus-nightly-intel.yaml} | 8 ++++++++
 util/weekly_build/03_SetupEnv.sh                          | 1 -
 util/weekly_build/ShellSetup.sh                           | 8 ++------
 3 files changed, 10 insertions(+), 7 deletions(-)
 rename .github/workflows/{nautilus-nightly-intel.yaml.tmp => nautilus-nightly-intel.yaml} (79%)

diff --git a/.github/workflows/nautilus-nightly-intel.yaml.tmp b/.github/workflows/nautilus-nightly-intel.yaml
similarity index 79%
rename from .github/workflows/nautilus-nightly-intel.yaml.tmp
rename to .github/workflows/nautilus-nightly-intel.yaml
index b6ffd15fa..5225263bf 100644
--- a/.github/workflows/nautilus-nightly-intel.yaml.tmp
+++ b/.github/workflows/nautilus-nightly-intel.yaml
@@ -6,6 +6,9 @@ on:
     branches:
       - develop
     types: [opened, synchronize, reopened]
+  # Is this UTC?
+  #schedule:
+  #  - cron: '0 8 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -39,3 +42,8 @@ jobs:
       uses: ./.github/actions/Nightly_03_SetupEnv
     - name: 04_SpackInstall
       uses: ./.github/actions/Nightly_04_SpackInstall
+    - name: 05_BuildCache
+      uses: ./.github/actions/Nightly_05_BuildCache
+    # Skip step 6 - app test - for the moment
+    - name: 07_Cleanup
+      uses: ./.github/actions/Nightly_07_Cleanup
diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 6dc944d09..8bfa74b13 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -21,7 +21,6 @@ for compiler in $COMPILERS; do
     spack env activate .
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
-    #spack concretize --fresh --somethingnotsupported > log.concretize 2>&1
     spack_wrapper log.concretize concretize --fresh
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
     spack concretize --fresh --force
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 106965dee..5091f214c 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -11,17 +11,13 @@ if [ ${RUNDIR::1} != "/" ]; then
   exit 1
 fi
 
-#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"}
+# Defaults to empty, can be overwritten in site config
+PACKAGES_TO_TEST=""
 
 function alert_cmd {
   echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system."
 }
 
-function spack_install_exe {
-  #spack $* | tee -a log.install 2>&1
-  spack $*
-}
-
 function spack_wrapper {
   logfile=$1
   shift

From efb23cc5546be7579917dc68e39b9de58040b5b0 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 23 Oct 2024 05:37:17 -0600
Subject: [PATCH 57/61] Introduce spack_wrapper calls

---
 util/weekly_build/03_SetupEnv.sh     |  2 +-
 util/weekly_build/04_SpackInstall.sh |  8 +++++---
 util/weekly_build/ShellSetup.sh      | 11 +++++++----
 util/weekly_build/sites/acorn.sh     |  1 +
 4 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 5ad8a69c6..8bfa74b13 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -21,7 +21,7 @@ for compiler in $COMPILERS; do
     spack env activate .
     spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}"
     # Check for duplicates and fail before doing the "real" concretization:
-    spack concretize --fresh | tee log.concretize
+    spack_wrapper log.concretize concretize --fresh
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
     spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index eaac57dfd..faeca8456 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -30,12 +30,14 @@ for compiler in $COMPILERS; do
     else
       mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}")
     fi
-    spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch
+    spack_wrapper log.fetch mirror create --dependencies \
+        --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \
+        ${PACKAGES_TO_INSTALL:---all}
     # Just install the packages we're testing (+dependencies):
     if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then
-      spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test
+      spack_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
     fi
     # Install the rest of the stack as usual:
-    spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install
+    spack_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL
   done
 done
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index e5f25608a..5091f214c 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -11,15 +11,18 @@ if [ ${RUNDIR::1} != "/" ]; then
   exit 1
 fi
 
-#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"}
+# Defaults to empty, can be overwritten in site config
+PACKAGES_TO_TEST=""
 
 function alert_cmd {
   echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system."
 }
 
-function spack_install_exe {
-  #spack $* | tee -a log.install 2>&1
-  spack $*
+function spack_wrapper {
+  logfile=$1
+  shift
+  set -o pipefail
+  spack $* 2>&1 | tee -a $logfile
 }
 
 # Include platform-dependent configuration
diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh
index d84094f7b..d795400c4 100644
--- a/util/weekly_build/sites/acorn.sh
+++ b/util/weekly_build/sites/acorn.sh
@@ -1,6 +1,7 @@
 module load gcc/11.2.0 python/3.11.7
 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"}
 TEMPLATES=${TEMPLATES:-"unified-dev"}
+# TODO REPLACE WITH spack_wrapper
 function spack_install_exe {
 #  set +e
 #  ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) &

From d75cb5b511cfe7ed95ab82c77352c20ae12579af Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 23 Oct 2024 05:38:02 -0600
Subject: [PATCH 58/61] Update github actions and workflows

---
 .../Nightly_01_DirectorySetup/action.yaml     |  5 +-
 .../Nightly_02_GetSpackStack/action.yaml      |  1 -
 .github/workflows/nautilus-nightly-intel.yaml | 49 +++++++++++++++++++
 3 files changed, 50 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/nautilus-nightly-intel.yaml

diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml
index 2660a3745..5bcf5051c 100644
--- a/.github/actions/Nightly_01_DirectorySetup/action.yaml
+++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml
@@ -13,9 +13,6 @@ runs:
       # Get day of week for later use and write to file
       DOW=$(date +%u)
       echo "${DOW}" > DOW_SAVE.log
-      ## DH* TODO REMOVE LATER?
-      #RUNDIR=${BASEDIR}/${RUNID}
-      #rm -vfr ${RUNDIR}
-      ## *DH
+      #
       cd util/weekly_build
       ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml
index 7377bfb88..268adaf29 100644
--- a/.github/actions/Nightly_02_GetSpackStack/action.yaml
+++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml
@@ -7,6 +7,5 @@ runs:
     shell: bash
     run: |
       RUNID=$(<RUNID_SAVE.log)
-      echo "Got RUNID from file: '${RUNID}'"
       cd util/weekly_build
       ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM}
diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml
new file mode 100644
index 000000000..5225263bf
--- /dev/null
+++ b/.github/workflows/nautilus-nightly-intel.yaml
@@ -0,0 +1,49 @@
+name: nautilus unified-env intel
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - develop
+    types: [opened, synchronize, reopened]
+  # Is this UTC?
+  #schedule:
+  #  - cron: '0 8 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BASEDIR: /p/work2/heinzell/spack-stack-nightly/intel
+  PLATFORM: nautilus
+  COMPILERS: intel
+
+jobs:
+
+  nightly-build-intel:
+    runs-on: ['self-hosted','nautilus-heinzell']
+    # Set timeout to 12hrs
+    timeout-minutes: 720
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        clean: true
+    - name: 01_DirectorySetup
+      uses: ./.github/actions/Nightly_01_DirectorySetup
+    - name: 02_GetSpackStack
+      uses: ./.github/actions/Nightly_02_GetSpackStack
+    - name: 03_SetupEnv
+      uses: ./.github/actions/Nightly_03_SetupEnv
+    - name: 04_SpackInstall
+      uses: ./.github/actions/Nightly_04_SpackInstall
+    - name: 05_BuildCache
+      uses: ./.github/actions/Nightly_05_BuildCache
+    # Skip step 6 - app test - for the moment
+    - name: 07_Cleanup
+      uses: ./.github/actions/Nightly_07_Cleanup

From 97fb35215d174f098ed5db1ad261f962c7354868 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 23 Oct 2024 05:40:58 -0600
Subject: [PATCH 59/61] Update
 .github/actions/Nightly_04_SpackInstall/action.yaml

---
 .github/actions/Nightly_04_SpackInstall/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml
index 3d86123e1..b6472ccda 100644
--- a/.github/actions/Nightly_04_SpackInstall/action.yaml
+++ b/.github/actions/Nightly_04_SpackInstall/action.yaml
@@ -8,7 +8,7 @@ runs:
     run: |
       RUNID=$(<RUNID_SAVE.log)
       DOW=$(<DOW_SAVE.log)
-      # Bbuild everything from scratch on Sundays, otherwise
+      # Build everything from scratch on Sundays, otherwise
       # use build cache to speed up the installations.
       # Monday is 1 ... Sunday is 7
       if [[ $DOW == 7 ]]; then

From beab899135441924b50a3b9cca9b4291dc1203ba Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 5 Nov 2024 11:13:55 -0700
Subject: [PATCH 60/61] Remove extraneous spack concretize command from
 util/weekly_build/03_SetupEnv.sh

---
 util/weekly_build/03_SetupEnv.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh
index 8bfa74b13..a98fa6cdd 100755
--- a/util/weekly_build/03_SetupEnv.sh
+++ b/util/weekly_build/03_SetupEnv.sh
@@ -23,7 +23,6 @@ for compiler in $COMPILERS; do
     # Check for duplicates and fail before doing the "real" concretization:
     spack_wrapper log.concretize concretize --fresh
     ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf
-    spack concretize --fresh --force
 #   The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested.
 #    spack concretize --force --fresh --test all | tee log.concretize_test
   done

From e63117e8df4a99bb84bcfe55c68cb43b2fc0fa2e Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 5 Nov 2024 11:15:02 -0700
Subject: [PATCH 61/61] Reinstantiate spack_install_exe as
 spack_install_wrapper

---
 util/weekly_build/04_SpackInstall.sh | 4 ++--
 util/weekly_build/ShellSetup.sh      | 7 +++++++
 util/weekly_build/sites/acorn.sh     | 4 +++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh
index faeca8456..a40a3f6f1 100755
--- a/util/weekly_build/04_SpackInstall.sh
+++ b/util/weekly_build/04_SpackInstall.sh
@@ -35,9 +35,9 @@ for compiler in $COMPILERS; do
         ${PACKAGES_TO_INSTALL:---all}
     # Just install the packages we're testing (+dependencies):
     if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then
-      spack_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
+      spack_install_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST
     fi
     # Install the rest of the stack as usual:
-    spack_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL
+    spack_install_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL
   done
 done
diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh
index 5091f214c..c516e6fb3 100644
--- a/util/weekly_build/ShellSetup.sh
+++ b/util/weekly_build/ShellSetup.sh
@@ -25,6 +25,13 @@ function spack_wrapper {
   spack $* 2>&1 | tee -a $logfile
 }
 
+function spack_install_wrapper {
+  logfile=$1
+  shift
+  set -o pipefail
+  spack $* 2>&1 | tee -a $logfile
+}
+
 # Include platform-dependent configuration
 . $(dirname $0)/sites/${PLATFORM}.sh
 
diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh
index d84094f7b..f20f7da29 100644
--- a/util/weekly_build/sites/acorn.sh
+++ b/util/weekly_build/sites/acorn.sh
@@ -1,7 +1,9 @@
 module load gcc/11.2.0 python/3.11.7
 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"}
 TEMPLATES=${TEMPLATES:-"unified-dev"}
-function spack_install_exe {
+function spack_install_wrapper {
+  logfile=$1
+  shift
 #  set +e
 #  ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) &
 #  ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) &