From 1e334563c93ef75ce2d21447574ebf5047910d8a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 10 Jan 2024 10:11:36 -0700 Subject: [PATCH 1/4] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..bd242d02b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = feature/pull_spack_backports_v0p21p1 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index cc9780cd0..5fd82305d 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit cc9780cd0e4f0a44137af65862d7a5d876d03fc8 +Subproject commit 5fd82305d9d5978cbe840eba2d47bda5766b8b27 From f64a56676f128b08bd3b08bf711c977d1aac9a3d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 10 Jan 2024 13:23:26 -0700 Subject: [PATCH 2/4] Attempt to fix duplicate packages by installing Intel and GNU stacks one after the other --- .github/workflows/ubuntu-rnd-x86_64.yaml | 127 +++++++++++++++-------- 1 file changed, 83 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml index b808fc58b..a22b59224 100644 --- a/.github/workflows/ubuntu-rnd-x86_64.yaml +++ b/.github/workflows/ubuntu-rnd-x86_64.yaml @@ -61,9 +61,6 @@ jobs: # For buildcaches spack config add config:install_tree:padded_length:200 - # Set compiler and MPI - sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%gcc', '\%intel'\]/g" ${ENVDIR}/spack.yaml - # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" @@ -74,28 +71,45 @@ jobs: spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache" spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache" - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -c -d log.concretize.${ENVNAME} -i fms -i crtm -i esmf -i mapl - - # Update spack source cache - spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/ - spack buildcache update-index local-binary - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Create/update binary cache - echo "Create/update build cache for environment ${TEMPLATE} ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${TEMPLATE} - spack buildcache create -a -u /mnt/experiments-efs/spack-stack/build-cache/ - - # Next steps: synchronize source and build cache to a central/combined mirror? - #echo "Next steps ..." + # Loop over compilers + compilers=(gcc intel) + cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original + + for compiler in "${!compilers[@]}" + do + # Set compiler and MPI + cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml + sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml + cat spack.yaml + + # Concretize and check for duplicates + spack concretize 2>&1 | tee log.concretize.${ENVNAME}.${compiler} + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl + + # Update spack source cache + spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache + + # Add binary cache if requested + if [ "$USE_BINARY_CACHE" = true ] ; then + spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/ + spack buildcache update-index local-binary + echo "Packages in spack binary cache:" + spack buildcache list + fi + + # Create/update binary cache + echo "Create/update build cache for environment ${TEMPLATE} and compiler ${compiler}..." + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler} + spack buildcache create -a -u /mnt/experiments-efs/spack-stack/build-cache/ + + # Next steps: synchronize source and build cache to a central/combined mirror? + #echo "Next steps ..." + + # Remove binary cache for next round of concretization + if [ "$USE_BINARY_CACHE" = true ] ; then + spack mirror rm local-binary + fi + done spack env deactivate rm -fr /home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache @@ -113,9 +127,6 @@ jobs: spack env activate ${ENVDIR} spack add ${{ inputs.specs || '' }} - # Set compiler and MPI - sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%gcc', '\%intel'\]/g" ${ENVDIR}/spack.yaml - # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" @@ -126,26 +137,43 @@ jobs: spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache" spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache" - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.${ENVNAME} - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -c -d log.concretize.${ENVNAME} -i fms -i crtm -i esmf -i mapl + # Loop over compilers + compilers=(gcc intel) + cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original - # Update spack source cache - spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache + for compiler in "${!compilers[@]}" + do + # Set compiler and MPI + cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml + sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml + cat spack.yaml - # Add binary cache - spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/ - spack buildcache update-index local-binary - echo "Packages in spack binary cache:" - spack buildcache list + # Concretize and check for duplicates + spack concretize 2>&1 | tee log.concretize.${ENVNAME}.${compiler} + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl - # Install from binary cache - echo "Install environment ${TEMPLATE} from binary cache ..." - spack install --fail-fast --no-check-signature 2>&1 | tee log.install.${TEMPLATE} - spack clean -a + # Update spack source cache + spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache + + # Add binary cache + spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/ + spack buildcache update-index local-binary + echo "Packages in spack binary cache:" + spack buildcache list + + # Install from binary cache + echo "Install environment ${TEMPLATE} for compiler ${compiler} from binary cache ..." + spack install --fail-fast --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler} + spack clean -a + + # Create modules + spack module lmod refresh -y + spack stack setup-meta-modules + + # Remove binary cache for next round of concretization + spack mirror rm local-binary + done - spack module lmod refresh -y - spack stack setup-meta-modules spack env deactivate - name: test-env @@ -170,6 +198,17 @@ jobs: module load soca-env/1.0.0 module list + module purge + module load stack-gcc/9.4.0 + module load stack-openmpi/4.1.4 + module load stack-python/3.10.13 + module available + + module load jedi-ufs-env/1.0.0 + module load ewok-env/1.0.0 + module load soca-env/1.0.0 + module list + # Create empty file to signal that this environment is ready for testing touch /home/ubuntu/spack-stack/CI/unified-env/${TODAY}.ready From 7be984c4f40319f5e5b42b5d49eac79cc87bf85d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 13 Jan 2024 19:32:25 -0700 Subject: [PATCH 3/4] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index bd242d02b..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack - branch = feature/pull_spack_backports_v0p21p1 + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 5fd82305d..11e56fde2 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 5fd82305d9d5978cbe840eba2d47bda5766b8b27 +Subproject commit 11e56fde2d506dc66f4a848c13c690024bbb46dc From 0b06c7f9048d13db3a36a94b08123ff004aa55c7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 14 Jan 2024 16:27:18 -0700 Subject: [PATCH 4/4] Fix concretize arguments in .github/workflows/ubuntu-rnd-x86_64.yaml --- .github/workflows/ubuntu-rnd-x86_64.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml index a22b59224..c67a4733b 100644 --- a/.github/workflows/ubuntu-rnd-x86_64.yaml +++ b/.github/workflows/ubuntu-rnd-x86_64.yaml @@ -83,7 +83,7 @@ jobs: cat spack.yaml # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.${ENVNAME}.${compiler} + spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler} ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl # Update spack source cache @@ -149,7 +149,7 @@ jobs: cat spack.yaml # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.${ENVNAME}.${compiler} + spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler} ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl # Update spack source cache