diff --git a/.github/workflows/scripts/only_latest_easystacks.sh b/.github/workflows/scripts/only_latest_easystacks.sh new file mode 100755 index 0000000000..acc9d3279a --- /dev/null +++ b/.github/workflows/scripts/only_latest_easystacks.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# This script figures out the latest version of EasyBuild being used for the installation of easystack +# files. +# +# This file is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Alan O'Cais (CECAM) +# +# license: GPLv2 +# + +EESSI_VERSION=${EESSI_VERSION:-"2023.06"} + +directory="easystacks/software.eessi.io/${EESSI_VERSION}" +# List of example filenames +files=($(find "$directory" -name "*.yml" | grep -e '-eb-')) +[ -n "$DEBUG" ] && echo "${files[@]}" + +versions=() +# Loop over each filename +for filename in "${files[@]}"; do + # Extract the semantic version using grep + version=$(echo "$filename" | grep -oP '(?<=eb-)\d+\.\d+\.\d+?(?=-)') + + # Output the result + [ -n "$DEBUG" ] && echo "Filename: $filename" + [ -n "$DEBUG" ] && echo "Extracted version: $version" + [ -n "$DEBUG" ] && echo + versions+=("$version") +done +highest_version=$(printf "%s\n" "${versions[@]}" | sort -V | tail -n 1) + +[ -n "$DEBUG" ] && echo "Highest version: $highest_version" +[ -n "$DEBUG" ] && echo +[ -n "$DEBUG" ] && echo "Matching files:" +all_latest_easystacks=($(find $directory -type f -name "*eb-$highest_version*.yml")) + +accel_latest_easystacks=() +cpu_latest_easystacks=() + +# Loop through the array and split based on partial matching of string +accel="/accel/" +for item in "${all_latest_easystacks[@]}"; do + if [[ "$item" == *"$accel"* ]]; then + accel_latest_easystacks+=("$item") + else + cpu_latest_easystacks+=("$item") + fi +done + +# Output the results +if [ -n "$ACCEL_EASYSTACKS" ]; then + echo "${accel_latest_easystacks[@]}" +else + echo "${cpu_latest_easystacks[@]}" +fi diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 6f592cf4c4..ca3792f6ef 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -57,15 +57,20 @@ jobs: env | grep ^EESSI | sort # first check the CPU-only builds for this CPU target - echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" + for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then if grep -q 2022b <<<"${easystack_file}"; then - # skip the check of installed software on zen4 for foss/2022b builds - continue - elif grep -q CUDA <<<"${easystack_file}"; then - # skip the check of install CUDA software in the CPU path for zen4 - continue + # skip the check of installed software on zen4 for foss/2022b builds + continue + fi + if [[ $easystack_file == *"rebuilds"* ]]; then + # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that + new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) + # first clean out the options then clean out the .eb name + sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack + diff --unified=0 "$easystack_file" "$new_easystack" || : + easystack_file="$new_easystack" fi fi echo "check missing installations for ${easystack_file}..." @@ -82,7 +87,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 280de294af..79f6acc733 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -25,17 +25,6 @@ easystack=$1 LOCAL_TMPDIR=$(mktemp -d) -# Clone the develop branch of EasyBuild and use that to search for easyconfigs - -if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then - git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs - export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs -fi - -# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr -tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack}) -grep -v from-pr ${easystack} > ${tmp_easystack} - source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild @@ -45,34 +34,11 @@ ${EB:-eb} --show-config echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'" -if [ "$exit_code" -ne 0 ] && [ ! -z "$pr_exceptions" ]; then - # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR - # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg - # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as - # well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario - - # Let's use awk so we can allow for exceptions if we are given a PR diff file - awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack" - awk_command=${awk_command/\\/} # Strip out the backslash we needed for ! - eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack} - - msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}," - msg="${msg} allowing for --from-pr's that were added in this PR..." - echo ${msg} - eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out - ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out} - exit_code_with_from_pr=${PIPESTATUS[0]} - - # If now we succeeded, the reason must be that we originally stripped the --from-pr's - if [ "$exit_code_with_from_pr" -eq 0 ]; then - fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)" - fi -fi check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml rename to easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240506-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml