diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh index 5ddc27d8d..3d4c4a717 100755 --- a/.github/scripts/monitor_slurm_job.sh +++ b/.github/scripts/monitor_slurm_job.sh @@ -82,14 +82,58 @@ is_terminal_state() { esac } +# Bound how long a job may sit un-started in the queue. On a preemptible QOS +# (Phoenix 'embers') a job can stay PENDING for hours, burning the CI job +# timeout and holding a runner slot; fail early so it reads as queue starvation, +# not a test failure. 0 = wait indefinitely. +# +# The budget has to clear a normal bad day on a busy machine, or it converts +# routine queue pressure into red CI. Frontier's own numbers make the case: +# over one week, MFC jobs on `batch` waited p50=1m but p90=96m and p95=176m, +# with a 466m tail. A 90-minute budget cut into that distribution, tripping on +# 11% of the jobs that did eventually start -- plus the ones that never did. +# Four hours clears p95 with room to spare while staying well inside the 480m +# job-level `timeout-minutes`, which remains the real backstop. +: "${SLURM_MAX_QUEUE_SECONDS:=14400}" # 4 hours +# Reject a non-integer override rather than silently skipping the budget. +if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then + echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2 + exit 1 +fi +queue_start=$(date +%s) + +abort_queue_starvation() { + local waited="$1" + echo "##[error]SLURM job $job_id did not start within ${waited}s (SLURM_MAX_QUEUE_SECONDS=$SLURM_MAX_QUEUE_SECONDS)." + echo "QUEUE STARVATION: the cluster scheduler could not start this job in time." + echo "This is an infrastructure / queue-availability problem, NOT a code or test failure." + echo "Cancelling the queued job so it does not keep holding a CI runner slot." + scancel "$job_id" 2>/dev/null || true + exit 75 # EX_TEMPFAIL — distinguishes queue starvation from a real test failure +} + # Wait for file to appear, using robust state checking. -# Never give up due to transient squeue/sacct failures — the CI job timeout -# is the ultimate backstop. +# Never give up due to transient squeue/sacct failures — the queue-wait budget +# above (or the CI job timeout) is the ultimate backstop. echo "Waiting for job to start..." unknown_count=0 while [ ! -f "$output_file" ]; do state=$(get_job_state "$job_id") + # A started job (RUNNING/COMPLETING) whose output file is merely NFS-delayed + # is exempt, so work in progress is never killed here. + if [ "$SLURM_MAX_QUEUE_SECONDS" -gt 0 ]; then + case "$state" in + RUNNING|COMPLETING) ;; + *) + waited=$(( $(date +%s) - queue_start )) + if [ "$waited" -ge "$SLURM_MAX_QUEUE_SECONDS" ]; then + abort_queue_starvation "$waited" + fi + ;; + esac + fi + case "$state" in PENDING|CONFIGURING|PREEMPTED) unknown_count=0 diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index a5f0dbf96..16035419b 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -34,8 +34,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Detect job type from submitted script basename script_basename="$(basename "$script_path" .sh)" case "$script_basename" in - bench*) job_type="bench" ;; - *) job_type="test" ;; + bench*) job_type="bench" ;; + build-and-test*) job_type="buildtest" ;; + *) job_type="test" ;; esac # --- Cluster configuration --- @@ -47,6 +48,9 @@ case "$cluster" in qos="embers" extra_sbatch="#SBATCH --requeue" test_time="03:00:00" + # Combined build+test needs build headroom on top of the test budget; + # kept modest to still backfill under 'embers'. + buildtest_time="03:30:00" bench_time="04:00:00" gpu_partition_dynamic=true ;; @@ -58,7 +62,10 @@ case "$cluster" in # CFD154; submitting under it now fails outright with "Invalid qos # specification". "normal" is the only QOS on this allocation without a # one-job-at-a-time cap, so it is the only one that can run the CI - # matrix concurrently. + # matrix concurrently. Note that the g1 partition carries its own + # partition QOS (also named "g1"), which slurmctld applies on its own + # when a job lands there. Do not add --qos=g1: CFD154 has no + # association with it and sbatch rejects the job outright. qos="normal" # Let each job's slurmstepd broker its own steps instead of routing # every srun through slurmctld. The in-job test suite launches ~1700+ @@ -85,11 +92,11 @@ case "$cluster" in esac # --- Time limit --- -if [ "$job_type" = "bench" ]; then - sbatch_time="#SBATCH -t $bench_time" -else - sbatch_time="#SBATCH -t $test_time" -fi +case "$job_type" in + bench) sbatch_time="#SBATCH -t $bench_time" ;; + buildtest) sbatch_time="#SBATCH -t ${buildtest_time:-$test_time}" ;; + *) sbatch_time="#SBATCH -t $test_time" ;; +esac # --- Device-specific SBATCH options --- if [ "$device" = "cpu" ]; then @@ -101,9 +108,11 @@ if [ "$device" = "cpu" ]; then #SBATCH --mem-per-cpu=8G" ;; frontier|frontier_amd) + # g1 is a dedicated 64-node carve-out; its nodes are not in batch, + # so CI starts promptly instead of queueing behind the machine. sbatch_device_opts="\ #SBATCH -n 32 -#SBATCH -p batch" +#SBATCH -p g1" ;; esac elif [ "$device" = "gpu" ]; then @@ -131,7 +140,7 @@ elif [ "$device" = "gpu" ]; then frontier|frontier_amd) sbatch_device_opts="\ #SBATCH -n 8 -#SBATCH -p batch" +#SBATCH -p g1" ;; esac else @@ -196,12 +205,13 @@ set -x cd "\$SLURM_SUBMIT_DIR" echo "Running in \$(pwd):" -job_slug="$job_slug" -job_device="$device" -job_interface="$interface" -job_shard="$shard" -job_variant="$variant" -job_cluster="$cluster" +# Exported so wrapper scripts (build-and-test.sh) run child scripts that inherit these. +export job_slug="$job_slug" +export job_device="$device" +export job_interface="$interface" +export job_shard="$shard" +export job_variant="$variant" +export job_cluster="$cluster" export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME" . ./mfc.sh load -c $compiler_flag -m $module_mode diff --git a/.github/workflows/common/build-and-test.sh b/.github/workflows/common/build-and-test.sh new file mode 100755 index 000000000..1889889ef --- /dev/null +++ b/.github/workflows/common/build-and-test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Build then test in a single SLURM allocation so the scheduler queue wait is +# paid once instead of twice (Phoenix 'embers' is queue-bound, not work-bound). +# submit-slurm-job.sh exports the job_* vars so these child scripts inherit +# them; cwd is the workspace root, so build/ persists into test.sh's --no-build. + +set -euo pipefail + +echo "=== [build-and-test] Build phase ===" +bash .github/workflows/common/build.sh + +echo "=== [build-and-test] Test phase ===" +bash .github/workflows/common/test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87163fe37..eba8b1d17 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -455,7 +455,8 @@ jobs: run: bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }} - name: Build - if: ${{ !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }} + # Phoenix builds+tests in one allocation (see "Build & Test"); others build separately. + if: ${{ matrix.cluster != 'phoenix' && !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }} run: bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }} - name: Build (concurrent variants) @@ -474,7 +475,13 @@ jobs: for p in $pids; do wait "$p" || rc=1; done exit $rc + - name: Build & Test + # Phoenix: build+test in one allocation so the 'embers' queue wait is paid once. + if: matrix.cluster == 'phoenix' + run: bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build-and-test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }} + - name: Test + if: matrix.cluster != 'phoenix' run: bash .github/scripts/submit-slurm-job.sh .github/workflows/common/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }} - name: Cancel SLURM Jobs @@ -497,14 +504,17 @@ jobs: fi echo "build_slug=build-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT" echo "test_slug=test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT" + echo "combined_slug=build-and-test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT" - name: Print Logs if: always() run: | - # The build slug matches the plain log and, where the build is split per - # variant, the build-...-base/-chem ones. - for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.test_slug }}.out; do - [ -f "$f" ] && echo "=== $f ===" && cat "$f" + # build_slug matches the plain log (and per-variant base/chem builds); + # combined_slug matches Phoenix's single build-and-test allocation. + # Use `if` (not `[ -f ] && ...`) so a missing file — e.g. test_slug on + # Phoenix's combined jobs — doesn't make the loop exit non-zero under -e. + for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.combined_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do + if [ -f "$f" ]; then echo "=== $f ==="; cat "$f"; fi done - name: Archive Logs