ci(phoenix): bound SLURM queue wait and combine build+test into one allocation

.github/scripts/monitor_slurm_job.sh

@@ -82,14 +82,58 @@ is_terminal_state() {
   esac
 }
 
+# Bound how long a job may sit un-started in the queue. On a preemptible QOS
+# (Phoenix 'embers') a job can stay PENDING for hours, burning the CI job
+# timeout and holding a runner slot; fail early so it reads as queue starvation,
+# not a test failure. 0 = wait indefinitely.
+#
+# The budget has to clear a normal bad day on a busy machine, or it converts
+# routine queue pressure into red CI. Frontier's own numbers make the case:
+# over one week, MFC jobs on `batch` waited p50=1m but p90=96m and p95=176m,
+# with a 466m tail. A 90-minute budget cut into that distribution, tripping on
+# 11% of the jobs that did eventually start -- plus the ones that never did.
+# Four hours clears p95 with room to spare while staying well inside the 480m
+# job-level `timeout-minutes`, which remains the real backstop.
+: "${SLURM_MAX_QUEUE_SECONDS:=14400}"   # 4 hours
+# Reject a non-integer override rather than silently skipping the budget.
+if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2
+  exit 1
+fi
+queue_start=$(date +%s)
+
+abort_queue_starvation() {
+  local waited="$1"
+  echo "##[error]SLURM job $job_id did not start within ${waited}s (SLURM_MAX_QUEUE_SECONDS=$SLURM_MAX_QUEUE_SECONDS)."
+  echo "QUEUE STARVATION: the cluster scheduler could not start this job in time."
+  echo "This is an infrastructure / queue-availability problem, NOT a code or test failure."
+  echo "Cancelling the queued job so it does not keep holding a CI runner slot."
+  scancel "$job_id" 2>/dev/null || true
+  exit 75   # EX_TEMPFAIL — distinguishes queue starvation from a real test failure
+}
+
 # Wait for file to appear, using robust state checking.
-# Never give up due to transient squeue/sacct failures — the CI job timeout
-# is the ultimate backstop.
+# Never give up due to transient squeue/sacct failures — the queue-wait budget
+# above (or the CI job timeout) is the ultimate backstop.
 echo "Waiting for job to start..."
 unknown_count=0
 while [ ! -f "$output_file" ]; do
   state=$(get_job_state "$job_id")
 
+  # A started job (RUNNING/COMPLETING) whose output file is merely NFS-delayed
+  # is exempt, so work in progress is never killed here.
+  if [ "$SLURM_MAX_QUEUE_SECONDS" -gt 0 ]; then
+    case "$state" in
+      RUNNING|COMPLETING) ;;
+      *)
+        waited=$(( $(date +%s) - queue_start ))
+        if [ "$waited" -ge "$SLURM_MAX_QUEUE_SECONDS" ]; then
+          abort_queue_starvation "$waited"
+        fi
+        ;;
+    esac
+  fi
+
   case "$state" in
     PENDING|CONFIGURING|PREEMPTED)
       unknown_count=0

.github/scripts/submit-slurm-job.sh

@@ -34,8 +34,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Detect job type from submitted script basename
 script_basename="$(basename "$script_path" .sh)"
 case "$script_basename" in
-    bench*) job_type="bench" ;;
-    *)      job_type="test"  ;;
+    bench*)          job_type="bench" ;;
+    build-and-test*) job_type="buildtest" ;;
+    *)               job_type="test"  ;;
 esac
 
 # --- Cluster configuration ---
@@ -47,6 +48,9 @@ case "$cluster" in
         qos="embers"
         extra_sbatch="#SBATCH --requeue"
         test_time="03:00:00"
+        # Combined build+test needs build headroom on top of the test budget;
+        # kept modest to still backfill under 'embers'.
+        buildtest_time="03:30:00"
         bench_time="04:00:00"
         gpu_partition_dynamic=true
         ;;
@@ -58,7 +62,10 @@ case "$cluster" in
         # CFD154; submitting under it now fails outright with "Invalid qos
         # specification". "normal" is the only QOS on this allocation without a
         # one-job-at-a-time cap, so it is the only one that can run the CI
-        # matrix concurrently.
+        # matrix concurrently. Note that the g1 partition carries its own
+        # partition QOS (also named "g1"), which slurmctld applies on its own
+        # when a job lands there. Do not add --qos=g1: CFD154 has no
+        # association with it and sbatch rejects the job outright.
         qos="normal"
         # Let each job's slurmstepd broker its own steps instead of routing
         # every srun through slurmctld. The in-job test suite launches ~1700+
@@ -85,11 +92,11 @@ case "$cluster" in
 esac
 
 # --- Time limit ---
-if [ "$job_type" = "bench" ]; then
-    sbatch_time="#SBATCH -t $bench_time"
-else
-    sbatch_time="#SBATCH -t $test_time"
-fi
+case "$job_type" in
+    bench)     sbatch_time="#SBATCH -t $bench_time" ;;
+    buildtest) sbatch_time="#SBATCH -t ${buildtest_time:-$test_time}" ;;
+    *)         sbatch_time="#SBATCH -t $test_time" ;;
+esac
 
 # --- Device-specific SBATCH options ---
 if [ "$device" = "cpu" ]; then
@@ -101,9 +108,11 @@ if [ "$device" = "cpu" ]; then
 #SBATCH --mem-per-cpu=8G"
             ;;
         frontier|frontier_amd)
+            # g1 is a dedicated 64-node carve-out; its nodes are not in batch,
+            # so CI starts promptly instead of queueing behind the machine.
             sbatch_device_opts="\
 #SBATCH -n 32
-#SBATCH -p batch"
+#SBATCH -p g1"
             ;;
     esac
 elif [ "$device" = "gpu" ]; then
@@ -131,7 +140,7 @@ elif [ "$device" = "gpu" ]; then
         frontier|frontier_amd)
             sbatch_device_opts="\
 #SBATCH -n 8
-#SBATCH -p batch"
+#SBATCH -p g1"
             ;;
     esac
 else
@@ -196,12 +205,13 @@ set -x
 cd "\$SLURM_SUBMIT_DIR"
 echo "Running in \$(pwd):"
 
-job_slug="$job_slug"
-job_device="$device"
-job_interface="$interface"
-job_shard="$shard"
-job_variant="$variant"
-job_cluster="$cluster"
+# Exported so wrapper scripts (build-and-test.sh) run child scripts that inherit these.
+export job_slug="$job_slug"
+export job_device="$device"
+export job_interface="$interface"
+export job_shard="$shard"
+export job_variant="$variant"
+export job_cluster="$cluster"
 export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME"
 
 . ./mfc.sh load -c $compiler_flag -m $module_mode

.github/workflows/common/build-and-test.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Build then test in a single SLURM allocation so the scheduler queue wait is
+# paid once instead of twice (Phoenix 'embers' is queue-bound, not work-bound).
+# submit-slurm-job.sh exports the job_* vars so these child scripts inherit
+# them; cwd is the workspace root, so build/ persists into test.sh's --no-build.
+
+set -euo pipefail
+
+echo "=== [build-and-test] Build phase ==="
+bash .github/workflows/common/build.sh
+
+echo "=== [build-and-test] Test phase ==="
+bash .github/workflows/common/test.sh

.github/workflows/test.yml

@@ -455,7 +455,8 @@ jobs:
         run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
-        if:   ${{ !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }}
+        # Phoenix builds+tests in one allocation (see "Build & Test"); others build separately.
+        if:   ${{ matrix.cluster != 'phoenix' && !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }}
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
       - name: Build (concurrent variants)
@@ -474,7 +475,13 @@ jobs:
           for p in $pids; do wait "$p" || rc=1; done
           exit $rc
 
+      - name: Build & Test
+        # Phoenix: build+test in one allocation so the 'embers' queue wait is paid once.
+        if:   matrix.cluster == 'phoenix'
+        run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build-and-test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
+
       - name: Test
+        if:   matrix.cluster != 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
       - name: Cancel SLURM Jobs
@@ -497,14 +504,17 @@ jobs:
           fi
           echo "build_slug=build-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
           echo "test_slug=test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
+          echo "combined_slug=build-and-test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
 
       - name: Print Logs
         if:   always()
         run:  |
-          # The build slug matches the plain log and, where the build is split per
-          # variant, the build-...-base/-chem ones.
-          for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.test_slug }}.out; do
-            [ -f "$f" ] && echo "=== $f ===" && cat "$f"
+          # build_slug matches the plain log (and per-variant base/chem builds);
+          # combined_slug matches Phoenix's single build-and-test allocation.
+          # Use `if` (not `[ -f ] && ...`) so a missing file — e.g. test_slug on
+          # Phoenix's combined jobs — doesn't make the loop exit non-zero under -e.
+          for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.combined_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do
+            if [ -f "$f" ]; then echo "=== $f ==="; cat "$f"; fi
           done
 
       - name: Archive Logs