From d95d6264bb5f1c10c331b0670f946f45b35508f9 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Tue, 25 Aug 2026 12:08:03 -0400
Subject: [PATCH 1/6] ci(phoenix): bound SLURM queue wait and combine
 build+test into one allocation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Phoenix CI jobs were failing as opaque ~8h 'cancelled': jobs sat PENDING for
hours on the preemptible 'embers' QOS, burning the 480-min job timeout and
holding self-hosted runner slots (which backs up unrelated PRs).

A) monitor_slurm_job.sh: bound the queue wait (SLURM_MAX_QUEUE_SECONDS, default
   90m). If a job never starts, scancel and fail fast with an explicit
   'queue starvation — infrastructure, not code' message and exit 75. A
   RUNNING job with a merely NFS-delayed output file is exempt.

B) Phoenix builds+tests in a single SLURM allocation (build-and-test.sh) so the
   scheduler queue wait is paid once instead of twice. submit-slurm-job.sh
   exports job_* so the child scripts inherit them; adds a 'buildtest' time
   budget (3h30m). Other clusters keep the separate build/test steps.
---
 .github/scripts/monitor_slurm_job.sh       | 39 ++++++++++++++++++++--
 .github/scripts/submit-slurm-job.sh        | 33 ++++++++++--------
 .github/workflows/common/build-and-test.sh | 19 +++++++++++
 .github/workflows/test.yml                 | 17 ++++++++--
 4 files changed, 90 insertions(+), 18 deletions(-)
 create mode 100755 .github/workflows/common/build-and-test.sh

diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh
index 5ddc27d8d7..c40c43509f 100755
--- a/.github/scripts/monitor_slurm_job.sh
+++ b/.github/scripts/monitor_slurm_job.sh
@@ -82,14 +82,49 @@ is_terminal_state() {
   esac
 }
 
+# Bound how long the job may sit un-started in the scheduler queue.
+# Under a contended, preemptible QOS (e.g. Phoenix 'embers') a job can stay
+# PENDING for many hours, burning the entire CI job timeout, holding a
+# self-hosted runner slot hostage, and surfacing as an opaque ~8 h 'cancelled'.
+# Give up after a bounded queue wait with a loud, greppable message so the
+# failure reads as infrastructure (queue starvation), not a code/test failure.
+# Set SLURM_MAX_QUEUE_SECONDS=0 to wait indefinitely (previous behaviour).
+: "${SLURM_MAX_QUEUE_SECONDS:=5400}"   # 90 minutes
+queue_start=$(date +%s)
+
+abort_queue_starvation() {
+  local waited="$1"
+  echo "##[error]SLURM job $job_id did not start within ${waited}s (SLURM_MAX_QUEUE_SECONDS=$SLURM_MAX_QUEUE_SECONDS)."
+  echo "QUEUE STARVATION: the cluster scheduler could not start this job in time."
+  echo "This is an infrastructure / queue-availability problem, NOT a code or test failure."
+  echo "Cancelling the queued job so it does not keep holding a CI runner slot."
+  scancel "$job_id" 2>/dev/null || true
+  exit 75   # EX_TEMPFAIL — distinguishes queue starvation from a real test failure
+}
+
 # Wait for file to appear, using robust state checking.
-# Never give up due to transient squeue/sacct failures — the CI job timeout
-# is the ultimate backstop.
+# Never give up due to transient squeue/sacct failures — the queue-wait budget
+# above (or the CI job timeout) is the ultimate backstop.
 echo "Waiting for job to start..."
 unknown_count=0
 while [ ! -f "$output_file" ]; do
   state=$(get_job_state "$job_id")
 
+  # Enforce the queue-wait budget. A job that has actually started
+  # (RUNNING/COMPLETING) but whose output file is merely NFS-delayed is exempt,
+  # so a job that is already doing work is never killed here.
+  if [ "$SLURM_MAX_QUEUE_SECONDS" -gt 0 ]; then
+    case "$state" in
+      RUNNING|COMPLETING) ;;
+      *)
+        waited=$(( $(date +%s) - queue_start ))
+        if [ "$waited" -ge "$SLURM_MAX_QUEUE_SECONDS" ]; then
+          abort_queue_starvation "$waited"
+        fi
+        ;;
+    esac
+  fi
+
   case "$state" in
     PENDING|CONFIGURING|PREEMPTED)
       unknown_count=0
diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh
index a5f0dbf96f..1954c6f94b 100755
--- a/.github/scripts/submit-slurm-job.sh
+++ b/.github/scripts/submit-slurm-job.sh
@@ -34,8 +34,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Detect job type from submitted script basename
 script_basename="$(basename "$script_path" .sh)"
 case "$script_basename" in
-    bench*) job_type="bench" ;;
-    *)      job_type="test"  ;;
+    bench*)          job_type="bench" ;;
+    build-and-test*) job_type="buildtest" ;;
+    *)               job_type="test"  ;;
 esac
 
 # --- Cluster configuration ---
@@ -47,6 +48,10 @@ case "$cluster" in
         qos="embers"
         extra_sbatch="#SBATCH --requeue"
         test_time="03:00:00"
+        # Combined build+test runs both phases in one allocation (one queue
+        # wait instead of two). It needs headroom for the build on top of the
+        # test budget; kept modest so it still backfills well under 'embers'.
+        buildtest_time="03:30:00"
         bench_time="04:00:00"
         gpu_partition_dynamic=true
         ;;
@@ -85,11 +90,11 @@ case "$cluster" in
 esac
 
 # --- Time limit ---
-if [ "$job_type" = "bench" ]; then
-    sbatch_time="#SBATCH -t $bench_time"
-else
-    sbatch_time="#SBATCH -t $test_time"
-fi
+case "$job_type" in
+    bench)     sbatch_time="#SBATCH -t $bench_time" ;;
+    buildtest) sbatch_time="#SBATCH -t $buildtest_time" ;;
+    *)         sbatch_time="#SBATCH -t $test_time" ;;
+esac
 
 # --- Device-specific SBATCH options ---
 if [ "$device" = "cpu" ]; then
@@ -196,12 +201,14 @@ set -x
 cd "\$SLURM_SUBMIT_DIR"
 echo "Running in \$(pwd):"
 
-job_slug="$job_slug"
-job_device="$device"
-job_interface="$interface"
-job_shard="$shard"
-job_variant="$variant"
-job_cluster="$cluster"
+# Exported so a wrapper script (e.g. build-and-test.sh) can run build.sh and
+# test.sh as child processes that inherit these, not just inline this shell.
+export job_slug="$job_slug"
+export job_device="$device"
+export job_interface="$interface"
+export job_shard="$shard"
+export job_variant="$variant"
+export job_cluster="$cluster"
 export GITHUB_EVENT_NAME="$GITHUB_EVENT_NAME"
 
 . ./mfc.sh load -c $compiler_flag -m $module_mode
diff --git a/.github/workflows/common/build-and-test.sh b/.github/workflows/common/build-and-test.sh
new file mode 100755
index 0000000000..b1f0d3a700
--- /dev/null
+++ b/.github/workflows/common/build-and-test.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Combined build-then-test for a single SLURM allocation.
+# Runs inside a SLURM job via submit-slurm-job.sh. Doing both phases in one
+# allocation means the scheduler queue wait is paid once instead of twice —
+# on a contended, preemptible QOS (Phoenix 'embers') the queue wait, not the
+# work, is what makes CI slow and flaky.
+#
+# Relies on submit-slurm-job.sh exporting the job_* vars (job_device,
+# job_interface, job_cluster, ...) so the child scripts inherit them. cwd is
+# $SLURM_SUBMIT_DIR (the workspace root), so build/ persists from build.sh into
+# test.sh's --no-build run.
+
+set -euo pipefail
+
+echo "=== [build-and-test] Build phase ==="
+bash .github/workflows/common/build.sh
+
+echo "=== [build-and-test] Test phase ==="
+bash .github/workflows/common/test.sh
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 87163fe379..5d765ded08 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -455,7 +455,9 @@ jobs:
         run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
-        if:   ${{ !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }}
+        # Phoenix builds+tests in one allocation (see "Build & Test" below); every
+        # other cluster keeps the separate build job.
+        if:   ${{ matrix.cluster != 'phoenix' && !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }}
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
       - name: Build (concurrent variants)
@@ -474,7 +476,14 @@ jobs:
           for p in $pids; do wait "$p" || rc=1; done
           exit $rc
 
+      - name: Build & Test
+        # Phoenix: build and test in a single SLURM allocation so the scheduler
+        # queue wait (preemptible 'embers' QOS) is paid once instead of twice.
+        if:   matrix.cluster == 'phoenix'
+        run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build-and-test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
+
       - name: Test
+        if:   matrix.cluster != 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
       - name: Cancel SLURM Jobs
@@ -497,13 +506,15 @@ jobs:
           fi
           echo "build_slug=build-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
           echo "test_slug=test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
+          echo "combined_slug=build-and-test-${{ matrix.device }}-${{ matrix.interface }}${SHARD_SUFFIX}" >> "$GITHUB_OUTPUT"
 
       - name: Print Logs
         if:   always()
         run:  |
           # The build slug matches the plain log and, where the build is split per
-          # variant, the build-...-base/-chem ones.
-          for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.test_slug }}.out; do
+          # variant, the build-...-base/-chem ones. The combined slug matches
+          # Phoenix's single build-and-test allocation.
+          for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.combined_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do
             [ -f "$f" ] && echo "=== $f ===" && cat "$f"
           done
 

From 3585f35dad9f05d3d5f7c2181b3c778515c00dd4 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Tue, 25 Aug 2026 12:31:24 -0400
Subject: [PATCH 2/6] ci: fall back buildtest_time to test_time for non-phoenix
 clusters

Addresses review: buildtest_time was only defined in the phoenix cluster
config, so a build-and-test.sh submission on any other cluster would hit an
unbound-variable crash under 'set -u'. Use ${buildtest_time:-$test_time} so the
combined job type is safe on every cluster (only phoenix uses it today).
---
 .github/scripts/submit-slurm-job.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh
index 1954c6f94b..2358494000 100755
--- a/.github/scripts/submit-slurm-job.sh
+++ b/.github/scripts/submit-slurm-job.sh
@@ -92,7 +92,7 @@ esac
 # --- Time limit ---
 case "$job_type" in
     bench)     sbatch_time="#SBATCH -t $bench_time" ;;
-    buildtest) sbatch_time="#SBATCH -t $buildtest_time" ;;
+    buildtest) sbatch_time="#SBATCH -t ${buildtest_time:-$test_time}" ;;
     *)         sbatch_time="#SBATCH -t $test_time" ;;
 esac
 

From f5aa4b35c979be3bf8239ab42cc04dd9a58b2431 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Tue, 25 Aug 2026 12:57:31 -0400
Subject: [PATCH 3/6] ci: reject non-integer SLURM_MAX_QUEUE_SECONDS instead of
 silently disabling
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A mistyped override (e.g. '90m') would make the '[ -gt ]' comparison fail its
if-condition and silently skip the queue-wait budget — disabling the feature
without warning. Validate up front and exit 1 with a clear message.
---
 .github/scripts/monitor_slurm_job.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh
index c40c43509f..62f70d8e80 100755
--- a/.github/scripts/monitor_slurm_job.sh
+++ b/.github/scripts/monitor_slurm_job.sh
@@ -90,6 +90,12 @@ is_terminal_state() {
 # failure reads as infrastructure (queue starvation), not a code/test failure.
 # Set SLURM_MAX_QUEUE_SECONDS=0 to wait indefinitely (previous behaviour).
 : "${SLURM_MAX_QUEUE_SECONDS:=5400}"   # 90 minutes
+# Fail loudly on a non-integer override rather than silently disabling the
+# budget (a bad `[ -gt ]` comparison would otherwise just skip the check).
+if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then
+  echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2
+  exit 1
+fi
 queue_start=$(date +%s)
 
 abort_queue_starvation() {

From ebc6df244a53699e3e6d0ff2c977ccb469ee5a98 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Tue, 25 Aug 2026 13:08:30 -0400
Subject: [PATCH 4/6] ci: trim verbose comments in Phoenix CI changes

---
 .github/scripts/monitor_slurm_job.sh       | 19 +++++++------------
 .github/scripts/submit-slurm-job.sh        |  8 +++-----
 .github/workflows/common/build-and-test.sh | 14 ++++----------
 .github/workflows/test.yml                 | 11 ++++-------
 4 files changed, 18 insertions(+), 34 deletions(-)

diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh
index 62f70d8e80..cb69e5c016 100755
--- a/.github/scripts/monitor_slurm_job.sh
+++ b/.github/scripts/monitor_slurm_job.sh
@@ -82,16 +82,12 @@ is_terminal_state() {
   esac
 }
 
-# Bound how long the job may sit un-started in the scheduler queue.
-# Under a contended, preemptible QOS (e.g. Phoenix 'embers') a job can stay
-# PENDING for many hours, burning the entire CI job timeout, holding a
-# self-hosted runner slot hostage, and surfacing as an opaque ~8 h 'cancelled'.
-# Give up after a bounded queue wait with a loud, greppable message so the
-# failure reads as infrastructure (queue starvation), not a code/test failure.
-# Set SLURM_MAX_QUEUE_SECONDS=0 to wait indefinitely (previous behaviour).
+# Bound how long a job may sit un-started in the queue. On a preemptible QOS
+# (Phoenix 'embers') a job can stay PENDING for hours, burning the CI job
+# timeout and holding a runner slot; fail early so it reads as queue starvation,
+# not a test failure. 0 = wait indefinitely.
 : "${SLURM_MAX_QUEUE_SECONDS:=5400}"   # 90 minutes
-# Fail loudly on a non-integer override rather than silently disabling the
-# budget (a bad `[ -gt ]` comparison would otherwise just skip the check).
+# Reject a non-integer override rather than silently skipping the budget.
 if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then
   echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2
   exit 1
@@ -116,9 +112,8 @@ unknown_count=0
 while [ ! -f "$output_file" ]; do
   state=$(get_job_state "$job_id")
 
-  # Enforce the queue-wait budget. A job that has actually started
-  # (RUNNING/COMPLETING) but whose output file is merely NFS-delayed is exempt,
-  # so a job that is already doing work is never killed here.
+  # A started job (RUNNING/COMPLETING) whose output file is merely NFS-delayed
+  # is exempt, so work in progress is never killed here.
   if [ "$SLURM_MAX_QUEUE_SECONDS" -gt 0 ]; then
     case "$state" in
       RUNNING|COMPLETING) ;;
diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh
index 2358494000..6803b37669 100755
--- a/.github/scripts/submit-slurm-job.sh
+++ b/.github/scripts/submit-slurm-job.sh
@@ -48,9 +48,8 @@ case "$cluster" in
         qos="embers"
         extra_sbatch="#SBATCH --requeue"
         test_time="03:00:00"
-        # Combined build+test runs both phases in one allocation (one queue
-        # wait instead of two). It needs headroom for the build on top of the
-        # test budget; kept modest so it still backfills well under 'embers'.
+        # Combined build+test needs build headroom on top of the test budget;
+        # kept modest to still backfill under 'embers'.
         buildtest_time="03:30:00"
         bench_time="04:00:00"
         gpu_partition_dynamic=true
@@ -201,8 +200,7 @@ set -x
 cd "\$SLURM_SUBMIT_DIR"
 echo "Running in \$(pwd):"
 
-# Exported so a wrapper script (e.g. build-and-test.sh) can run build.sh and
-# test.sh as child processes that inherit these, not just inline this shell.
+# Exported so wrapper scripts (build-and-test.sh) run child scripts that inherit these.
 export job_slug="$job_slug"
 export job_device="$device"
 export job_interface="$interface"
diff --git a/.github/workflows/common/build-and-test.sh b/.github/workflows/common/build-and-test.sh
index b1f0d3a700..1889889ef9 100755
--- a/.github/workflows/common/build-and-test.sh
+++ b/.github/workflows/common/build-and-test.sh
@@ -1,14 +1,8 @@
 #!/bin/bash
-# Combined build-then-test for a single SLURM allocation.
-# Runs inside a SLURM job via submit-slurm-job.sh. Doing both phases in one
-# allocation means the scheduler queue wait is paid once instead of twice —
-# on a contended, preemptible QOS (Phoenix 'embers') the queue wait, not the
-# work, is what makes CI slow and flaky.
-#
-# Relies on submit-slurm-job.sh exporting the job_* vars (job_device,
-# job_interface, job_cluster, ...) so the child scripts inherit them. cwd is
-# $SLURM_SUBMIT_DIR (the workspace root), so build/ persists from build.sh into
-# test.sh's --no-build run.
+# Build then test in a single SLURM allocation so the scheduler queue wait is
+# paid once instead of twice (Phoenix 'embers' is queue-bound, not work-bound).
+# submit-slurm-job.sh exports the job_* vars so these child scripts inherit
+# them; cwd is the workspace root, so build/ persists into test.sh's --no-build.
 
 set -euo pipefail
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5d765ded08..dc2d4f1163 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -455,8 +455,7 @@ jobs:
         run:  bash .github/workflows/${{ matrix.cluster }}/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
-        # Phoenix builds+tests in one allocation (see "Build & Test" below); every
-        # other cluster keeps the separate build job.
+        # Phoenix builds+tests in one allocation (see "Build & Test"); others build separately.
         if:   ${{ matrix.cluster != 'phoenix' && !(matrix.cluster == 'frontier_amd' && matrix.device == 'gpu') }}
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
@@ -477,8 +476,7 @@ jobs:
           exit $rc
 
       - name: Build & Test
-        # Phoenix: build and test in a single SLURM allocation so the scheduler
-        # queue wait (preemptible 'embers' QOS) is paid once instead of twice.
+        # Phoenix: build+test in one allocation so the 'embers' queue wait is paid once.
         if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build-and-test.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} ${{ matrix.shard }}
 
@@ -511,9 +509,8 @@ jobs:
       - name: Print Logs
         if:   always()
         run:  |
-          # The build slug matches the plain log and, where the build is split per
-          # variant, the build-...-base/-chem ones. The combined slug matches
-          # Phoenix's single build-and-test allocation.
+          # build_slug matches the plain log (and per-variant base/chem builds);
+          # combined_slug matches Phoenix's single build-and-test allocation.
           for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.combined_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do
             [ -f "$f" ] && echo "=== $f ===" && cat "$f"
           done

From 3e3651dee21d7eb9adb3970085615649b8ee1b9d Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Wed, 26 Aug 2026 16:59:15 -0400
Subject: [PATCH 5/6] ci: fix Print Logs failing on Phoenix combined jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 'Print Logs' loop used '[ -f ] && ...', whose last iteration returns
non-zero when the file is absent. On Phoenix combined jobs the loop ends with
test_slug.out, which never exists (Phoenix now emits build-and-test-*.out), so
under 'bash -e' the step exited 1 — failing every Phoenix job, even successful
ones. Use 'if [ -f ]' so a missing file is skipped without failing the step.
---
 .github/workflows/test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index dc2d4f1163..eba8b1d179 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -511,8 +511,10 @@ jobs:
         run:  |
           # build_slug matches the plain log (and per-variant base/chem builds);
           # combined_slug matches Phoenix's single build-and-test allocation.
+          # Use `if` (not `[ -f ] && ...`) so a missing file — e.g. test_slug on
+          # Phoenix's combined jobs — doesn't make the loop exit non-zero under -e.
           for f in ${{ steps.log.outputs.build_slug }}*.out ${{ steps.log.outputs.combined_slug }}.out ${{ steps.log.outputs.test_slug }}.out; do
-            [ -f "$f" ] && echo "=== $f ===" && cat "$f"
+            if [ -f "$f" ]; then echo "=== $f ==="; cat "$f"; fi
           done
 
       - name: Archive Logs

From 28545a4d9b4ea155a8bc12da186eb1090b8883e5 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Wed, 26 Aug 2026 19:10:15 -0500
Subject: [PATCH 6/6] ci: run Frontier CI on the g1 partition, and raise the
 queue-wait budget to 4h (#1767)

---
 .github/scripts/monitor_slurm_job.sh | 10 +++++++++-
 .github/scripts/submit-slurm-job.sh  | 11 ++++++++---
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/monitor_slurm_job.sh b/.github/scripts/monitor_slurm_job.sh
index cb69e5c016..3d4c4a7177 100755
--- a/.github/scripts/monitor_slurm_job.sh
+++ b/.github/scripts/monitor_slurm_job.sh
@@ -86,7 +86,15 @@ is_terminal_state() {
 # (Phoenix 'embers') a job can stay PENDING for hours, burning the CI job
 # timeout and holding a runner slot; fail early so it reads as queue starvation,
 # not a test failure. 0 = wait indefinitely.
-: "${SLURM_MAX_QUEUE_SECONDS:=5400}"   # 90 minutes
+#
+# The budget has to clear a normal bad day on a busy machine, or it converts
+# routine queue pressure into red CI. Frontier's own numbers make the case:
+# over one week, MFC jobs on `batch` waited p50=1m but p90=96m and p95=176m,
+# with a 466m tail. A 90-minute budget cut into that distribution, tripping on
+# 11% of the jobs that did eventually start -- plus the ones that never did.
+# Four hours clears p95 with room to spare while staying well inside the 480m
+# job-level `timeout-minutes`, which remains the real backstop.
+: "${SLURM_MAX_QUEUE_SECONDS:=14400}"   # 4 hours
 # Reject a non-integer override rather than silently skipping the budget.
 if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then
   echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2
diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh
index 6803b37669..16035419bf 100755
--- a/.github/scripts/submit-slurm-job.sh
+++ b/.github/scripts/submit-slurm-job.sh
@@ -62,7 +62,10 @@ case "$cluster" in
         # CFD154; submitting under it now fails outright with "Invalid qos
         # specification". "normal" is the only QOS on this allocation without a
         # one-job-at-a-time cap, so it is the only one that can run the CI
-        # matrix concurrently.
+        # matrix concurrently. Note that the g1 partition carries its own
+        # partition QOS (also named "g1"), which slurmctld applies on its own
+        # when a job lands there. Do not add --qos=g1: CFD154 has no
+        # association with it and sbatch rejects the job outright.
         qos="normal"
         # Let each job's slurmstepd broker its own steps instead of routing
         # every srun through slurmctld. The in-job test suite launches ~1700+
@@ -105,9 +108,11 @@ if [ "$device" = "cpu" ]; then
 #SBATCH --mem-per-cpu=8G"
             ;;
         frontier|frontier_amd)
+            # g1 is a dedicated 64-node carve-out; its nodes are not in batch,
+            # so CI starts promptly instead of queueing behind the machine.
             sbatch_device_opts="\
 #SBATCH -n 32
-#SBATCH -p batch"
+#SBATCH -p g1"
             ;;
     esac
 elif [ "$device" = "gpu" ]; then
@@ -135,7 +140,7 @@ elif [ "$device" = "gpu" ]; then
         frontier|frontier_amd)
             sbatch_device_opts="\
 #SBATCH -n 8
-#SBATCH -p batch"
+#SBATCH -p g1"
             ;;
     esac
 else