limit jumpbox job concurrency

NVIDIA · Oct 15, 2024 · de43bb4 · de43bb4
1 parent d0800ed
commit de43bb4
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 21 deletions.
diff --git a/.github/workflows/_test_maxtext.yaml b/.github/workflows/_test_maxtext.yaml
@@ -44,6 +44,8 @@ jobs:
       fail-fast: false
 
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
 
     steps:
       - name: Print environment variables
@@ -195,6 +197,8 @@ jobs:
       fail-fast: false
 
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
 
     steps:
       - name: Print environment variables

diff --git a/.github/workflows/_test_pax_rosetta.yaml b/.github/workflows/_test_pax_rosetta.yaml
@@ -37,12 +37,11 @@ jobs:
         - [1, 8, 1, 1]
         - [1, 1, 2, 4]
       fail-fast: false
-
     runs-on: jumpbox
-
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}    
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-pax-single-process-multi-device-te
-
     steps:
       - name: Print environment variables
         run: env
@@ -259,8 +258,9 @@ jobs:
             EVALUATE: true
             ADDITIONAL_ARGS: "--model-type LLaMA70BProxy --evaluate"
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-pax-multi-node-te
     steps:
@@ -457,8 +457,9 @@ jobs:
         - [4, 2, 1, 1]
         - [4, 2, 1, 2]
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-pax-multi-node
     steps:
@@ -649,8 +650,9 @@ jobs:
         PARALLEL_CONFIG:
         - [1, 8, 1, 1]
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-pax-single-node-dropout-te
     steps:
@@ -843,8 +845,9 @@ jobs:
         PARALLEL_CONFIG:
         - [1, 8, 1, 1]
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-pax-single-process-evaluation-te
     steps:

diff --git a/.github/workflows/_test_t5x_rosetta.yaml b/.github/workflows/_test_t5x_rosetta.yaml
@@ -47,8 +47,9 @@ jobs:
             ADDITIONAL_ARGS: ""
             EXTRA_GIN_ARGS: "--gin.train/utils.DatasetConfig.pack=False --gin.train_eval/utils.DatasetConfig.pack=False"
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-t5x-single-process-multi-device
     steps:
@@ -241,8 +242,9 @@ jobs:
             ADDITIONAL_ARGS: "--enable-te 0"
             EXTRA_GIN_ARGS: ""
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-t5x-multi-gpu-multi-node
     steps:
@@ -427,8 +429,9 @@ jobs:
       matrix:
         N_GPU: [8]
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-t5x-vit-single-process-multi-device
     steps:
@@ -597,8 +600,9 @@ jobs:
         N_GPU: [1, 8]
         N_NODE: [1, 2]
       fail-fast: false
-
     runs-on: jumpbox
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     env:
       BADGE_FILENAME_PREFIX: badge-rosetta-t5x-vit-multi-gpu-multi-node
     steps:

diff --git a/.github/workflows/_test_upstream_pax.yaml b/.github/workflows/_test_upstream_pax.yaml
@@ -37,9 +37,9 @@ jobs:
         - [1, 8, 1, 1]
         - [1, 1, 2, 4]
       fail-fast: false
-
     runs-on: jumpbox
-
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Print environment variables
         run: env
@@ -211,9 +211,9 @@ jobs:
             EVALUATE: true
             ADDITIONAL_ARGS: "--model-type LLaMA70BProxy --evaluate"
       fail-fast: false
-
     runs-on: jumpbox
-
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Print environment variables
         run: env
@@ -360,9 +360,9 @@ jobs:
         PARALLEL_CONFIG:
         - [1, 8, 1, 1]
       fail-fast: false
-
     runs-on: jumpbox
-
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Print environment variables
         run: env

diff --git a/.github/workflows/_test_upstream_t5x.yaml b/.github/workflows/_test_upstream_t5x.yaml
@@ -45,9 +45,9 @@ jobs:
             N_GPU: 2
             ADDITIONAL_ARGS: "--enable-fmha 1"
       fail-fast: false
-
     runs-on: jumpbox
-
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Print environment variables
         run: env
@@ -190,9 +190,9 @@ jobs:
             N_NODE: 2
             ADDITIONAL_ARGS: "--enable-fmha 1"
       fail-fast: false
-
     runs-on: jumpbox
-
+    concurrency:
+      group: ${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Print environment variables
         run: env