From cb207540a51d8fb7fdd12d52ba434ecf2419cab0 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 13:36:06 -0700 Subject: [PATCH 01/15] Combine all json into single summary at the end --- .github/workflows/_finalize.yaml | 65 +++++++++++++++++------------ .github/workflows/_sitrep_mgmn.yaml | 1 + 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/.github/workflows/_finalize.yaml b/.github/workflows/_finalize.yaml index 936afccec..954abd021 100644 --- a/.github/workflows/_finalize.yaml +++ b/.github/workflows/_finalize.yaml @@ -136,39 +136,50 @@ jobs: cat "$f" | jq -r '.summary' | tee -a $GITHUB_STEP_SUMMARY done - - name: Concatenate all sitreps + - name: Concatenate all json data shell: bash -x -e {0} run: | - # combine all sitreps files into a single file, where each sitrep json sits - # in a field named by the folder that contained it - echo "[" >> fsitrep.json - sitrep_files=($(find . -name "sitrep.json")) - total_files=${#sitrep_files[@]} - - # parse all the sitrep files and consolidate them into one - for ((i=0; i> fsitrep.json - if [ $i != $((total_files-1)) ]; then - echo -n "," >> fsitrep.json + # Check if the metric_summary.json file exists and read it, else use default + if [ -f "$dir/metric_summary.json" ]; then + metricSummary=$(<"$dir/metric_summary.json") fi + + # Use Bash globbing to find the *-status.json file and read it, else use default + statusFiles=($dir*-status.json) + if [ -f "${statusFiles[0]}" ]; then + status=$(<"${statusFiles[0]}") + fi + + # Use jq to merge the JSON data + combinedJson=$(jq --arg dirName "$dirName" \ + --argjson sitrep "$sitrep" \ + --argjson status "$status" \ + --argjson metricSummary "$metricSummary" \ + '.[$dirName] = {"sitrep": $sitrep, "status": $status, "metric_summary": $metricSummary}' <<<"$combinedJson") done - echo "]" >> fsitrep.json - mv fsitrep.json sitrep.json - + + # Output the combined JSON to the file, nicely formatted + echo "$combinedJson" | jq '.' > "$output" + mv combined.json sitrep.json + - name: Upload training logs as artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/_sitrep_mgmn.yaml b/.github/workflows/_sitrep_mgmn.yaml index 45d67898b..fa6abf1f0 100644 --- a/.github/workflows/_sitrep_mgmn.yaml +++ b/.github/workflows/_sitrep_mgmn.yaml @@ -149,3 +149,4 @@ jobs: path: | sitrep.json ${{ inputs.BADGE_FILENAME }} + metrics_summary.json From c2e47b4545493abbc1a5c5aef31e17cba0af3a80 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 15:03:43 -0700 Subject: [PATCH 02/15] Try _sandbox.yaml --- .github/workflows/_sandbox.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 7b90b72ca..8d4b0de5e 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -25,7 +25,7 @@ jobs: a smooth integration process once the changes are ready to be merged. Usage: - + 1. In your development branch, modify the sandbox.yml workflow file to include the new actions you want to test. Make sure to commit the changes to the development branch. @@ -39,3 +39,9 @@ jobs: revert the changes to the sandbox.yml file in the main branch to keep it empty for future testing. EOF + + + finalize: + if: "!cancelled()" + uses: ./.github/workflows/_finalize.yaml + secrets: inherit From 146eba664ce5475dd50df10a1a0dd39d1f157946 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:36:45 -0700 Subject: [PATCH 03/15] Hardcode a download artifacts step --- .github/workflows/_sandbox.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 8d4b0de5e..dbf2deb4f 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -40,6 +40,20 @@ jobs: keep it empty for future testing. EOF + download-artifacts: + runs-on: ubuntu-22.04 + steps: + - name: Download + run: | + UTIL_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + bash ${UTIL_DIR}/download_artifacts.sh 8449523553 + + - name: Upload metrics test json logs + uses: actions/upload-artifact@v4 + with: + name: all-artifacts + path: | + * finalize: if: "!cancelled()" From 3a1eda507484fd0cd678a4cc4f99c70478d99bac Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:39:36 -0700 Subject: [PATCH 04/15] Change path --- .github/workflows/_sandbox.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index dbf2deb4f..bb56abe34 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -45,8 +45,8 @@ jobs: steps: - name: Download run: | - UTIL_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - bash ${UTIL_DIR}/download_artifacts.sh 8449523553 + pwd; ls; + bash workflows/scripts/download_artifacts.sh 8449523553 - name: Upload metrics test json logs uses: actions/upload-artifact@v4 From 4a2ca24cf977450eab2da5cfe32960e8b861e122 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:41:12 -0700 Subject: [PATCH 05/15] Fix path again --- .github/workflows/_sandbox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index bb56abe34..aa48a5a5f 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -46,7 +46,7 @@ jobs: - name: Download run: | pwd; ls; - bash workflows/scripts/download_artifacts.sh 8449523553 + bash .github/workflows/baselines/download_artifacts.sh 8449523553 - name: Upload metrics test json logs uses: actions/upload-artifact@v4 From 6c0a0df2fb4689e2bb21d8376a77747c6bd498a4 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:43:25 -0700 Subject: [PATCH 06/15] Checkout before running --- .github/workflows/_sandbox.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index aa48a5a5f..86213eb0a 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -43,7 +43,10 @@ jobs: download-artifacts: runs-on: ubuntu-22.04 steps: - - name: Download + - name: Check out the repository under ${GITHUB_WORKSPACE} + uses: actions/checkout@v4 + + - name: Download run: | pwd; ls; bash .github/workflows/baselines/download_artifacts.sh 8449523553 From 2385a64b80fa9665f74fab7413f9a39ab5b34ca2 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:43:45 -0700 Subject: [PATCH 07/15] Fix typo --- .github/workflows/_sandbox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 86213eb0a..818b1bb10 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -46,7 +46,7 @@ jobs: - name: Check out the repository under ${GITHUB_WORKSPACE} uses: actions/checkout@v4 - - name: Download + - name: Download run: | pwd; ls; bash .github/workflows/baselines/download_artifacts.sh 8449523553 From b7ced5151daf666b7b91978a04e58eb2ebee347a Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:45:14 -0700 Subject: [PATCH 08/15] Pass token --- .github/workflows/_sandbox.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 818b1bb10..854d6eaad 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -50,6 +50,8 @@ jobs: run: | pwd; ls; bash .github/workflows/baselines/download_artifacts.sh 8449523553 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload metrics test json logs uses: actions/upload-artifact@v4 From eb77d9c8eb222cf171b6defa6bf21691ea8a326a Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:48:10 -0700 Subject: [PATCH 09/15] Change the uploaded path --- .github/workflows/_sandbox.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 854d6eaad..03332c43f 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -49,7 +49,8 @@ jobs: - name: Download run: | pwd; ls; - bash .github/workflows/baselines/download_artifacts.sh 8449523553 + bash .github/workflows/baselines/download_artifacts.sh 8449523553; + ls -l env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -58,7 +59,7 @@ jobs: with: name: all-artifacts path: | - * + 8449523553/* finalize: if: "!cancelled()" From 9cb9d0044e64cae267bee1d987dcf6a9a30e3c79 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 18:50:15 -0700 Subject: [PATCH 10/15] Add needs for finalize --- .github/workflows/_sandbox.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 03332c43f..100165bc4 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -62,6 +62,7 @@ jobs: 8449523553/* finalize: + needs: [download-artifacts] if: "!cancelled()" uses: ./.github/workflows/_finalize.yaml secrets: inherit From 79bae2ce5d4e2b0a205ed26ae8b85db2fddba6a5 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 19:23:18 -0700 Subject: [PATCH 11/15] Slight modification to parsing --- .github/workflows/_finalize.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_finalize.yaml b/.github/workflows/_finalize.yaml index 954abd021..11f9989b4 100644 --- a/.github/workflows/_finalize.yaml +++ b/.github/workflows/_finalize.yaml @@ -145,7 +145,8 @@ jobs: # Loop over subdirectories in the current directory for dir in */; do - dirName="${dir%/}" && [ -d "$dir" ] || continue + echo $dir + dirName=$(basename $dir) && [ -d "$dir" ] || continue # Initialize default empty JSON objects sitrep="{}" @@ -163,9 +164,11 @@ jobs: fi # Use Bash globbing to find the *-status.json file and read it, else use default - statusFiles=($dir*-status.json) - if [ -f "${statusFiles[0]}" ]; then - status=$(<"${statusFiles[0]}") + statusFiles=("$dir"/*-status.json) + statusFile=${statusFiles[0]} + if [ -f $statusFile ]; then + echo "Found status file: $statusFile" + status=$(<"${statusFile}") fi # Use jq to merge the JSON data From 3da7870cf3fa6a84ee4a7e4fcadf5bde0811c57a Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Wed, 27 Mar 2024 19:28:20 -0700 Subject: [PATCH 12/15] Temp path --- .github/workflows/_finalize.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_finalize.yaml b/.github/workflows/_finalize.yaml index 11f9989b4..67b9a3978 100644 --- a/.github/workflows/_finalize.yaml +++ b/.github/workflows/_finalize.yaml @@ -144,7 +144,7 @@ jobs: combinedJson="{}" # Loop over subdirectories in the current directory - for dir in */; do + for dir in all-artifacts/*; do echo $dir dirName=$(basename $dir) && [ -d "$dir" ] || continue From 733237c665b3830e9c0ae2f6e97298bd7edc0993 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Mon, 8 Apr 2024 13:59:26 -0700 Subject: [PATCH 13/15] Remove unused step --- .github/workflows/_finalize.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/_finalize.yaml b/.github/workflows/_finalize.yaml index 67b9a3978..9475a5a05 100644 --- a/.github/workflows/_finalize.yaml +++ b/.github/workflows/_finalize.yaml @@ -129,13 +129,6 @@ jobs: - name: Download artifacts uses: actions/download-artifact@v4 - - name: Write output to step summary - shell: bash -x -e {0} - run: | - find -name "sitrep.json" | while read -s f; do - cat "$f" | jq -r '.summary' | tee -a $GITHUB_STEP_SUMMARY - done - - name: Concatenate all json data shell: bash -x -e {0} run: | From 4d753fd58244c9e6c25364eb74b7944b0d5378bf Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Mon, 8 Apr 2024 14:44:21 -0700 Subject: [PATCH 14/15] Change loop syntax --- .github/workflows/_finalize.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_finalize.yaml b/.github/workflows/_finalize.yaml index 9475a5a05..c22ecf127 100644 --- a/.github/workflows/_finalize.yaml +++ b/.github/workflows/_finalize.yaml @@ -137,7 +137,7 @@ jobs: combinedJson="{}" # Loop over subdirectories in the current directory - for dir in all-artifacts/*; do + while IFS= read -r dir; do echo $dir dirName=$(basename $dir) && [ -d "$dir" ] || continue @@ -160,7 +160,6 @@ jobs: statusFiles=("$dir"/*-status.json) statusFile=${statusFiles[0]} if [ -f $statusFile ]; then - echo "Found status file: $statusFile" status=$(<"${statusFile}") fi @@ -170,7 +169,7 @@ jobs: --argjson status "$status" \ --argjson metricSummary "$metricSummary" \ '.[$dirName] = {"sitrep": $sitrep, "status": $status, "metric_summary": $metricSummary}' <<<"$combinedJson") - done + done < <(find . -maxdepth 1 -type d) # Output the combined JSON to the file, nicely formatted echo "$combinedJson" | jq '.' > "$output" From e6906825f656d043959e366dbe064289e2de5310 Mon Sep 17 00:00:00 2001 From: Clive Unger Date: Tue, 9 Apr 2024 14:06:12 -0700 Subject: [PATCH 15/15] Revert sandbox back to original form. --- .github/workflows/_sandbox.yaml | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 100165bc4..64eb397ff 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -38,31 +38,4 @@ jobs: merge the development branch into the main branch. Remember to revert the changes to the sandbox.yml file in the main branch to keep it empty for future testing. - EOF - - download-artifacts: - runs-on: ubuntu-22.04 - steps: - - name: Check out the repository under ${GITHUB_WORKSPACE} - uses: actions/checkout@v4 - - - name: Download - run: | - pwd; ls; - bash .github/workflows/baselines/download_artifacts.sh 8449523553; - ls -l - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload metrics test json logs - uses: actions/upload-artifact@v4 - with: - name: all-artifacts - path: | - 8449523553/* - - finalize: - needs: [download-artifacts] - if: "!cancelled()" - uses: ./.github/workflows/_finalize.yaml - secrets: inherit + EOF \ No newline at end of file