From e8043a5b50bc3b82cf8f91d20f3e2150ce8b01c5 Mon Sep 17 00:00:00 2001
From: Olli Lupton <olupton@nvidia.com>
Date: Thu, 10 Oct 2024 16:18:46 +0200
Subject: [PATCH] Improve TE test reporting, increase timeout (#1084)

- Increase timeout as A100 tests take >1 hour to run
- Report the test failures/errors, not collection failures/errors
---
 .github/workflows/_ci.yaml        | 5 +++--
 .github/workflows/_test_unit.yaml | 7 ++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/_ci.yaml b/.github/workflows/_ci.yaml
index 426764323..17ad5f06b 100644
--- a/.github/workflows/_ci.yaml
+++ b/.github/workflows/_ci.yaml
@@ -528,13 +528,14 @@ jobs:
       STATISTICS_SCRIPT: |
         summary_line=$(tail -n1 test-te.log)
         errors=$(echo $summary_line | grep -oE '[0-9]+ error' | awk '{print $1} END { if (!NR) print 0}')
-        passed_tests=$(cat pytest-report.jsonl | jq -r 'select(."$report_type" == "CollectReport" and .outcome == "passed") | .outcome' | wc -l)
-        failed_tests=$(cat pytest-report.jsonl | jq -r 'select(."$report_type" == "CollectReport" and .outcome == "failed") | .outcome' | wc -l)          
+        passed_tests=$(cat pytest-report.jsonl | jq -r 'select(."$report_type" == "TestReport" and .when == "call" and .outcome == "passed") | .outcome' | wc -l)
+        failed_tests=$(cat pytest-report.jsonl | jq -r 'select(."$report_type" == "TestReport" and .when == "call" and .outcome == "failed") | .outcome' | wc -l)
         total_tests=$((failed_tests + passed_tests))
         echo "TOTAL_TESTS=${total_tests}" >> $GITHUB_OUTPUT
         echo "ERRORS=${errors}" >> $GITHUB_OUTPUT
         echo "PASSED_TESTS=${passed_tests}" >> $GITHUB_OUTPUT
         echo "FAILED_TESTS=${failed_tests}" >> $GITHUB_OUTPUT
+      TIMEOUT_MINUTES: 120
       ARTIFACTS: |
         test-te.log
         pytest-report.jsonl
diff --git a/.github/workflows/_test_unit.yaml b/.github/workflows/_test_unit.yaml
index 2edfb2249..fa29557e0 100644
--- a/.github/workflows/_test_unit.yaml
+++ b/.github/workflows/_test_unit.yaml
@@ -19,6 +19,10 @@ on:
         type: string
         description: 'Test artifacts to collect'
         required: false
+      TIMEOUT_MINUTES:
+        type: number
+        description: 'Maximum test runtime, in minutes'
+        default: 60
 
 jobs:
   runner:
@@ -26,7 +30,7 @@ jobs:
     with:
       NAME: "A100"
       LABELS: "A100,${{ github.run_id }}"
-      TIME: "01:00:00"
+      TIME: "${{ inputs.TIMEOUT_MINUTES }}:00"
     secrets: inherit
 
   run-unit-test:
@@ -67,6 +71,7 @@ jobs:
       - name: Run tests
         shell: bash -x -e {0}
         continue-on-error: true
+        timeout-minutes: ${{ inputs.TIMEOUT_MINUTES }}
         run: |
           ${{ inputs.EXECUTE }}