diff --git a/.github/workflows/llm_unit_tests.yml b/.github/workflows/llm_unit_tests.yml
index 9c1bf86d86b..e9a531002fc 100644
--- a/.github/workflows/llm_unit_tests.yml
+++ b/.github/workflows/llm_unit_tests.yml
@@ -37,30 +37,33 @@ on:
 jobs:
   llm-cpp-build:
     uses: ./.github/workflows/llm-binary-build.yml
+  setup-python-version:
+    runs-on: [self-hosted, llm, ubuntu-20.04-lts]
+    outputs:
+      python-version: ${{ steps.setup-python-version.outputs.python-version }}
+    steps:
+      - name: setup-python-version
+        id: setup-python-version
+        run: |
+          if ${{ github.event_name == 'pull_request' }}; then
+            python_version='["3.9"]'
+          else
+            python_version='["3.9", "3.10", "3.11"]'
+          fi
+          list=$(echo ${python_version} | jq -c)
+          echo "python-version=${list}" >> "$GITHUB_OUTPUT"
   llm-unit-test:
-    needs: llm-cpp-build
+    needs: [setup-python-version, llm-cpp-build]
     strategy:
       fail-fast: false
       matrix:
+        os: [windows, ubuntu-20.04-lts]
+        python-version: ${{ fromJson(needs.setup-python-version.outputs.python-version) }}
         include:
           - os: windows
             instruction: AVX-VNNI-UT
-            python-version: "3.9"
-          - os: ubuntu-20.04-lts
-            instruction: avx512
-            python-version: "3.9"
-          - os: windows
-            instruction: AVX-VNNI-UT
-            python-version: "3.10"
-          - os: ubuntu-20.04-lts
-            instruction: avx512
-            python-version: "3.10"
-          - os: windows
-            instruction: AVX-VNNI-UT
-            python-version: "3.11"
           - os: ubuntu-20.04-lts
             instruction: avx512
-            python-version: "3.11"
     runs-on: [self-hosted, llm, "${{matrix.instruction}}", "${{matrix.os}}"]
     env:
       THREAD_NUM: 24
@@ -204,23 +207,12 @@ jobs:
           pip install -U pandas==2.0.3
           bash python/llm/test/run-llm-langchain-tests.sh
   llm-unit-test-on-arc:
-    needs: llm-cpp-build
+    needs: [setup-python-version, llm-cpp-build]
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - pytorch-version: "2.1"
-            python-version: "3.9"
-          - pytorch-version: "2.1"
-            python-version: "3.10"
-          - pytorch-version: "2.1"
-            python-version: "3.11"
-          - pytorch-version: "2.0"
-            python-version: "3.9"
-          - pytorch-version: "2.0"
-            python-version: "3.10"
-          - pytorch-version: "2.0"
-            python-version: "3.11"
+        pytorch-version: ['2.1', '2.0']
+        python-version: ${{ fromJson(needs.setup-python-version.outputs.python-version) }}
     runs-on: [self-hosted, llm, arc]
     env:
       OMP_NUM_THREADS: 16