diff --git a/action.yml b/action.yml
index af81b58..bbfbc5b 100644
--- a/action.yml
+++ b/action.yml
@@ -153,6 +153,9 @@ runs:
     - name: Run JS tests
       if: ${{ inputs.js == 'true' }}
       shell: bash
+      env:
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+        GITHUB_REPOSITORY: ${{  github.repository }}
       run: |
         echo "Running JS tests"
         if [[ ! -f package.json ]]; then
@@ -216,25 +219,27 @@ runs:
             echo "git diff found modified files when it should not have:"
             echo $GIT_DIFF
             echo "sha1sum of files that are different:"
-            COPIED_COUNT=0
             for FILEPATH in $(git diff --cached --name-only); do
               if [[ -f $FILEPATH ]]; then
                 sha1sum $FILEPATH
-                # Only copy if the file is less than 10 megabytes to prevent malicous behaviour
-                # Only copy a max of 10 files, also to prevent malicious behaviour
-                MEGABYTES=$(ls -l --b=M "$DIST_DIR/$FILEPATH" | cut -d " " -f5 | sed "s/M//")
-                if (($MEGABYTES >= 10)); then
-                  echo "File $FILEPATH is larger than 10 megabytes, not copying"
-                elif (($COPIED_COUNT >= 10)); then
-                  echo "More than 10 files have been copied, not copying $FILEPATH"
-                else
-                  cp "$DIST_DIR/$FILEPATH" artifacts
-                  COPIED_COUNT=$((COPIED_COUNT+1))
-                fi
               fi
             done
-            echo "Files that are different have been copied to artifacts directory"
-            exit 1
+            # silverstripe/silverstripe-admin CMS 4 has a known issue where dist files are
+            # sometimes generated differenty from the same source code on subsequent builds.
+            # This doesn't happen in CMS 5, so presumably it's something to do with and
+            # older version of webpack
+            DO_EXIT="true"
+            if [[ $GITHUB_REPOSITORY == "silverstripe/silverstripe-admin" ]]; then
+              # Get first character of $GITHUB_REF_NAME
+              MAJOR_BRANCH=${GITHUB_REF_NAME:0:1}
+              if [[ $MAJOR_BRANCH == "1" ]]; then
+                echo "Ignoring differences in dist files for CMS 4 version of silverstripe/silverstripe-admin"
+                DO_EXIT="false"
+              fi
+            fi
+            if [[ $DO_EXIT == "true" ]]; then
+              exit 1
+            fi
           fi
         fi
         if [[ $(cat package.json | jq -r '.scripts.test') != 'null' ]]; then