Replace CI UT test with pytest (#1593)

Signed-off-by: chensuyue <suyue.chen@intel.com>

.azure-pipelines/scripts/ut/3x/run_3x_ort.sh

@@ -6,31 +6,28 @@ echo "${test_case}"
 # install requirements
 echo "set up UT env..."
 pip install -r /neural-compressor/test/3x/onnxrt/requirements.txt
-pip install coverage
+pip install pytest-cov
+pip install pytest-html
 pip list
 
 export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_ort
 inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
-cd /neural-compressor/test || exit 1
-find ./3x/onnxrt/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
-find ./3x/common/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'>> run.sh
+cd /neural-compressor/test/3x || exit 1
+rm -rf torch
+rm -rf tensorflow
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}
 ut_log_name=${LOG_DIR}/ut_3x_ort.log
+pytest --cov="${inc_path}" -vs --disable-warnings --html=report.html --self-contained-html . 2>&1 | tee -a ${ut_log_name}
 
-echo "cat run.sh..."
-sort run.sh -o run.sh
-cat run.sh | tee ${ut_log_name}
-echo "------UT start-------"
-bash -x run.sh 2>&1 | tee -a ${ut_log_name}
 cp .coverage ${LOG_DIR}/.coverage
+cp report.html ${LOG_DIR}/
 
-echo "------UT end -------"
-
-if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "core dumped" ${ut_log_name}) != 0 ] \
-|| [ $(grep -c "ModuleNotFoundError:" ${ut_log_name}) != 0 ] || [ $(grep -c "ImportError:" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then
-    echo "Find errors in UT test, please check the output..."
+if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
+    echo "Find errors in pytest case, please check the output..."
+    echo "Please search for '== FAILURES ==' or '== ERRORS =='"
     exit 1
 fi
+
 echo "UT finished successfully! "

.azure-pipelines/scripts/ut/3x/run_3x_pt.sh

@@ -6,59 +6,27 @@ echo "${test_case}"
 # install requirements
 echo "set up UT env..."
 pip install -r /neural-compressor/test/3x/torch/requirements.txt
-pip install coverage
+pip install pytest-cov
+pip install pytest-html
 pip list
 
 export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_pt
 inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
 cd /neural-compressor/test/3x || exit 1
-grep -lrv "import pytest" --include="test*.py" ./torch | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run_unittest.sh
-grep -lrv "import pytest" --include="test*.py" ./common | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'>> run_unittest.sh
-grep -lr "import pytest" --include="test*.py" ./torch | sed 's,\.\/,coverage run --source='"${inc_path}"' --append -m pytest --disable-warnings -v ,g' > run_pytest.sh
-grep -lr "import pytest" --include="test*.py" ./common | sed 's,\.\/,coverage run --source='"${inc_path}"' --append -m pytest --disable-warnings -v ,g'>> run_pytest.sh
+rm -rf tensorflow
+rm -rf onnxrt
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}
 ut_log_name=${LOG_DIR}/ut_3x_pt.log
-
-# unittest and pytest has some incompatible issue, so separate the test.
-echo "cat run_unittest.sh..."
-sort run_unittest.sh -o run_unittest.sh
-cat run_unittest.sh | tee ${ut_log_name}
-echo "------unittest start-------"
-bash -x run_unittest.sh 2>&1 | tee -a ${ut_log_name}
-echo "------unittest end -------"
-
-if [ -s run_pytest.sh ]; then
-    echo "cat run_pytest.sh..."
-    sort run_pytest.sh -o run_pytest.sh
-    cat run_pytest.sh | tee -a ${ut_log_name}
-    echo "------pytest start-------"
-    bash -x run_pytest.sh 2>&1 | tee -a ${ut_log_name}
-    echo "------pytest end -------"
-fi
+pytest --cov="${inc_path}" -vs --disable-warnings --html=report.html --self-contained-html . 2>&1 | tee -a ${ut_log_name}
 
 cp .coverage ${LOG_DIR}/.coverage
+cp report.html ${LOG_DIR}/
 
-ut_status="passed"
-# check unittest issue
-if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "core dumped" ${ut_log_name}) != 0 ] \
-|| [ $(grep -c "ModuleNotFoundError:" ${ut_log_name}) != 0 ] || [ $(grep -c "ImportError:" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then
-    echo "Find errors in unittest case, please check the output..."
-    echo "Please search for 'FAILED' or 'core dumped' or 'ModuleNotFoundError:' or 'ImportError:'"
-    ut_status="failed"
-fi
-
-# check pytest issue
-if [ -s run_pytest.sh ]; then
-    if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed ' ${ut_log_name}) == 0 ]; then
-        echo "Find errors in pytest case, please check the output..."
-        echo "Please search for '== FAILURES ==' or '== ERRORS =='"
-        ut_status="failed"
-    fi
-fi
-
-if [ "$ut_status" = "failed" ]; then
+if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
+    echo "Find errors in pytest case, please check the output..."
+    echo "Please search for '== FAILURES ==' or '== ERRORS =='"
     exit 1
 fi
 

.azure-pipelines/scripts/ut/3x/run_3x_tf.sh

@@ -6,39 +6,41 @@ echo "${test_case}"
 # install requirements
 echo "set up UT env..."
 pip install -r /neural-compressor/test/3x/tensorflow/requirements.txt
-pip install coverage
+pip install pytest-cov
+pip install pytest-html
+pip install pytest-html-merger
 pip list
 
 export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_tf
 inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
-cd /neural-compressor/test || exit 1
-find ./3x/tensorflow/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
-find ./3x/common/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'>> run.sh
-sed -i '/tensorflow\/keras\//d' run.sh
-
-find ./3x/tensorflow/keras/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run_keras.sh
+cd /neural-compressor/test/3x || exit 1
+rm -rf torch
+rm -rf onnxrt
+mv tensorflow/keras ../3x_keras
 
 LOG_DIR=/neural-compressor/log_dir
 mkdir -p ${LOG_DIR}
 ut_log_name=${LOG_DIR}/ut_3x_tf.log
+pytest --cov="${inc_path}" -vs --disable-warnings --html=report_tf_quant.html --self-contained-html ./tensorflow/quantization 2>&1 | tee -a ${ut_log_name}
+rm -rf tensorflow/quantization
+pytest --cov="${inc_path}" --cov-append -vs --disable-warnings --html=report_tf.html --self-contained-html ./tensorflow 2>&1 | tee -a ${ut_log_name}
 
-echo "cat run.sh..."
-sort run.sh -o run.sh
-cat run.sh | tee ${ut_log_name}
-echo "cat run_keras.sh..."
-sort run_keras.sh -o run_keras.sh
-cat run_keras.sh | tee ${ut_log_name}
-echo "------UT start-------"
-bash -x run.sh 2>&1 | tee -a ${ut_log_name}
+rm -rf tensorflow/*
+mv ../3x_keras tensorflow/keras
 pip install intel-extension-for-tensorflow[cpu]
-bash -x run_keras.sh 2>&1 | tee -a ${ut_log_name}
-cp .coverage ${LOG_DIR}/.coverage
+pytest --cov="${inc_path}" --cov-append -vs --disable-warnings --html=report_keras.html --self-contained-html ./tensorflow/keras 2>&1 | tee -a ${ut_log_name}
 
-echo "------UT end -------"
+mkdir -p report
+mv *.html report
+pytest_html_merger -i ./report -o ./report.html
 
-if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "core dumped" ${ut_log_name}) != 0 ] \
-|| [ $(grep -c "ModuleNotFoundError:" ${ut_log_name}) != 0 ] || [ $(grep -c "ImportError:" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then
-    echo "Find errors in UT test, please check the output..."
+cp .coverage ${LOG_DIR}/.coverage
+cp report.html ${LOG_DIR}/
+
+if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
+    echo "Find errors in pytest case, please check the output..."
+    echo "Please search for '== FAILURES ==' or '== ERRORS =='"
     exit 1
 fi
+
 echo "UT finished successfully! "

.azure-pipelines/scripts/ut/env_setup.sh

@@ -105,6 +105,7 @@ fi
 # test deps
 pip install coverage
 pip install pytest
+pip install pytest-html
 
 pip list
 echo "[DEBUG] list pipdeptree..."

.azure-pipelines/ut-3x-ort.yml

@@ -11,8 +11,10 @@ pr:
       - neural_compressor/common
       - neural_compressor/onnxrt
       - test/3x/onnxrt
+      - test/3x/common
       - setup.py
       - requirements_ort.txt
+      - .azure-pipelines/scripts/ut/3x/run_3x_ort.sh
 
 pool: ICX-16C
 

.azure-pipelines/ut-3x-pt.yml

@@ -14,7 +14,7 @@ pr:
       - test/3x/common
       - setup.py
       - requirements_pt.txt
-      - .azure-pipelines/scripts/ut/3x/collect_log_3x.sh
+      - .azure-pipelines/scripts/ut/3x/run_3x_pt.sh
 
 pool: ICX-16C
 

.azure-pipelines/ut-3x-tf.yml

@@ -11,8 +11,10 @@ pr:
       - neural_compressor/common
       - neural_compressor/tensorflow
       - test/3x/tensorflow
+      - test/3x/common
       - setup.py
       - requirements_tf.txt
+      - .azure-pipelines/scripts/ut/3x/run_3x_tf.sh
 
 pool: ICX-16C
 

test/3x/torch/amp/test_fp8.py → test/3x/torch/amp/test_fp8_amp.py

@@ -6,10 +6,10 @@
 import torch
 
 from neural_compressor.torch.amp import autocast
-from neural_compressor.torch.utils import is_hpex_available, logger
+from neural_compressor.torch.utils import is_hpex_available
 
-if not is_hpex_available():
-    exit()
+# if not is_hpex_available():
+#     exit()
 
 
 class M(torch.nn.Module):

test/3x/torch/quantization/fp8/test_fp8.py

@@ -1,28 +1,26 @@
 import copy
-import os
 import shutil
 import unittest
 
-from neural_compressor.torch.utils import is_hpex_available
-
-if not is_hpex_available():
-    exit()
 import torch
 
-from neural_compressor.torch.algorithms.habana_fp8 import quantize_dynamic
-from neural_compressor.torch.algorithms.habana_fp8.modules import (
-    FP8BatchMatmul,
-    FP8DynamicBatchMatmul,
-    FP8DynamicLinear,
-    FP8DynamicMatmul,
-    FP8Linear,
-    FP8Matmul,
-)
-from neural_compressor.torch.quantization import quantize
-from neural_compressor.torch.quantization.config import FP8QConfig, get_default_fp8_qconfig
-from neural_compressor.torch.quantization.modules import BatchMatmul, Matmul
-
-torch.set_grad_enabled(False)
+from neural_compressor.torch.utils import is_hpex_available
+
+if is_hpex_available():
+    from neural_compressor.torch.algorithms.habana_fp8 import quantize_dynamic
+    from neural_compressor.torch.algorithms.habana_fp8.modules import (
+        FP8BatchMatmul,
+        FP8DynamicBatchMatmul,
+        FP8DynamicLinear,
+        FP8DynamicMatmul,
+        FP8Linear,
+        FP8Matmul,
+    )
+    from neural_compressor.torch.quantization import quantize
+    from neural_compressor.torch.quantization.config import FP8QConfig, get_default_fp8_qconfig
+    from neural_compressor.torch.quantization.modules import BatchMatmul, Matmul
+
+    torch.set_grad_enabled(False)
 
 
 class M(torch.nn.Module):