Add script & case for benchmark utility (#272)

* Add new feature for benchmark utility;

maint/script/benchmark.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+declare THIS_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+declare MODELS=$THIS_SCRIPT_DIR/../../models
+
+create_cuda_container(){
+    docker kill $1 >/dev/null 2>&1 || true
+    docker rm $1 >/dev/null 2>&1 || true
+    docker run --gpus all --name $1 -t -d --net=host -e EXEC_BASH=1 -v $THIS_SCRIPT_DIR/../../:/nnfusion -v $THIS_SCRIPT_DIR/../../../frozenmodels:/frozenmodels -w /nnfusion $2 bash
+}
+
+# check if inside one docker container(for testing)
+if [ -f "/.dockerenv" ]; then
+    $THIS_SCRIPT_DIR/build.sh
+    python3 $THIS_SCRIPT_DIR/../../test/nnfusion/scripts/e2e_tests.py $THIS_SCRIPT_DIR/../../test/nnfusion/scripts/perf.json
+else
+
+    if [ ! -d "$THIS_SCRIPT_DIR/../../models/frozenmodels/" ]; then
+        # prepare models
+        if [ ! -d "$THIS_SCRIPT_DIR/../../../frozenmodels/" ]; then
+            $THIS_SCRIPT_DIR/download_models.sh
+        fi
+    fi
+
+    # use nnfusion_base for build / test cpu
+    create_cuda_container nnfusion_cuda_dev nnfusion/cuda:10.2-cudnn7-devel-ubuntu18.04
+    if [ $? -ne 0 ]; then
+        echo "One or many Docker containers were not built. Run ./build_containers.sh ."
+        exit 1
+    fi
+
+    # build & install
+    # docker exec -t nnfusion_cuda_dev /nnfusion/maint/script/build.sh
+
+    failed=0
+
+    res=`lspci | grep -i nvidia`
+    if [ ! -z "$res" ]; then
+        echo "Launch Cuda container to benchmark:"
+        docker exec -t nnfusion_cuda_dev /nnfusion/maint/script/benchmark.sh &
+    fi
+
+    for job in `jobs -p`
+    do
+        wait $job || let "failed+=1"
+    done
+
+    docker exec -t nnfusion_cuda_dev sh -c "rm -rf /nnfusion/build /nnfusion/nnfusion_rt"
+    docker exec -t nnfusion_cuda_dev sh -c "find /nnfusion -type d -name '__pycache__' | xargs rm -rf"
+
+    if [ ! "$failed" == "0" ]; then
+        echo "Test execution failed."
+        exit 1
+    fi
+
+fi

test/nnfusion/scripts/e2e_tests.py

@@ -179,7 +179,7 @@ def report(self):
 
         hostname = socket.gethostname()
         print("=========================================\n\n")
-        print(hostname + "\tE2E Test report")
+        print(hostname + "\tE2E Model report")
         print("\n\n=========================================\n")
         report = ("\n".join(report_list))
         print(report)

test/nnfusion/scripts/evaluator/e2e_evaluator.py

@@ -4,7 +4,7 @@
 import uuid, os, logging, sys, multiprocessing, tempfile
 
 class E2EEvaluator:
-    def __init__(self, testcase, codegen_folder = "cuda_codegen", default_device = "CUDA", working_foler = ".", nnfusion_cli = "", nnfusion_cli_arg = ""):
+    def __init__(self, testcase, codegen_folder = "cuda_codegen", default_device = "CUDA", working_foler = ".", nnfusion_cli = "", nnfusion_cli_arg = "", perf_mode = False):
         self.codegen_folder = codegen_folder
         self.default_device = default_device
         self.testcase = testcase
@@ -14,6 +14,9 @@ def __init__(self, testcase, codegen_folder = "cuda_codegen", default_device = "
         else:
             self.nnfusion_cli = nnfusion_cli
         self.nnfusion_cli_arg = nnfusion_cli_arg
+
+        self.perf_mode = True
+        self.latency = 0
 
     def load_default_nnfusion_cli(self):
         nnf_clis = [os.path.join(os.path.dirname(os.path.abspath(
@@ -67,6 +70,19 @@ def allclose(self):
             logging.error("%s result missmatch."%self.testcase.casename)
             return False
         return True 
+
+    def exectime(self):
+        code = os.system("cd %s/nnfusion_rt/%s/ && ./main_test > result.txt"%(self.working_foler, self.codegen_folder))
+        if code != 0:
+            logging.error("%s execution failed."%self.testcase.casename)
+            return False
+        if not os.path.exists("%s/nnfusion_rt/%s/result.txt"%(self.working_foler, self.codegen_folder)):
+            logging.error("Failed at compiling phase.")
+            return False
+        result_file = open("%s/nnfusion_rt/%s/result.txt"%(self.working_foler, self.codegen_folder))
+        results = result_file.readlines()
+        latency = self.testcase.latency(results)
+        return latency
 
     def report(self):
         os.system("rm -rf %s/nnfusion_rt"%self.working_foler)
@@ -76,9 +92,14 @@ def report(self):
         if not self.build():
             os.system("rm -rf %s/nnfusion_rt"%self.working_foler)
             return False
-        if not self.allclose():
-            os.system("rm -rf %s/nnfusion_rt"%self.working_foler)
-            return False
+
+        if self.perf_mode is False:
+            if not self.allclose():
+                os.system("rm -rf %s/nnfusion_rt"%self.working_foler)
+                return False
+        else:
+            self.latency = self.exectime()
+
         os.system("rm -rf %s/nnfusion_rt"%self.working_foler)
         return True
 
@@ -92,16 +113,19 @@ def E2EExecutor(TestCases, devname, report_list, nnf, nnf_args):
     for test in TestCases:
         logging.info("Testing " + test.casename)
         if test.valid():
-            eval = E2EEvaluator(test, configs[devname][0], configs[devname][1], tmpdir, nnf, nnf_args)
+            perf_mode = "latency" in dir(test)
+            eval = E2EEvaluator(test, configs[devname][0], configs[devname][1], tmpdir, nnf, nnf_args, perf_mode)
             report = devname + "\t" + test.casename + '\t' + ",".join(test.tags) + "\t";
             if eval.report():
                 report += "Succeed!"
             else:
                 eval = E2EEvaluator(test, configs[devname][0], configs[devname][1], tmpdir)
                 if eval.report():
-                    report += "Succeed!"
+                    report += ",\tSucceed"
                 else:
-                    report += "Failed"
+                    report += ",\tFailed"
+            if eval.perf_mode:
+                report += ",\t" + str(eval.latency)
             logging.info(report)
             report_list.append(report)
     # clean
@@ -114,4 +138,4 @@ def CLIExecutor(info, report_list):
     if os.system(side_cli) == 0:
        report_list.append(side_cli + "\tSucceed!") 
     else:
-       report_list.append(side_cli + "\tFailed") 
+       report_list.append(side_cli + "\tFailed") 

test/nnfusion/scripts/perf.json

@@ -0,0 +1,20 @@
+{
+    "env": {
+      "PYTHONDONTWRITEBYTECODE" : {
+        "set" : 1
+      },
+      "HSA_USERPTR_FOR_PAGED_MEM": {
+        "set": 0
+      },
+      "LD_LIBRARY_PATH": {
+        "append": "/usr/local/lib"
+      },
+      "HIP_VISIBLE_DEVICES" : {
+        "set": 1
+      }
+    },
+    "device_capability": ["CUDA"],
+    "enabled_tags": {
+      "CUDA" : ["perf"]
+    }
+  }

test/nnfusion/scripts/testcase_configs/naive_tests.json

@@ -51,4 +51,4 @@
             "comment": "Naive json descriptor for test case."
         }
     ]
-}
+}

test/nnfusion/scripts/testcases/__init__.py

@@ -4,12 +4,14 @@
 from testcases.testcase import *
 import testcases.naive_cases
 import testcases.cpu_perf_cases
+import testcases.perf_cases
 
 TestCases = list()
 
 tests_load_funtion = {
     "naive_case_single_line": testcases.naive_cases.create_naive_case_single_line, "naive_case_multi_lines": testcases.naive_cases.create_naive_case_multi_lines, 
-    "cpu_perf_case_single_line": testcases.cpu_perf_cases.create_cpu_perf_case_single_line, "cpu_perf_case_single_lines": testcases.cpu_perf_cases.create_cpu_perf_case_multi_lines
+    "cpu_perf_case_single_line": testcases.cpu_perf_cases.create_cpu_perf_case_single_line, "cpu_perf_case_single_lines": testcases.cpu_perf_cases.create_cpu_perf_case_multi_lines,
+    "perf_case" : testcases.perf_cases.create_perf_case
 }
 
 def parse_tests(base_folder, json_data):

test/nnfusion/scripts/testcases/perf_cases.py

@@ -0,0 +1,34 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import logging
+import os
+from testcases.testcase import *
+
+
+class PerfOutput(TestCase):
+    def __init__(self, casename, filename, tags, flag):
+        self.casename = casename
+        self.filename = filename
+        self.tags = tags
+        self.flag = flag
+
+    # Get data from output of main_test
+    def allclose(self, raw_strdata):
+        return True
+
+    def latency(self, raw_strdata):
+        real_time = float(raw_strdata[-1].strip("\n").split(" ")[-2][:-1])
+        return real_time
+        #return raw_strdata[-1]
+
+
+def create_perf_case(base_folder, json_data):
+    testcase = json_data["testcase"]
+    tags = json_data["tag"]
+    filename = os.path.join(base_folder, json_data["filename"])
+    flag = ""
+    if "flag" in json_data:
+        flag = json_data["flag"]
+
+    return PerfOutput(testcase, filename, tags, flag)