From 10213a555134db295b9f6f330abb7537003bf9ea Mon Sep 17 00:00:00 2001 From: Wenxiang Hu Date: Mon, 24 May 2021 10:48:25 +0800 Subject: [PATCH] Add script & case for benchmark utility (#272) * Add new feature for benchmark utility; --- maint/script/benchmark.sh | 59 +++++++++++++++++++ test/nnfusion/scripts/e2e_tests.py | 2 +- .../scripts/evaluator/e2e_evaluator.py | 40 ++++++++++--- test/nnfusion/scripts/perf.json | 20 +++++++ .../scripts/testcase_configs/naive_tests.json | 2 +- test/nnfusion/scripts/testcases/__init__.py | 4 +- test/nnfusion/scripts/testcases/perf_cases.py | 34 +++++++++++ 7 files changed, 150 insertions(+), 11 deletions(-) create mode 100755 maint/script/benchmark.sh create mode 100644 test/nnfusion/scripts/perf.json create mode 100644 test/nnfusion/scripts/testcases/perf_cases.py diff --git a/maint/script/benchmark.sh b/maint/script/benchmark.sh new file mode 100755 index 000000000..7babdd02c --- /dev/null +++ b/maint/script/benchmark.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +declare THIS_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +declare MODELS=$THIS_SCRIPT_DIR/../../models + +create_cuda_container(){ + docker kill $1 >/dev/null 2>&1 || true + docker rm $1 >/dev/null 2>&1 || true + docker run --gpus all --name $1 -t -d --net=host -e EXEC_BASH=1 -v $THIS_SCRIPT_DIR/../../:/nnfusion -v $THIS_SCRIPT_DIR/../../../frozenmodels:/frozenmodels -w /nnfusion $2 bash +} + +# check if inside one docker container(for testing) +if [ -f "/.dockerenv" ]; then + $THIS_SCRIPT_DIR/build.sh + python3 $THIS_SCRIPT_DIR/../../test/nnfusion/scripts/e2e_tests.py $THIS_SCRIPT_DIR/../../test/nnfusion/scripts/perf.json +else + + if [ ! -d "$THIS_SCRIPT_DIR/../../models/frozenmodels/" ]; then + # prepare models + if [ ! -d "$THIS_SCRIPT_DIR/../../../frozenmodels/" ]; then + $THIS_SCRIPT_DIR/download_models.sh + fi + fi + + # use nnfusion_base for build / test cpu + create_cuda_container nnfusion_cuda_dev nnfusion/cuda:10.2-cudnn7-devel-ubuntu18.04 + if [ $? -ne 0 ]; then + echo "One or many Docker containers were not built. Run ./build_containers.sh ." + exit 1 + fi + + # build & install + # docker exec -t nnfusion_cuda_dev /nnfusion/maint/script/build.sh + + failed=0 + + res=`lspci | grep -i nvidia` + if [ ! -z "$res" ]; then + echo "Launch Cuda container to benchmark:" + docker exec -t nnfusion_cuda_dev /nnfusion/maint/script/benchmark.sh & + fi + + for job in `jobs -p` + do + wait $job || let "failed+=1" + done + + docker exec -t nnfusion_cuda_dev sh -c "rm -rf /nnfusion/build /nnfusion/nnfusion_rt" + docker exec -t nnfusion_cuda_dev sh -c "find /nnfusion -type d -name '__pycache__' | xargs rm -rf" + + if [ ! "$failed" == "0" ]; then + echo "Test execution failed." + exit 1 + fi + +fi diff --git a/test/nnfusion/scripts/e2e_tests.py b/test/nnfusion/scripts/e2e_tests.py index 85beb0110..40ddf4fe9 100644 --- a/test/nnfusion/scripts/e2e_tests.py +++ b/test/nnfusion/scripts/e2e_tests.py @@ -179,7 +179,7 @@ def report(self): hostname = socket.gethostname() print("=========================================\n\n") - print(hostname + "\tE2E Test report") + print(hostname + "\tE2E Model report") print("\n\n=========================================\n") report = ("\n".join(report_list)) print(report) diff --git a/test/nnfusion/scripts/evaluator/e2e_evaluator.py b/test/nnfusion/scripts/evaluator/e2e_evaluator.py index 0bb43d31a..d8f45ca7a 100644 --- a/test/nnfusion/scripts/evaluator/e2e_evaluator.py +++ b/test/nnfusion/scripts/evaluator/e2e_evaluator.py @@ -4,7 +4,7 @@ import uuid, os, logging, sys, multiprocessing, tempfile class E2EEvaluator: - def __init__(self, testcase, codegen_folder = "cuda_codegen", default_device = "CUDA", working_foler = ".", nnfusion_cli = "", nnfusion_cli_arg = ""): + def __init__(self, testcase, codegen_folder = "cuda_codegen", default_device = "CUDA", working_foler = ".", nnfusion_cli = "", nnfusion_cli_arg = "", perf_mode = False): self.codegen_folder = codegen_folder self.default_device = default_device self.testcase = testcase @@ -14,6 +14,9 @@ def __init__(self, testcase, codegen_folder = "cuda_codegen", default_device = " else: self.nnfusion_cli = nnfusion_cli self.nnfusion_cli_arg = nnfusion_cli_arg + + self.perf_mode = True + self.latency = 0 def load_default_nnfusion_cli(self): nnf_clis = [os.path.join(os.path.dirname(os.path.abspath( @@ -67,6 +70,19 @@ def allclose(self): logging.error("%s result missmatch."%self.testcase.casename) return False return True + + def exectime(self): + code = os.system("cd %s/nnfusion_rt/%s/ && ./main_test > result.txt"%(self.working_foler, self.codegen_folder)) + if code != 0: + logging.error("%s execution failed."%self.testcase.casename) + return False + if not os.path.exists("%s/nnfusion_rt/%s/result.txt"%(self.working_foler, self.codegen_folder)): + logging.error("Failed at compiling phase.") + return False + result_file = open("%s/nnfusion_rt/%s/result.txt"%(self.working_foler, self.codegen_folder)) + results = result_file.readlines() + latency = self.testcase.latency(results) + return latency def report(self): os.system("rm -rf %s/nnfusion_rt"%self.working_foler) @@ -76,9 +92,14 @@ def report(self): if not self.build(): os.system("rm -rf %s/nnfusion_rt"%self.working_foler) return False - if not self.allclose(): - os.system("rm -rf %s/nnfusion_rt"%self.working_foler) - return False + + if self.perf_mode is False: + if not self.allclose(): + os.system("rm -rf %s/nnfusion_rt"%self.working_foler) + return False + else: + self.latency = self.exectime() + os.system("rm -rf %s/nnfusion_rt"%self.working_foler) return True @@ -92,16 +113,19 @@ def E2EExecutor(TestCases, devname, report_list, nnf, nnf_args): for test in TestCases: logging.info("Testing " + test.casename) if test.valid(): - eval = E2EEvaluator(test, configs[devname][0], configs[devname][1], tmpdir, nnf, nnf_args) + perf_mode = "latency" in dir(test) + eval = E2EEvaluator(test, configs[devname][0], configs[devname][1], tmpdir, nnf, nnf_args, perf_mode) report = devname + "\t" + test.casename + '\t' + ",".join(test.tags) + "\t"; if eval.report(): report += "Succeed!" else: eval = E2EEvaluator(test, configs[devname][0], configs[devname][1], tmpdir) if eval.report(): - report += "Succeed!" + report += ",\tSucceed" else: - report += "Failed" + report += ",\tFailed" + if eval.perf_mode: + report += ",\t" + str(eval.latency) logging.info(report) report_list.append(report) # clean @@ -114,4 +138,4 @@ def CLIExecutor(info, report_list): if os.system(side_cli) == 0: report_list.append(side_cli + "\tSucceed!") else: - report_list.append(side_cli + "\tFailed") \ No newline at end of file + report_list.append(side_cli + "\tFailed") diff --git a/test/nnfusion/scripts/perf.json b/test/nnfusion/scripts/perf.json new file mode 100644 index 000000000..5dbfa71f2 --- /dev/null +++ b/test/nnfusion/scripts/perf.json @@ -0,0 +1,20 @@ +{ + "env": { + "PYTHONDONTWRITEBYTECODE" : { + "set" : 1 + }, + "HSA_USERPTR_FOR_PAGED_MEM": { + "set": 0 + }, + "LD_LIBRARY_PATH": { + "append": "/usr/local/lib" + }, + "HIP_VISIBLE_DEVICES" : { + "set": 1 + } + }, + "device_capability": ["CUDA"], + "enabled_tags": { + "CUDA" : ["perf"] + } + } \ No newline at end of file diff --git a/test/nnfusion/scripts/testcase_configs/naive_tests.json b/test/nnfusion/scripts/testcase_configs/naive_tests.json index 0f4056d01..977053125 100755 --- a/test/nnfusion/scripts/testcase_configs/naive_tests.json +++ b/test/nnfusion/scripts/testcase_configs/naive_tests.json @@ -51,4 +51,4 @@ "comment": "Naive json descriptor for test case." } ] -} \ No newline at end of file +} diff --git a/test/nnfusion/scripts/testcases/__init__.py b/test/nnfusion/scripts/testcases/__init__.py index 00692d84c..3a9d64b4e 100644 --- a/test/nnfusion/scripts/testcases/__init__.py +++ b/test/nnfusion/scripts/testcases/__init__.py @@ -4,12 +4,14 @@ from testcases.testcase import * import testcases.naive_cases import testcases.cpu_perf_cases +import testcases.perf_cases TestCases = list() tests_load_funtion = { "naive_case_single_line": testcases.naive_cases.create_naive_case_single_line, "naive_case_multi_lines": testcases.naive_cases.create_naive_case_multi_lines, - "cpu_perf_case_single_line": testcases.cpu_perf_cases.create_cpu_perf_case_single_line, "cpu_perf_case_single_lines": testcases.cpu_perf_cases.create_cpu_perf_case_multi_lines + "cpu_perf_case_single_line": testcases.cpu_perf_cases.create_cpu_perf_case_single_line, "cpu_perf_case_single_lines": testcases.cpu_perf_cases.create_cpu_perf_case_multi_lines, + "perf_case" : testcases.perf_cases.create_perf_case } def parse_tests(base_folder, json_data): diff --git a/test/nnfusion/scripts/testcases/perf_cases.py b/test/nnfusion/scripts/testcases/perf_cases.py new file mode 100644 index 000000000..ae3005576 --- /dev/null +++ b/test/nnfusion/scripts/testcases/perf_cases.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import logging +import os +from testcases.testcase import * + + +class PerfOutput(TestCase): + def __init__(self, casename, filename, tags, flag): + self.casename = casename + self.filename = filename + self.tags = tags + self.flag = flag + + # Get data from output of main_test + def allclose(self, raw_strdata): + return True + + def latency(self, raw_strdata): + real_time = float(raw_strdata[-1].strip("\n").split(" ")[-2][:-1]) + return real_time + #return raw_strdata[-1] + + +def create_perf_case(base_folder, json_data): + testcase = json_data["testcase"] + tags = json_data["tag"] + filename = os.path.join(base_folder, json_data["filename"]) + flag = "" + if "flag" in json_data: + flag = json_data["flag"] + + return PerfOutput(testcase, filename, tags, flag)