From 9362816e1c00156dbaa8a291f4f6947b051f8a5c Mon Sep 17 00:00:00 2001 From: valentinThomazic Date: Fri, 30 Aug 2024 22:23:03 +0200 Subject: [PATCH] fix simulation errors not detected on ci w/ tandem (#2486) --- verif/sim/cva6.py | 60 +++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/verif/sim/cva6.py b/verif/sim/cva6.py index 078ff1bc86..6a94e64caa 100644 --- a/verif/sim/cva6.py +++ b/verif/sim/cva6.py @@ -463,6 +463,7 @@ def run_assembly(asm_test, iss_yaml, isa, target, mabi, gcc_opts, iss_opts, outp # ISS simulation test_log_name = test_name or asm for iss in iss_list: + tandem_sim = iss != "spike" and os.environ.get('SPIKE_TANDEM') != None run_cmd("mkdir -p %s/%s_sim" % (output_dir, iss)) if log_format == 1: log = ("%s/%s_sim/%s_%d.%s.log" % (output_dir, iss, test_log_name, test_iteration, target)) @@ -475,10 +476,12 @@ def run_assembly(asm_test, iss_yaml, isa, target, mabi, gcc_opts, iss_opts, outp logging.info("[%0s] Running ISS simulation: %s" % (iss, cmd)) if "spike" in iss: ratio = 10 else: ratio = 1 + if tandem_sim: + generate_yaml_report(yaml, target, isa, test_log_name, testlist, iss, True) run_cmd(cmd, iss_timeout//ratio, debug_cmd = debug_cmd) logging.info("[%0s] Running ISS simulation: %s ...done" % (iss, elf)) - if (iss != "spike" and os.environ.get('SPIKE_TANDEM') != None): - tandem_postprocess(yaml, target, isa, test_log_name, log, testlist, iss) + if tandem_sim: + tandem_postprocess(yaml, target, isa, test_log_name, log, testlist, iss) if len(iss_list) == 2: compare_iss_log(iss_list, log_list, report) @@ -516,33 +519,26 @@ def run_assembly_from_dir(asm_test_dir, iss_yaml, isa, mabi, gcc_opts, iss, def tandem_postprocess(tandem_report, target, isa, test_name, log, testlist, iss, iterations = None): - report_ok = analyze_tandem_report(tandem_report) - generate_yaml_report(tandem_report, target, isa, test_name, testlist, iss, iterations, report_ok) - process_verilator_sim_log(log, log + ".csv") - + analyze_tandem_report(tandem_report) + process_verilator_sim_log(log, log + ".csv") + generate_yaml_report(tandem_report, target, isa, test_name, testlist, iss, False , iterations) def analyze_tandem_report(yaml_path): - if (os.path.exists(yaml_path)): - with open(yaml_path, 'r') as f: - data = yaml.safe_load(f) - try: - mismatches = data["mismatches"] - mismatches_count = (data["mismatches_count"]) - instr_count = (data["instr_count"]) - exit_code = (data["exit_code"]) - matches_count = instr_count - mismatches_count - logging.info("TANDEM Result : %s (exit code %s) with %s mismatches and %s matches" - % (data["exit_cause"], exit_code, mismatches_count, matches_count)) - return True - except KeyError: - logging.info("Incomplete TANDEM YAML report") - return False - else: - logging.info("TANDEM YAML not found") - return True - -def generate_yaml_report(yaml_path, target, isa, test, testlist, iss, iteration, report_ok): - if(os.path.exists(yaml_path) and report_ok): + with open(yaml_path, 'r') as f: + data = yaml.safe_load(f) + try: + mismatches_count = (data["mismatches_count"]) + instr_count = (data["instr_count"]) + exit_code = (data["exit_code"]) + matches_count = instr_count - mismatches_count + logging.info("TANDEM Result : %s (exit code %s) with %s mismatches and %s matches" + % (data["exit_cause"], exit_code, mismatches_count, matches_count)) + except KeyError: + logging.info("Incomplete TANDEM YAML report") + + +def generate_yaml_report(yaml_path, target, isa, test, testlist, iss, initial_creation , iteration = None): + if not initial_creation: with open(yaml_path, 'r') as f: report = yaml.safe_load(f) else: @@ -658,6 +654,7 @@ def run_c(c_test, iss_yaml, isa, target, mabi, gcc_opts, iss_opts, output_dir, # ISS simulation test_log_name = test_name or c for iss in iss_list: + tandem_sim = iss != "spike" and os.environ.get('SPIKE_TANDEM') != None run_cmd("mkdir -p %s/%s_sim" % (output_dir, iss)) if log_format == 1: log = ("%s/%s_sim/%s_%d.%s.log" % (output_dir, iss, test_log_name, test_iteration, target)) @@ -670,8 +667,12 @@ def run_c(c_test, iss_yaml, isa, target, mabi, gcc_opts, iss_opts, output_dir, logging.info("[%0s] Running ISS simulation: %s" % (iss, cmd)) if "spike" in iss: ratio = 10 else: ratio = 1 + if tandem_sim: + generate_yaml_report(yaml, target, isa, test_log_name, testlist, iss, True) run_cmd(cmd, iss_timeout//ratio, debug_cmd = debug_cmd) logging.info("[%0s] Running ISS simulation: %s ...done" % (iss, elf)) + if tandem_sim: + tandem_postprocess(yaml, target, isa, test_log_name, log, testlist, iss) if (iss != "spike" and os.environ.get('SPIKE_TANDEM') != None): tandem_postprocess(yaml, target, isa, test_log_name, log, testlist, iss) @@ -730,6 +731,7 @@ def iss_sim(test_list, output_dir, iss_list, iss_yaml, iss_opts, base_cmd = parse_iss_yaml(iss, iss_yaml, isa, target, setting_dir, debug_cmd, priv, spike_params) logging.info("%s sim log dir: %s" % (iss, log_dir)) run_cmd_output(["mkdir", "-p", log_dir]) + tandem_sim = iss != "spike" and os.environ.get('SPIKE_TANDEM') != None for test in test_list: if 'no_iss' in test and test['no_iss'] == 1: continue @@ -744,12 +746,14 @@ def iss_sim(test_list, output_dir, iss_list, iss_yaml, iss_opts, cmd += ' ' cmd += test['iss_opts'] logging.info("Running %s sim: %s" % (iss, elf)) + if tandem_sim: + generate_yaml_report(yaml, target, isa, test['test'], "generated tests", iss, True, i) if iss == "ovpsim": run_cmd(cmd, timeout_s, check_return_code=False, debug_cmd = debug_cmd) else: run_cmd(cmd, timeout_s, debug_cmd = debug_cmd) logging.debug(cmd) - if (iss != "spike" and os.environ.get('SPIKE_TANDEM') != None): + if tandem_sim: tandem_postprocess(yaml, target, isa, test['test'], log, "generated tests", iss, i)