From b190901be2551a23b0bc199233ea9e64d6f2e3eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Wed, 2 Sep 2026 12:04:41 +0200 Subject: [PATCH] Report a failed verification as two .mat files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed comparison used to produce a CSV and a dygraph HTML page per differing variable, plus an index page: `diffSimulationResults` wrote the CSVs itself, testmodel.py the HTML, and every double became text on the way. It now writes two files, the result and the reference reduced to the differing variables with `filterSimulationResults`, and the report links them to the OMPlot page of the OpenModelica playground, which runs the same tube comparison in the browser and plots every variable on demand: https://playground.openmodelica.org/latest/omplot/?result=…&reference=… The page needs absolute URLs, so library.html.tpl resolves the two relative paths at load time; without JavaScript the link is the result file itself. `diffSimulationResults` is called with an empty prefix, which (since the matching omc change) writes no per-variable CSV. Filtering a CSV reference needs an omc with that change as well; an older one logs the failure to the .err and the report has no reference file for that model. The files server has to allow cross-origin reads (Access-Control- Allow-Origin) for the page to fetch them. Assisted-by: Claude Fable 5.1 --- library.html.tpl | 10 +++++++ test.py | 13 +++------ testmodel.py | 75 ++++++++++++------------------------------------ 3 files changed, 32 insertions(+), 66 deletions(-) diff --git a/library.html.tpl b/library.html.tpl index 6d17218..9813ba9 100644 --- a/library.html.tpl +++ b/library.html.tpl @@ -55,5 +55,15 @@ Config: 
#config#
ModelVerifiedSimulateTotal buildModelParsingFrontendBackendSimCodeTemplatesCompileTotal Execution #testsHTML# + diff --git a/test.py b/test.py index 9fc2fec..f0c7fd3 100755 --- a/test.py +++ b/test.py @@ -1288,19 +1288,14 @@ def artifactSuffix(simulator): filename_prefix = "files/%s_%s" % (s[2],s[1]) stagePublished(stageRoot, workspace_prefix, filename_prefix, suffix) stageShared(stageRoot, "files/%s_%s" % (s[2],s[1]), suffix) - filesList.write("/%s*diff*csv\n" % filename_prefix) - filesList.write("/%s*diff*html\n" % filename_prefix) if is_non_zero_file(workspace_prefix+".sim"): filesList.write("/%s.sim\n" % filename_prefix) errPrefix = "files/%s_%s" % (s[2],s[1]) if is_non_zero_file(errPrefix+".err"): filesList.write("/%s.err\n" % errPrefix) - variables = (s[3].get("diff") or {}).get("vars") or [] - if len(variables)>0: - filesList.write("/%s.diff.html\n" % filename_prefix) - for v in variables: - filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v)) - filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v)) + if (s[3].get("diff") or {}).get("vars"): + filesList.write("/%s.diff.mat\n" % filename_prefix) + filesList.write("/%s.diff.ref.mat\n" % filename_prefix) filesList.close() testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s\n' % (lambda filename_prefix, errPrefix, diff: @@ -1311,7 +1306,7 @@ def artifactSuffix(simulator): # Nothing was compared, whatever phase the model reports. " " if diff is None else (("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else - ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), + ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), checkPhase(s[3]["phase"], 6), timeSeconds(s[3].get("sim") or 0), checkPhase(s[3]["phase"], 5), diff --git a/testmodel.py b/testmodel.py index 315734b..d0e89ff 100755 --- a/testmodel.py +++ b/testmodel.py @@ -855,8 +855,9 @@ def verifyAgainstReference(resFile, prefix, stat): fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start)) start=monotonic() try: - (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % - (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) + # An empty diffPrefix: no per-variable CSV, the two filtered files below are the report. + (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % + (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) except TimeoutError as e: with open(errFile, 'a+') as fp: fp.write("Timeout error for diffSimulationResults") @@ -875,63 +876,23 @@ def verifyAgainstReference(resFile, prefix, stat): resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile) fp.write("\nVariables in the result:" ) fp.write(",".join(resVars)+"\n") - diffFiles = [prefix + "." + var for var in diffVars] stat["diff"]["vars"]=diffVars + # Both files reduced to the differing variables: what the report's OMPlot + # link compares, instead of a CSV and an HTML page per variable. + filterDiffVars(resFile, prefix + ".mat", diffVars) + filterDiffVars(referenceFile, prefix + ".ref.mat", diffVars) - # Create a file containing only the calibrated variables, for easy display - lstfiles = "\n".join(['
  • %s (javascript) (csv)
    • ' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles]) - with open(prefix+".html", 'w') as fp: - fp.write('

    %s differences from the reference file

    startTime: %g

    stopTime: %g

    Simulated using tolerance: %g

    ' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles)) - for var in diffVars: - if "/" in var: - continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below - with open(prefix+"."+var+".html", 'w') as fp: - fp.write(""" - - - - - -
    -

    - - - - - - - - - - - - Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.

    - - - """ % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var)) - +def filterDiffVars(inFile, outFile, diffVars): + vars = ", ".join('"%s"' % v.replace('\\', '\\\\').replace('"', '\\"') for v in diffVars) + try: + ok = sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "%s", vars={%s}, removeDescription=false)' % (inFile, outFile, vars), conf["ulimitOmc"]) + except TimeoutError as e: + ok = False + if not ok: + with open(errFile, 'a+') as fp: + fp.write("Failed to write the differing variables of %s to %s:\n%s" % (inFile, outFile, omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False))) + if os.path.exists(outFile): + os.remove(outFile) # The first simulator's results are the ones every non-FMI code path expects. # There is nothing to compare when it never produced any.