Report a failed verification as two .mat files

library.html.tpl

@@ -55,5 +55,15 @@ Config: <pre>#config#</pre>
 <tr><th>Model</th><th>Verified</th><th>Simulate</th><th>Total buildModel</th><th>Parsing</th><th>Frontend</th><th>Backend</th><th>SimCode</th><th>Templates</th><th>Compile</th><th>Total Execution</th></tr>
 #testsHTML#
 </table>
+<script>
+// A failed verification links its two .mat files (the differing variables of
+// the result and of the reference) to the OMPlot page, which plots the tube
+// comparison of each variable. The page needs absolute URLs.
+const omplot = 'https://playground.openmodelica.org/latest/omplot/';
+for (const a of document.querySelectorAll('a[data-omplot]')) {
+  const abs = (f) => encodeURIComponent(new URL(f, location.href).href);
+  a.href = omplot + '?result=' + abs(a.dataset.omplot + '.mat') + '&reference=' + abs(a.dataset.omplot + '.ref.mat');
+}
+</script>
 </body>
 </html>

test.py

@@ -1288,19 +1288,14 @@ def artifactSuffix(simulator):
       filename_prefix = "files/%s_%s" % (s[2],s[1])
       stagePublished(stageRoot, workspace_prefix, filename_prefix, suffix)
       stageShared(stageRoot, "files/%s_%s" % (s[2],s[1]), suffix)
-      filesList.write("/%s*diff*csv\n" % filename_prefix)
-      filesList.write("/%s*diff*html\n" % filename_prefix)
       if is_non_zero_file(workspace_prefix+".sim"):
         filesList.write("/%s.sim\n" % filename_prefix)
       errPrefix = "files/%s_%s" % (s[2],s[1])
       if is_non_zero_file(errPrefix+".err"):
         filesList.write("/%s.err\n" % errPrefix)
-      variables = (s[3].get("diff") or {}).get("vars") or []
-      if len(variables)>0:
-        filesList.write("/%s.diff.html\n" % filename_prefix)
-      for v in variables:
-        filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v))
-        filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v))
+      if (s[3].get("diff") or {}).get("vars"):
+        filesList.write("/%s.diff.mat\n" % filename_prefix)
+        filesList.write("/%s.diff.ref.mat\n" % filename_prefix)
     filesList.close()
     testsHTML = "\n".join(['<tr><td>%s%s</td><td bgcolor="%s">%s</td><td bgcolor="%s">%s</td><td bgcolor="%s">%s</td><td>%s</td><td bgcolor="%s">%s</td><td bgcolor="%s">%s</td><td bgcolor="%s">%s</td><td bgcolor="%s">%s</td><td bgcolor="%s">%s</td><td>%s</td></tr>\n' %
       (lambda filename_prefix, errPrefix, diff:
@@ -1311,7 +1306,7 @@ def artifactSuffix(simulator):
         # Nothing was compared, whatever phase the model reports.
         "&nbsp;" if diff is None else
         (("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else
-        ('%s (<a href="%s.diff.html">%d/%d failed</a>)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))),
+        ('%s (<a href="%s.diff.mat" data-omplot="%s.diff">%d/%d failed</a>)' % (timeSeconds(diff.get("time")), filename_prefix, filename_prefix, len(diff.get("vars")), diff.get("numCompared")))),
         checkPhase(s[3]["phase"], 6),
         timeSeconds(s[3].get("sim") or 0),
         checkPhase(s[3]["phase"], 5),

testmodel.py

@@ -855,8 +855,9 @@ def verifyAgainstReference(resFile, prefix, stat):
       fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start))
   start=monotonic()
   try:
-    (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' %
-                               (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"])
+    # An empty diffPrefix: no per-variable CSV, the two filtered files below are the report.
+    (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' %
+                               (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"])
   except TimeoutError as e:
     with open(errFile, 'a+') as fp:
       fp.write("Timeout error for diffSimulationResults")
@@ -875,63 +876,23 @@ def verifyAgainstReference(resFile, prefix, stat):
       resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile)
       fp.write("\nVariables in the result:" )
       fp.write(",".join(resVars)+"\n")
-    diffFiles = [prefix + "." + var for var in diffVars]
     stat["diff"]["vars"]=diffVars
+    # Both files reduced to the differing variables: what the report's OMPlot
+    # link compares, instead of a CSV and an HTML page per variable.
+    filterDiffVars(resFile, prefix + ".mat", diffVars)
+    filterDiffVars(referenceFile, prefix + ".ref.mat", diffVars)
 
-    # Create a file containing only the calibrated variables, for easy display
-    lstfiles = "\n".join(['<li>%s <a href="%s.html">(javascript)</a> <a href="%s.csv">(csv)</a></li>' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles])
-    with open(prefix+".html", 'w') as fp:
-      fp.write('<html lang="en"><body><h1>%s differences from the reference file</h1><p>startTime: %g</p><p>stopTime: %g</p><p>Simulated using tolerance: %g</p><ul>%s</ul></body></html>' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles))
-    for var in diffVars:
-      if "/" in var:
-        continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below
-      with open(prefix+"."+var+".html", 'w') as fp:
-        fp.write("""<html lang="en">
-  <head>
-  <script type="text/javascript" src="dygraph-combined.js"></script>
-      <style type="text/css">
-      #graphdiv {
-        position: absolute;
-        left: 10px;
-        right: 10px;
-        top: 40px;
-        bottom: 10px;
-      }
-      </style>
-  </head>
-  <body>
-  <div id="graphdiv"></div>
-  <p><input type=checkbox id="0" checked onClick="change(this)">
-  <label for="0">reference</label>
-  <input type=checkbox id="1" checked onClick="change(this)">
-  <label for="1">actual</label>
-  <input type=checkbox id="2" checked onClick="change(this)">
-  <label for="2">high</label>
-  <input type=checkbox id="3" checked onClick="change(this)">
-  <label for="3">low</label>
-  <input type=checkbox id="4" checked onClick="change(this)">
-  <label for="4">error</label>
-  <input type=checkbox id="5" onClick="change(this)">
-  <label for="5">actual (original)</label>
-  Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.</p>
-  <script type="text/javascript">
-  g = new Dygraph(document.getElementById("graphdiv"),
-                   "%s",{title: '"%s"',
-    legend: 'always',
-    connectSeparatedPoints: true,
-    xlabel: ['time'],
-    y2label: ['error'],
-    series : { 'error': { axis: 'y2' } },
-    colors: ['blue','red','teal','lightblue','orange','black'],
-    visibility: [true,true,true,true,true,false]
-  });
-  function change(el) {
-    g.setVisibility(parseInt(el.id), el.checked);
-  }
-  </script>
-  </body>
-  </html>""" % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var))
-
+def filterDiffVars(inFile, outFile, diffVars):
+  vars = ", ".join('"%s"' % v.replace('\\', '\\\\').replace('"', '\\"') for v in diffVars)
+  try:
+    ok = sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "%s", vars={%s}, removeDescription=false)' % (inFile, outFile, vars), conf["ulimitOmc"])
+  except TimeoutError as e:
+    ok = False
+  if not ok:
+    with open(errFile, 'a+') as fp:
+      fp.write("Failed to write the differing variables of %s to %s:\n%s" % (inFile, outFile, omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False)))
+    if os.path.exists(outFile):
+      os.remove(outFile)
 
 # The first simulator's results are the ones every non-FMI code path expects.
 # There is nothing to compare when it never produced any.