diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index b0cd1ac..50c439f 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -592,10 +592,11 @@ pipeline { sh "./report.py --branches='generateSymbolicJacobian' configs/conf.json" sh "mv overview.html overview-generateSymbolicJacobian.html" - sh "./report.py --branches='dataReconciliation dataReconciliation-newBackend' configs/dataReconciliation.json" + // master does not test these libraries. + sh "./report.py --branches='dataReconciliation dataReconciliation-newBackend' --invalid-branch=dataReconciliation configs/dataReconciliation.json" sh "mv overview.html overview-dataReconciliation.html" - sh "./report.py --branches='heavy_tests heavy_tests_wasm' configs/heavy_tests.json" + sh "./report.py --branches='heavy_tests heavy_tests_wasm' --invalid-branch=heavy_tests configs/heavy_tests.json" sh "mv overview.html overview-heavy_tests.html" sh "./report.py --branches='${env.GITBRANCHES_NEWINST}' configs/conf.json" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 852e4b9..6fdfd54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: - name: Verify that overview.html contains simulation and verification shell: bash run: | - if ! grep -q '${{ matrix.omc-version }}222222222' overview.html; then + if ! grep -q '${{ matrix.omc-version }}222222222' overview.html; then echo "Failed to find 2 simulating and 2 verifying models in overwiew.html:" cat overview.html exit 1 diff --git a/configs/conf.json b/configs/conf.json index 3b972c7..cd75e2c 100644 --- a/configs/conf.json +++ b/configs/conf.json @@ -152,8 +152,7 @@ { "library":"AES", "libraryVersion":"main", - "libraryVersionNameForTests":"", - "extraCustomCommands":["setCommandLineOptions(\"--allowNonStandardModelica=illegalConditionalContext\");"] + "libraryVersionNameForTests":"" }, { "library":"AixLib" @@ -326,14 +325,12 @@ "library":"Dynawo", "ulimitExeModels":{ "Dynawo.Examples.RVS.TestCases.TestB.TestBNoSvcNoLoadReset":330 - }, - "extraCustomCommands":["setCommandLineOptions(\"--allowNonStandardModelica=implicitParameterStartAttribute,illegalConditionalContext\");"] + } }, { "library":"Dynawo", "libraryVersion":"master", "libraryVersionNameForTests":"dev", - "extraCustomCommands":["setCommandLineOptions(\"--allowNonStandardModelica=implicitParameterStartAttribute,illegalConditionalContext\");"], "referenceFileExtension":"csv", "referenceFileNameDelimiter":".", "referenceFinalDot":"_res.", @@ -673,7 +670,6 @@ "libraryVersionNameForTests":"", "extraCustomCommands": [ - "setCommandLineOptions(\"--allowNonStandardModelica=nonStdEnumerationAsIntegers,nonStdTopLevelOuter\")", "setEnvironmentVar(\"publicData\", \"$libraryLocation/Tables/\")", "setEnvironmentVar(\"privateData\", \"$libraryLocation/Tables/\")", "setEnvironmentVar(\"superstructureTables\", \"$libraryLocation/Tables/superstructure/Tables/\")", diff --git a/doc/README.md b/doc/README.md index d93651e..378ac46 100644 --- a/doc/README.md +++ b/doc/README.md @@ -44,7 +44,10 @@ migrates on startup (`test.py`, around the `CREATE TABLE` block): | 0 | empty/new database; `omcversion` and `libversion` are created | | 1 | `libversion.confighash` added | | 2 | `parsing` added to every per-branch table | -| 3 | current layout | +| 3 | the layout of 2026-08 | +| 4 | `libversion.host` and `libversion.sysinfo` added | +| 5 | `maxrss` added to every per-branch table | +| 6 | `invalid` added to every per-branch table; current layout | A database with a higher `user_version` makes `test.py` exit rather than guess. @@ -72,7 +75,9 @@ CREATE TABLE if not exists [] ( verifyfail integer NOT NULL, -- number of variables that differ from the reference verifytotal integer NOT NULL, -- number of variables compared against the reference finalphase integer NOT NULL, -- how far the model got, see below - parsing real NOT NULL -- time loading/parsing the library [s] + parsing real NOT NULL, -- time loading/parsing the library [s] + maxrss integer NOT NULL, -- most resident memory of the test at once [bytes] + invalid integer NOT NULL -- 1 if the model needs non-standard Modelica, see below ) ``` diff --git a/library.html.tpl b/library.html.tpl index da90853..1dca0b8 100644 --- a/library.html.tpl +++ b/library.html.tpl @@ -2,6 +2,10 @@ #fileName# test using OpenModelica +

#fileName# test using OpenModelica

@@ -9,6 +13,7 @@ + @@ -19,6 +24,7 @@ +#Invalid# @@ -51,6 +57,7 @@ Optimization level: #optlevel#

Flags:
#customCommands#
Config:
#config#

Links are provided if getErrorString() or the simulation generates output. The links are coded with red if there were errors, yellow if there were warnings, and normal links if there are only notifications.

+#invalidNote#
TotalInvalid Frontend Backend SimCode
#Total##Frontend# #Backend# #SimCode#
#drHeader# #testsHTML# diff --git a/overview.html.tpl b/overview.html.tpl index 51d2c06..1abbdd6 100644 --- a/overview.html.tpl +++ b/overview.html.tpl @@ -1,28 +1,120 @@ + + #title# +
+

#title#

+ Invalid models: + #viewSwitch# +

Statistics

-
ModelVerified#simulateLabel#Total buildModelParsingFrontendBackendSimCodeTemplatesCompileTotal Execution
- - -
Number of libraries#numlibs#
Number of models#nummodels#
+
+
#numlibs#libraries
+
#nummodels#models
+
#numinvalid#invalid models
+
+

A model is invalid if the newest run of #invalidBranch# on its library translated it only because +--warnNonStandardModelica turned the errors of non-standard Modelica into warnings. The buttons at the top +select if the counts and times leave the invalid models out (the default), include them, or count only them.

Tested branches

- - +
#branches# -
BranchVersionBuild timeExecution timeSimulation time#simulateLabel## Verify# Total
+ +

Libraries

#entries# + diff --git a/report.py b/report.py index d34d37a..689398f 100755 --- a/report.py +++ b/report.py @@ -8,6 +8,11 @@ parser = argparse.ArgumentParser(description='OpenModelica library testing report generation tool') parser.add_argument('configs', nargs='*') parser.add_argument('--branches', default='master') +parser.add_argument('--invalid-branch', default='master', + help="The branch whose newest run of each library says which models are invalid: " + "omc translated them only because --warnNonStandardModelica turned the errors of " + "non-standard Modelica into warnings. They are left out of the counts, whatever " + "the other branches say about them.") resultsdb.addArgument(parser) args = parser.parse_args() configs = args.configs @@ -16,6 +21,7 @@ raise Exception("Error: Expected at least one configuration file to start the library test") branches = [shared.resultTable(br) for br in args.branches.split(" ")] +invalidBranch = shared.resultTable(args.invalid_branch) dates = {} dates_str = {} @@ -39,12 +45,22 @@ db = resultsdb.connect(args.db) cursor = db.cursor() -nmodels = {} -nsimulate = {} -nverify = {} -exectime = {} -simtime = {} +# (view, label): which models a library's tables count. +VIEWS = [("valid", "Exclude invalid"), ("all", "Include invalid"), ("invalid", "Only invalid")] +def inView(view, model, invalid): + return view == "all" or (model in invalid) == (view == "invalid") + +invalidModels = {} +if db.tableExists(invalidBranch) and "invalid" in db.columns(invalidBranch): + for libname in libnames: + v = cursor.execute("SELECT MAX(date) FROM %s WHERE libname=?" % db.quote(invalidBranch), (libname,)).fetchone() + if v and v[0]: + invalidModels[libname] = set(m for (m,) in cursor.execute( + "SELECT model FROM %s WHERE libname=? AND date=? AND finalphase>=0 AND invalid=1" % db.quote(invalidBranch), + (libname, v[0]))) + +rows = {} missing_branches = [] for branch in branches: one = None @@ -61,7 +77,7 @@ db.createDateIndex(branch) dates[branch] = {} - branch_nmodels = 0 + rows[branch] = {} for libname in libnames: cursor.execute("SELECT date FROM %s WHERE libname=? ORDER BY date DESC LIMIT 1" % db.quote(branch), (libname,)) v = cursor.fetchone() @@ -69,32 +85,73 @@ dates[branch][libname] = 0 continue dates[branch][libname] = v[0] - for x in cursor.execute("SELECT model FROM %s WHERE libname=? AND date=? AND finalphase>=0" % db.quote(branch), (libname,v[0])): - if libname not in libs: - libs[libname] = set() - libs[libname].add(x[0]) - branch_nmodels += 1 - nmodels[branch] = branch_nmodels - nsimulate[branch] = 0 - nverify[branch] = 0 - exectime[branch] = 0.0 - simtime[branch] = 0.0 + rows[branch][libname] = cursor.execute("SELECT model,finalphase,%s FROM %s WHERE libname=? AND date=? AND finalphase>=0" + % (",".join(fields), db.quote(branch)), (libname,v[0])).fetchall() + for r in rows[branch][libname]: + libs.setdefault(libname, set()).add(r[0]) -entries = "" # removing missing branches for branch in missing_branches: branches.remove(branch) +def counts(branch, lib, view): + """The number of models reaching each phase, and the time spent in each.""" + invalid = invalidModels.get(lib, set()) + selected = [r for r in rows[branch].get(lib, []) if inView(view, r[0], invalid)] + vs = [sum(1 for r in selected if r[1] >= i) for i in range(0,8)] + sums = [sum(r[2+i] or 0 for r in selected) for i in range(0,len(fields))] + return (vs, sums) + +totals = {} +for (view, _) in VIEWS: + totals[view] = {} + for branch in branches: + t = {"simulate": 0, "verify": 0, "total": 0, "exectime": 0.0, "simtime": 0.0} + for lib in libs: + (vs, sums) = counts(branch, lib, view) + t["simulate"] += vs[6] + t["verify"] += vs[7] + t["total"] += vs[0] + t["exectime"] += sums[0] + t["simtime"] += sums[7] + totals[view][branch] = t + +def countsTable(caption, rows): + return '\n%s%s
%s
\n' % (caption, entryhead, "".join(rows)) + +def libraryTables(lib, view): + phases = [] + times = [] + old_vs = None + for branch in branches: + (vs, sums) = counts(branch, lib, view) + link = '%s' % (branch,lib,lib,branch) + phases.append("%s%s\n" % (link, "".join('%s' % (' class="warning"' if old_vs and old_vs[i]>vs[i] else (' class="better"' if old_vs and old_vs[i]%s%s\n" % (link, "".join("%s" % friendlyStr(s) for s in sums))) + if old_vs: + old_vs = [max(vs[i], old_vs[i]) for i in range(0, len(vs))] + else: + old_vs = vs + return countsTable("Models that completed each phase", phases) + countsTable("Time spent in each phase", times) + +def viewSwitch(): + return '
%s
' % "".join( + '' % (view, "true" if view == "valid" else "false", label) + for (view, label) in VIEWS) + def checkEqual(iterator): return len(set(iterator)) <= 1 +entries = "" for lib in sorted(libs.keys()): - models = libs[lib] - entries += "

%s

\n" % lib + invalid = invalidModels.get(lib, set()) & libs[lib] + entries += '
\n' if invalid else '
\n' + entries += "

%s

\n" % lib branches_versions = [(cursor.execute("SELECT libversion FROM libversion WHERE libname=? AND branch=? ORDER BY date DESC LIMIT 1", (lib,branch)).fetchone() or ["unknown"])[0] for branch in branches] all_equal = checkEqual(branches_versions) if not all_equal: - entries += "\n" + entries += '
\n' entries += "%s\n" % "".join(["" % branch for branch in branches]) entries += "" for ver in branches_versions: @@ -102,70 +159,45 @@ def checkEqual(iterator): entries += "\n" entries += "
 
%s
Version
\n" else: - entries += "

Library version: %s

" % branches_versions[0] - entries += "\n" - entries += entryhead - old_vs = None - models = {} - for branch in branches: - master_models = [] - for i in range(0,8): - i_models = set() - for v in cursor.execute("SELECT model FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)): - i_models.add(v[0]) - master_models.append(i_models) - models[branch] = master_models - for branch in branches: - vs = [cursor.execute("SELECT COUNT(*) FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] - warnings = [] - entries += '' % (branch,lib,lib,branch) - for i in [0]+list(range(0,len(vs))): - diff1 = models[branch][i] - models[branches[-1]][i] - diff2 = models[branches[-1]][i] - models[branch][i] - diff_text = "" - if len(diff2)>0: - diff_text += "

Now working in %s, failed in %s:
\n" % (branches[-1],branch) + "
\n".join(sorted(diff2)) + "

" - if len(diff1)>0: - diff_text += "

Now failing in %s, worked in %s:
\n" % (branches[-1],branch) + "
\n".join(sorted(diff1)) + "

" - if False and diff_text: - diff_text += "

Debug:
\n" + "
\n".join(models[branch][i]) + "

" - entries += '%s%s' % (' class="warning"' if old_vs and old_vs[i]>vs[i] else (' class="better"' if old_vs and old_vs[i]%s' % diff_text) if diff_text else "") - entries += "" - nsimulate[branch] += vs[6] - nverify[branch] += vs[7] - if old_vs: - old_vs = [max(vs[i], old_vs[i]) for i in range(0, len(vs))] - else: - old_vs = vs - entries += "
%s
\n" - entries += "\n" - entries += entryhead - for branch in branches: - vs = [cursor.execute("SELECT COUNT(*) FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] - sums = [cursor.execute("SELECT SUM(%s) FROM %s WHERE date=? AND libname=?" % (fields[i],db.quote(branch)), (dates[branch][lib],lib)).fetchone()[0] or 0 for i in range(0,9)] - entries += '' % (branch,lib,lib,branch) - entries += ("\n" % (friendlyStr(sums[0]),friendlyStr(sums[1]),friendlyStr(sums[2]),friendlyStr(sums[3]),friendlyStr(sums[4]),friendlyStr(sums[5]),friendlyStr(sums[6]),friendlyStr(sums[7]),friendlyStr(sums[8]))) - exectime[branch] += sums[0] - simtime[branch] += sums[7] - entries += "
%s%s%s%s%s%s%s%s%s%s
\n" - # print(sorted(list(libs[lib]))) - -nummodels = sum(len(l) for l in libs.values()) -branches_lines = [("%s%s%s%s%s%d%d%d\n" % (html.escape(branch), html.escape( - (cursor.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0] - ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]), friendlyStr(simtime[branch]), - " class=\"warning\"" if nummodels!=nmodels[branch] else "", - nsimulate[branch], - nverify[branch], - nmodels[branch] -)) for branch in branches] + entries += '

Library version: %s

\n' % branches_versions[0] + if invalid: + entries += '

✘ %d invalid model%s

\n' % (len(invalid), "" if len(invalid) == 1 else "s") + entries += "
Invalid models

%s

\n" % "
\n".join(html.escape(m) for m in sorted(invalid)) + for (view, _) in VIEWS: + entries += '
\n%s
\n' % (view, libraryTables(lib, view)) + else: + entries += libraryTables(lib, "all") + entries += "
\n" + +nummodels = sum(len(models - invalidModels.get(lib, set())) for (lib, models) in libs.items()) +numinvalid = sum(len(models & invalidModels.get(lib, set())) for (lib, models) in libs.items()) +expected = {"valid": nummodels, "all": nummodels + numinvalid, "invalid": numinvalid} +omcversions = dict((branch, (cursor.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0]) + for branch in branches) +simulateLabel = "# " + simulationLabel if simulationLabel != "Simulation" else "# Simulate" + +BRANCH_CAPTIONS = {"valid": "Invalid models left out", "all": "Invalid models included", "invalid": "Only the invalid models"} + +def branchesTable(view): + t = totals[view] + lines = ["%s%s%s%s%s%d%d%d\n" % ( + html.escape(branch), html.escape(omcversions[branch]), html.escape(dates_str[branch]), + friendlyStr(t[branch]["exectime"]), friendlyStr(t[branch]["simtime"]), + t[branch]["simulate"], t[branch]["verify"], + ' class="warning"' if expected[view] != t[branch]["total"] else "", t[branch]["total"]) for branch in branches] + return ('
\n' + '\n%s
%s
BranchVersionBuild timeExecution timeSimulation time%s# Verify# Total
\n' + % (view, BRANCH_CAPTIONS[view], simulateLabel, "".join(lines))) + template = open("overview.html.tpl").read() replacements = ( (u"#title#", "OpenModelica Library Testing Overview"), - (u"#simulateLabel#", "# " + simulationLabel if simulationLabel != "Simulation" else "# Simulate"), - (u"#branches#", "\n".join(branches_lines)), + (u"#branches#", "".join(branchesTable(view) for (view, _) in VIEWS)), + (u"#viewSwitch#", viewSwitch()), (u"#numlibs#", str(len(libs))), (u"#nummodels#", str(nummodels)), + (u"#numinvalid#", str(numinvalid)), + (u"#invalidBranch#", html.escape(invalidBranch)), (u"#entries#", entries) ) open("overview.html", "w").write(multiple_replace(template, *replacements)) diff --git a/resultsdb.py b/resultsdb.py index ba4c24c..d41eb3b 100644 --- a/resultsdb.py +++ b/resultsdb.py @@ -39,6 +39,8 @@ ("compile", "real"), ("simulate", "real"), ("verify", "real"), ("verifyfail", "int"), ("verifytotal", "int"), ("finalphase", "int"), ("parsing", "real"), ("maxrss", "bigint"), + # 1 for a model that omc only translates with --warnNonStandardModelica + ("invalid", "int"), ] SQLITE_TYPES = {"bigint": "integer", "int": "integer", "real": "real", "text": "text"} POSTGRES_TYPES = {"bigint": "bigint", "int": "integer", "real": "double precision", "text": "text"} @@ -311,10 +313,12 @@ def createTables(self, branch): self.addLibversionHost(cursor) elif user_version == 3: self.addLibversionHost(cursor) - elif user_version not in (4, 5): + elif user_version not in (4, 5, 6): raise SystemExit("Unknown schema user_version=%d" % user_version) if 0 < user_version < 5: self.addMaxrss(cursor) + if 0 < user_version < 6: + self.addInvalid(cursor) cols = ", ".join("%s %s NOT NULL" % (c, SQLITE_TYPES[t]) for c, t in BRANCH_COLUMNS) cursor.execute("CREATE TABLE if not exists %s (%s)" % (self.quote(branch), cols)) @@ -322,7 +326,7 @@ def createTables(self, branch): cursor.execute("DROP INDEX IF EXISTS [idx_%s_date]" % branch) cursor.execute("DROP INDEX IF EXISTS idx_omcversion_date") cursor.execute("DROP INDEX IF EXISTS idx_libversion_date") - self.setUserVersion(5) + self.setUserVersion(6) def addMaxrss(self, cursor): """Add maxrss to every result table that predates it; older rows read 0.""" @@ -331,6 +335,13 @@ def addMaxrss(self, cursor): if "finalphase" in have and "maxrss" not in have: cursor.execute("ALTER TABLE %s ADD COLUMN maxrss integer NOT NULL DEFAULT(0)" % self.quote(tbl)) + def addInvalid(self, cursor): + """Add invalid to every result table that predates it; older rows read 0.""" + for tbl in self.tables(): + have = self.columns(tbl) + if "finalphase" in have and "invalid" not in have: + cursor.execute("ALTER TABLE %s ADD COLUMN invalid integer NOT NULL DEFAULT(0)" % self.quote(tbl)) + def addLibversionHost(self, cursor): """Add the host columns to an existing [libversion]. @@ -524,8 +535,11 @@ def createTables(self, branch): cols = ", ".join("%s %s%s" % (c, POSTGRES_TYPES[t], " NOT NULL" if c in BRANCH_KEY else "") for c, t in BRANCH_COLUMNS) cursor.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (self.quote(branch), cols)) - if "maxrss" not in self.columns(branch): + branchColumns = self.columns(branch) + if "maxrss" not in branchColumns: cursor.execute("ALTER TABLE %s ADD COLUMN IF NOT EXISTS maxrss bigint" % self.quote(branch)) + if "invalid" not in branchColumns: + cursor.execute("ALTER TABLE %s ADD COLUMN IF NOT EXISTS invalid integer" % self.quote(branch)) indexes = [("UNIQUE INDEX", "uq_%s_%s" % (tbl, "_".join(KEYS.get(tbl, BRANCH_KEY))), tbl, KEYS.get(tbl, BRANCH_KEY)) for tbl in ["omcversion", "libversion", branch]] indexes.append(("INDEX", "idx_%s_libname_date" % branch, branch, ["libname", "date"])) diff --git a/shared.py b/shared.py index 9984c69..949bc04 100644 --- a/shared.py +++ b/shared.py @@ -22,6 +22,9 @@ def resultTable(branch): # derives it, and the exceptions, from the master results. DEFAULT_ULIMIT_EXE = 240 +# The tag omc --warnNonStandardModelica adds to the warning for non-standard Modelica. +nonStandardRe = re.compile(r"\[Non-standard Modelica: --allowNonStandardModelica=(\w+)\]") + simCodeTargetRe = re.compile('--simCodeTarget=([^"\'\\s,;)]+)') def simCodeTargetFromCommands(target, commands): diff --git a/sqlite2postgres.py b/sqlite2postgres.py index 11eba77..e377913 100644 --- a/sqlite2postgres.py +++ b/sqlite2postgres.py @@ -65,6 +65,7 @@ ("finalphase", "integer"), ("parsing", "double precision"), ("maxrss", "bigint"), + ("invalid", "integer"), ] LOOKUP_COLUMNS = { diff --git a/test.py b/test.py index e21412d..873aa61 100755 --- a/test.py +++ b/test.py @@ -550,7 +550,10 @@ def testHelloWorld(cmd): 'setCommandLineOptions("-d=stateselection");', 'setCommandLineOptions("-d=execstat");', 'setMatchingAlgorithm("PFPlusExt");', - 'setIndexReductionMethod("dynamicStateSelection");' + 'setIndexReductionMethod("dynamicStateSelection");', + # Non-standard Modelica is translated anyway and the model marked invalid, + # see shared.nonStandardRe. + 'setCommandLineOptions("--warnNonStandardModelica");' ]: if testHelloWorld(cmd): defaultCustomCommands.append(cmd) @@ -1241,7 +1244,8 @@ def resultValues(model, libname, data, simulator=None): diff.get("numCompared") or 0, simulated.get("phase") or 0, data.get("parsing") or 0.0, - data.get("maxrss") or 0 + data.get("maxrss") or 0, + 1 if data.get("nonstandard") else 0 ) def removedModels(resultBranch, libname, tested): @@ -1259,7 +1263,7 @@ def removedModels(resultBranch, libname, tested): def removedValues(model, libname): """One row of a branch table saying the library no longer has that model.""" return (testRunStartTimeAsEpoch, libname, model, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, shared.DELETED_PHASE, 0.0, 0) + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, shared.DELETED_PHASE, 0.0, 0, 0) resultPlaceholders = ",".join("?" * len(resultsdb.BRANCH_COLUMNS)) @@ -1361,6 +1365,13 @@ def drSummary(conf, stats): return ("Data reconciliation", '%d' % ("#00FF00" if n == len(stats) else "#FFCC66", n), "Data reconciliation") +def invalidCross(data): + """A red cross for a model that needs non-standard Modelica, naming what it needs.""" + options = data.get("nonstandard") or [] + if not options: + return "" + return '✘ ' % html.escape(" ".join("--allowNonStandardModelica=%s" % o for o in options)) + def checkPhase(phase, n): if phase >= n: return "#00FF00" @@ -1580,9 +1591,11 @@ def publishLibrary(libname): filesList.write("/%s.diff.mat\n" % filename_prefix) filesList.write("/%s.diff.ref.mat\n" % filename_prefix) filesList.close() - testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n' % + testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n' % (lambda filename_prefix, errPrefix, diff: ( + ' class="invalid"' if s[3].get("nonstandard") else "", + invalidCross(s[3]), ('%s' % (errPrefix + ".err", html.escape(s[1]))) if is_non_zero_file(errPrefix + ".err") else html.escape(s[1]), (' (sim)' % (filename_prefix + ".sim")) if is_non_zero_file(filename_prefix + suffix + ".sim") else "", drLink(filename_prefix, s[3]), @@ -1611,6 +1624,7 @@ def publishLibrary(libname): ))(filename_prefix="files/%s_%s" % (s[2], s[1]), errPrefix="files/%s_%s" % (s[2], s[1]), diff=s[3].get("diff")) for s in natsorted(stats, key=lambda s: s[1])]) numSucceeded = [len(stats)] + [sum(1 if s[3]["phase"]>=i else 0 for s in stats) for i in range(1,8)] + numInvalid = sum(1 for s in stats if s[3].get("nonstandard")) gitReferenceFilesVersion = referenceFilesVersion(conf.get("referenceFiles")) @@ -1642,6 +1656,9 @@ def publishLibrary(libname): (u"#referenceFiles#", ('

Reference Files: %s

%s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults%s)

' % (html.escape(ompython_omc_version), html.escape(', tube=%s' % conf["reference_tube"] + (', tolerance=%g' % conf["reference_tolerance"] if conf["reference_tube"] in ("rectangle", "ellipse") and "reference_tolerance" in conf else "")) if conf.get("reference_tube") else "")) if ((conf.get("referenceFiles") or "") != "") else ""), (u"#Total#", html.escape(str(numSucceeded[0]))), + (u"#InvalidClass#", ' class="invalid"' if numInvalid else ""), + (u"#Invalid#", str(numInvalid)), + (u"#invalidNote#", '

A model marked ✘ is invalid: omc translates it only because --warnNonStandardModelica turns the errors of non-standard Modelica into warnings. Hover over the cross to see which --allowNonStandardModelica options it needs.

' if numInvalid else ""), (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), (u"#SimCodeColor#", checkNumSucceeded(numSucceeded, 3)), diff --git a/testmodel.py b/testmodel.py index 9305429..eb99534 100755 --- a/testmodel.py +++ b/testmodel.py @@ -375,6 +375,10 @@ def killChildren(sig, name): with open(errFile, 'a+') as fp: fp.write("Could not %s process: %s.\n" % (name, process.pid)) +def noteNonStandard(messages): + found = set(execstat["nonstandard"]) | set(shared.nonStandardRe.findall(messages or "")) + execstat["nonstandard"] = sorted(found) + def sendExpressionTimeout(omc, cmd, timeout): with open(errFile, 'a+') as fp: fp.write("%s [Timeout %s]\n" % (cmd, timeout)) @@ -382,8 +386,10 @@ def target(res): try: ignore = omc.sendExpression("alarm(%s)" % timeout) res[0] = omc.sendExpression(cmd) + messages = omc.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False) + noteNonStandard(messages) with open(errFile, 'a+') as fp: - fp.write(omc.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False)) + fp.write(messages) elapsed = omc.sendExpression("alarm(0)") with open(errFile, 'a+') as fp: fp.write("[Timeout remaining time %s]\n" % elapsed) @@ -484,6 +490,7 @@ def target(res): "diff":None, "phase":0, "maxrss":0, # bytes resident at once across omc, compilers and executable + "nonstandard":[], # the --allowNonStandardModelica options the translation needed # One entry per runner beyond the first, which reports itself in the keys # above; see configs/fmi-simulators.json, wasm-fmu-runners.json, solvers.json. "simulators":{} @@ -873,6 +880,7 @@ def simulateCmd(resimulate): buildFailed = (simres.get("messages") or "").startswith("Failed to build model") res = True err = omc.sendExpression("OpenModelica.Scripting.getErrorString()") +noteNonStandard(err) total = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_SIMULATE_TOTAL)")-total_before buildmodel = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_BUILD_MODEL)") templates = omc.sendExpression("OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_TEMPLATES)") diff --git a/testresults.md b/testresults.md index d715c5a..c6bd78d 100644 --- a/testresults.md +++ b/testresults.md @@ -73,6 +73,15 @@ The database also records _maxrss_: the most resident memory, in bytes, that the processes testing the model held at any one time - omc, the compilers running beside it and the simulation executable added together. +A model is _invalid_ when it is not standard Modelica: omc translates it only +because the testing sets `--warnNonStandardModelica`, which turns the errors +that `--allowNonStandardModelica` would remove into warnings. The library +reports mark such a model with a red cross; hover over it to see the options it +needs. The overviews take the newest run of `master` on each library as the +judge of which models are invalid (`report.py --invalid-branch` picks another +branch) and leave them out of the counts and times. The buttons at the top of +an overview show them included, or only them. + Clicking on the model name shows the log of phases from parsing to compilation. Clicking on the (sim) link shows the log of the runtime simulation.