diff --git a/README.md b/README.md index bfeb623..c4d5343 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ different OpenModelica versions, according to the conditions of the "git-directory": "ReferenceFiles" }, "reference_tube": "rectangle", // diffSimulationResults(tube=OpenModelica.Scripting.TubeAlgorithm.): rectangle, ellipse or ellipse2014 (when left out); needs an omc that has the tube argument + "reference_tolerance": 0.002, // rangeDelta of the rectangle and ellipse tubes (csv-compare's --tolerance); when left out, the default of diffSimulationResults, which is csv-compare's; ellipse2014 uses reference_rangeDelta "defaultTolerance": 1e-6, // tolerance for tests if not specified by the model, defaults to 1e-6 "defaultNumberOfIntervals": 2500, // number of intervals for tests if not specified by the model, defaults to 2500 "ulimitOmc":800, // specify a max timeout for a model build diff --git a/test.py b/test.py index 39d23d6..e21412d 100755 --- a/test.py +++ b/test.py @@ -40,6 +40,7 @@ parser.add_argument('--wasmfmu', '--wasmjitrunner', action='append', dest='wasmfmu', default=[], help="Export every model once as a wasm FMU (buildModelFMU with fmuType=me_cs, platforms={wasm,}, written unzipped) and simulate that one FMU each of these ways: 'me' and 'cs' are its FMI 3.0 interfaces, 'sim' runs the translated model the way simulate() does. Comma-separated or repeated; each one fills -, so --branch=wasm-jit with me,cs fills wasm-jit-me and wasm-jit-cs, beside the wasm-jit a job without this option fills. See configs/wasm-fmu-runners.json. Only for simCodeTarget=wasm-jit.") parser.add_argument('--solver', action='append', default=[], help="Build every model once and simulate it once per solver, so that testing another solver costs a simulation rather than a build. 'default' is the model's own solver and each further name is a -s the simulation is given; comma-separated or repeated. Every solver stores its results in the table its entry names, so --branch=master with default,cvode,gbode fills master, cvode and gbode. See configs/solvers.json.") parser.add_argument('--reference_tube', default='', help="Compare every library's results with diffSimulationResults(tube=OpenModelica.Scripting.TubeAlgorithm.) (rectangle, ellipse or ellipse2014) instead of its reference_tube; the omc of --ompython_omhome needs the tube argument.") +parser.add_argument('--reference_tolerance', type=float, default=None, help="The tolerance (rangeDelta) of the rectangle and ellipse tubes for every library instead of its reference_tolerance. Without either, diffSimulationResults uses its default, which is that of csv-compare.") parser.add_argument('--solverflags', default='', help="Simulate every model with the simflags of this configs/solvers.json entry, followed by --extrasimflags so that those can override them. Unlike --solver the results stay in --branch, and it works for every simCodeTarget, wasm-jit included.") parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024) parser.add_argument('--heavyjobs', help="How many of the models listed in --heavymodels may run at the same time. They are the handful that need gigabytes each, and running sixteen of them together is what takes the machine out of memory.", type=int, default=4) @@ -765,6 +766,8 @@ def simulatorKey(libname, runner): conf["coldHot"] = True if args.reference_tube: conf["reference_tube"] = args.reference_tube + if args.reference_tolerance is not None: + conf["reference_tolerance"] = args.reference_tolerance c=conf.copy() del(c["configFromFile"]) if "referenceFiles" in c: @@ -1637,7 +1640,7 @@ def publishLibrary(libname): (u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))), (u"#simFlags#", html.escape(conf.get("simFlags") or "")), (u"#referenceFiles#", ('

Reference Files: %s

%s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), - (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults%s)

' % (html.escape(ompython_omc_version), html.escape(', tube=%s' % conf["reference_tube"]) if conf.get("reference_tube") else "")) if ((conf.get("referenceFiles") or "") != "") else ""), + (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults%s)

' % (html.escape(ompython_omc_version), html.escape(', tube=%s' % conf["reference_tube"] + (', tolerance=%g' % conf["reference_tolerance"] if conf["reference_tube"] in ("rectangle", "ellipse") and "reference_tolerance" in conf else "")) if conf.get("reference_tube") else "")) if ((conf.get("referenceFiles") or "") != "") else ""), (u"#Total#", html.escape(str(numSucceeded[0]))), (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), diff --git a/testmodel.py b/testmodel.py index 2958f63..7005d91 100755 --- a/testmodel.py +++ b/testmodel.py @@ -1276,8 +1276,13 @@ def verifyAgainstReference(resFile, prefix, stat): # An empty diffPrefix: no per-variable CSV, the two filtered files below are the report. # Only passed when set: an omc without the tube argument fails the call. tube = conf.get("reference_tube") - (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g%s)' % - (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], ',tube=OpenModelica.Scripting.TubeAlgorithm.%s' % tube if tube else ""), conf["ulimitOmc"]) + # Without reference_tolerance, the csv-compare tubes get the default of diffSimulationResults, which is csv-compare's. + if tube in ("rectangle", "ellipse"): + rangeDelta = ',rangeDelta=%g' % conf["reference_tolerance"] if "reference_tolerance" in conf else "" + else: + rangeDelta = ',rangeDelta=%g' % conf["reference_rangeDelta"] + (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g%s%s)' % + (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], rangeDelta, ',tube=OpenModelica.Scripting.TubeAlgorithm.%s' % tube if tube else ""), conf["ulimitOmc"]) except TimeoutError as e: with open(errFile, 'a+') as fp: fp.write("Timeout error for diffSimulationResults")