diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 59efa30..a5f976c 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -39,6 +39,8 @@ pipeline { choice(name: 'pull_request_solver', choices: ['default', 'gbode', 'cvode', 'ida'], description: 'The solver a pull request is simulated with: default is the one each model asks for, the others are the configs/solvers.json entries of that name. With the C target, the baseline is then that solver\'s table, which the solvers job fills; nothing fills one for wasm-jit, so set pull_request_baseline yourself or compare against the default solver.') string(name: 'pull_request_simflags', defaultValue: '', description: 'Simulation flags appended to every model\'s, after those of pull_request_solver, e.g. -gbm=esdirk3 to run gbode with another method.') string(name: 'pull_request_baseline', defaultValue: '', description: 'The branch a pull request is compared against. Left empty, the one pull_request_target is tested by: wasm-jit for wasm-jit, master for C, or for C with a pull_request_solver its own table. Its newest run has to come from pull_request_node, or the report compares the machines as much as the pull request.') + choice(name: 'pull_request_verifier', choices: ['nightly', 'pull request'], description: 'The omc that test.py and testmodel.py run through OMPython, which lists the models and compares the results with the reference files (diffSimulationResults): the nightly of the testing image, as for every other run, or the omc built from the pull request, to test a change of the comparison itself. A reference_tube in the configuration needs an omc that has the tube argument.') + choice(name: 'pull_request_tube', choices: ['default', 'rectangle', 'ellipse', 'ellipse2014'], description: 'The tube diffSimulationResults compares a pull request\'s results with: default is the reference_tube of each library, the others override it. Anything but default needs pull_request_verifier set to pull request until the nightly has the tube argument.') string(name: 'pull_request_config', defaultValue: 'configs/conf.json', description: 'What a pull request run tests. A full run takes days, so a smaller configuration file is often the better question to ask.') choice(name: 'pull_request_node', choices: ['ryzen-9950x', 'ryzen-5950x-1', 'ryzen-5950x-2-1', 'r7945-1'], description: 'The machine a pull request runs on. The default is the one that produces the master and wasm-jit runs it is compared against.') booleanParam(name: 'pull_request_comment', defaultValue: false, description: 'Post the summary of a pull request run as a comment on the pull request, replacing the one an earlier run posted. Needs an OpenModelica-Hudson credential; without it the report is still written and published, only not commented.') @@ -178,11 +180,14 @@ pipeline { ''' script { def config = params.pull_request_config ?: 'configs/conf.json' - def solverFlags = pullRequestSolver() == 'default' ? '' : "--solverflags=${pullRequestSolver()}" + def testFlags = pullRequestSolver() == 'default' ? '' : "--solverflags=${pullRequestSolver()}" + if ((params.pull_request_tube ?: 'default') != 'default') { + testFlags += " --reference_tube=${params.pull_request_tube}" + } if (pullRequestTarget() == 'C') { - runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", '', '', false, pullRequestSimflags(), solverFlags, false, false, 0, config) + runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", '', '', false, pullRequestSimflags(), testFlags, false, false, 0, config) } else { - runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, pullRequestSimflags(), "--nobuildmodel ${solverFlags}", false, false, 0, config, + runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, pullRequestSimflags(), "--nobuildmodel ${testFlags}", false, false, 0, config, wasmCmakeFlags()) } } @@ -716,6 +721,14 @@ def pullRequest() { return (params.pull_request ?: '').trim() } +// The omc of OMPython for the run `name`; see pull_request_verifier. +def verifierHome(name) { + if (name.startsWith('pr/') && params.pull_request_verifier == 'pull request') { + return '$OPENMODELICAHOME' + } + return '/usr' +} + def pullRequestTarget() { return params.pull_request_target ?: 'wasm-jit' } @@ -1277,7 +1290,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla ${cgroupReport} cd OpenModelicaLibraryTesting # Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions - PYTHONUNBUFFERED=1 stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG}${SOLVER_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 + PYTHONUNBUFFERED=1 stdbuf -oL -eL time ./test.py --ompython_omhome="${verifierHome(name)}" ${FMI_TESTING_FLAG}${SOLVER_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 """) sh 'date' } diff --git a/README.md b/README.md index fad3c4a..bfeb623 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ different OpenModelica versions, according to the conditions of the "git-ref": "main", "git-directory": "ReferenceFiles" }, + "reference_tube": "rectangle", // diffSimulationResults(tube=OpenModelica.Scripting.TubeAlgorithm.): rectangle, ellipse or ellipse2014 (when left out); needs an omc that has the tube argument "defaultTolerance": 1e-6, // tolerance for tests if not specified by the model, defaults to 1e-6 "defaultNumberOfIntervals": 2500, // number of intervals for tests if not specified by the model, defaults to 2500 "ulimitOmc":800, // specify a max timeout for a model build diff --git a/test.py b/test.py index 3128e61..39d23d6 100755 --- a/test.py +++ b/test.py @@ -39,6 +39,7 @@ parser.add_argument('--fmisimulator', action='append', default=[], help="FMI simulator to run the FMUs with, as 'name=command' or just the command. Repeat it to simulate every FMU with several tools without building it more than once; the first one stores its results in --branch and each further one in -, so --branch=master-fmi with OMSimulator and fmpy fills master-fmi and master-fmi-fmpy." ) parser.add_argument('--wasmfmu', '--wasmjitrunner', action='append', dest='wasmfmu', default=[], help="Export every model once as a wasm FMU (buildModelFMU with fmuType=me_cs, platforms={wasm,}, written unzipped) and simulate that one FMU each of these ways: 'me' and 'cs' are its FMI 3.0 interfaces, 'sim' runs the translated model the way simulate() does. Comma-separated or repeated; each one fills -, so --branch=wasm-jit with me,cs fills wasm-jit-me and wasm-jit-cs, beside the wasm-jit a job without this option fills. See configs/wasm-fmu-runners.json. Only for simCodeTarget=wasm-jit.") parser.add_argument('--solver', action='append', default=[], help="Build every model once and simulate it once per solver, so that testing another solver costs a simulation rather than a build. 'default' is the model's own solver and each further name is a -s the simulation is given; comma-separated or repeated. Every solver stores its results in the table its entry names, so --branch=master with default,cvode,gbode fills master, cvode and gbode. See configs/solvers.json.") +parser.add_argument('--reference_tube', default='', help="Compare every library's results with diffSimulationResults(tube=OpenModelica.Scripting.TubeAlgorithm.) (rectangle, ellipse or ellipse2014) instead of its reference_tube; the omc of --ompython_omhome needs the tube argument.") parser.add_argument('--solverflags', default='', help="Simulate every model with the simflags of this configs/solvers.json entry, followed by --extrasimflags so that those can override them. Unlike --solver the results stay in --branch, and it works for every simCodeTarget, wasm-jit included.") parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024) parser.add_argument('--heavyjobs', help="How many of the models listed in --heavymodels may run at the same time. They are the handful that need gigabytes each, and running sixteen of them together is what takes the machine out of memory.", type=int, default=4) @@ -762,6 +763,8 @@ def simulatorKey(libname, runner): conf["noBuildModel"] = True if args.coldhot: conf["coldHot"] = True + if args.reference_tube: + conf["reference_tube"] = args.reference_tube c=conf.copy() del(c["configFromFile"]) if "referenceFiles" in c: @@ -1634,7 +1637,7 @@ def publishLibrary(libname): (u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))), (u"#simFlags#", html.escape(conf.get("simFlags") or "")), (u"#referenceFiles#", ('

Reference Files: %s

%s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), - (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults)

' % html.escape(ompython_omc_version)) if ((conf.get("referenceFiles") or "") != "") else ""), + (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults%s)

' % (html.escape(ompython_omc_version), html.escape(', tube=%s' % conf["reference_tube"]) if conf.get("reference_tube") else "")) if ((conf.get("referenceFiles") or "") != "") else ""), (u"#Total#", html.escape(str(numSucceeded[0]))), (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), diff --git a/testmodel.py b/testmodel.py index 4ccf9c0..2958f63 100755 --- a/testmodel.py +++ b/testmodel.py @@ -1274,8 +1274,10 @@ def verifyAgainstReference(resFile, prefix, stat): start=monotonic() try: # An empty diffPrefix: no per-variable CSV, the two filtered files below are the report. - (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % - (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) + # Only passed when set: an omc without the tube argument fails the call. + tube = conf.get("reference_tube") + (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g%s)' % + (resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], ',tube=OpenModelica.Scripting.TubeAlgorithm.%s' % tube if tube else ""), conf["ulimitOmc"]) except TimeoutError as e: with open(errFile, 'a+') as fp: fp.write("Timeout error for diffSimulationResults")