Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions .CI/Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ pipeline {
choice(name: 'pull_request_solver', choices: ['default', 'gbode', 'cvode', 'ida'], description: 'The solver a pull request is simulated with: default is the one each model asks for, the others are the configs/solvers.json entries of that name. With the C target, the baseline is then that solver\'s table, which the solvers job fills; nothing fills one for wasm-jit, so set pull_request_baseline yourself or compare against the default solver.')
string(name: 'pull_request_simflags', defaultValue: '', description: 'Simulation flags appended to every model\'s, after those of pull_request_solver, e.g. -gbm=esdirk3 to run gbode with another method.')
string(name: 'pull_request_baseline', defaultValue: '', description: 'The branch a pull request is compared against. Left empty, the one pull_request_target is tested by: wasm-jit for wasm-jit, master for C, or for C with a pull_request_solver its own table. Its newest run has to come from pull_request_node, or the report compares the machines as much as the pull request.')
choice(name: 'pull_request_verifier', choices: ['nightly', 'pull request'], description: 'The omc that test.py and testmodel.py run through OMPython, which lists the models and compares the results with the reference files (diffSimulationResults): the nightly of the testing image, as for every other run, or the omc built from the pull request, to test a change of the comparison itself. A reference_tube in the configuration needs an omc that has the tube argument.')
choice(name: 'pull_request_tube', choices: ['default', 'rectangle', 'ellipse', 'ellipse2014'], description: 'The tube diffSimulationResults compares a pull request\'s results with: default is the reference_tube of each library, the others override it. Anything but default needs pull_request_verifier set to pull request until the nightly has the tube argument.')
string(name: 'pull_request_config', defaultValue: 'configs/conf.json', description: 'What a pull request run tests. A full run takes days, so a smaller configuration file is often the better question to ask.')
choice(name: 'pull_request_node', choices: ['ryzen-9950x', 'ryzen-5950x-1', 'ryzen-5950x-2-1', 'r7945-1'], description: 'The machine a pull request runs on. The default is the one that produces the master and wasm-jit runs it is compared against.')
booleanParam(name: 'pull_request_comment', defaultValue: false, description: 'Post the summary of a pull request run as a comment on the pull request, replacing the one an earlier run posted. Needs an OpenModelica-Hudson credential; without it the report is still written and published, only not commented.')
Expand Down Expand Up @@ -178,11 +180,14 @@ pipeline {
'''
script {
def config = params.pull_request_config ?: 'configs/conf.json'
def solverFlags = pullRequestSolver() == 'default' ? '' : "--solverflags=${pullRequestSolver()}"
def testFlags = pullRequestSolver() == 'default' ? '' : "--solverflags=${pullRequestSolver()}"
if ((params.pull_request_tube ?: 'default') != 'default') {
testFlags += " --reference_tube=${params.pull_request_tube}"
}
if (pullRequestTarget() == 'C') {
runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", '', '', false, pullRequestSimflags(), solverFlags, false, false, 0, config)
runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", '', '', false, pullRequestSimflags(), testFlags, false, false, 0, config)
} else {
runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, pullRequestSimflags(), "--nobuildmodel ${solverFlags}", false, false, 0, config,
runRegressiontest("pr/${pullRequest()}", "pr/${pullRequest()}", 'setCommandLineOptions("--simCodeTarget=wasm-jit")', '', false, pullRequestSimflags(), "--nobuildmodel ${testFlags}", false, false, 0, config,
wasmCmakeFlags())
}
}
Expand Down Expand Up @@ -716,6 +721,14 @@ def pullRequest() {
return (params.pull_request ?: '').trim()
}

// The omc of OMPython for the run `name`; see pull_request_verifier.
def verifierHome(name) {
if (name.startsWith('pr/') && params.pull_request_verifier == 'pull request') {
return '$OPENMODELICAHOME'
}
return '/usr'
}

def pullRequestTarget() {
return params.pull_request_target ?: 'wasm-jit'
}
Expand Down Expand Up @@ -1277,7 +1290,7 @@ def runRegressiontest(branch, name, extraFlags, omsHash, omcompiler, extrasimfla
${cgroupReport}
cd OpenModelicaLibraryTesting
# Force /usr/bin/omc as being used for generating the mos-files. Ensures consistent behavior among all tested OMC versions
PYTHONUNBUFFERED=1 stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG}${SOLVER_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1
PYTHONUNBUFFERED=1 stdbuf -oL -eL time ./test.py --ompython_omhome="${verifierHome(name)}" ${FMI_TESTING_FLAG}${SOLVER_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1
""")
sh 'date'
}
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ different OpenModelica versions, according to the conditions of the
"git-ref": "main",
"git-directory": "ReferenceFiles"
},
"reference_tube": "rectangle", // diffSimulationResults(tube=OpenModelica.Scripting.TubeAlgorithm.<this>): rectangle, ellipse or ellipse2014 (when left out); needs an omc that has the tube argument
"defaultTolerance": 1e-6, // tolerance for tests if not specified by the model, defaults to 1e-6
"defaultNumberOfIntervals": 2500, // number of intervals for tests if not specified by the model, defaults to 2500
"ulimitOmc":800, // specify a max timeout for a model build
Expand Down
5 changes: 4 additions & 1 deletion test.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
parser.add_argument('--fmisimulator', action='append', default=[], help="FMI simulator to run the FMUs with, as 'name=command' or just the command. Repeat it to simulate every FMU with several tools without building it more than once; the first one stores its results in --branch and each further one in <branch>-<name>, so --branch=master-fmi with OMSimulator and fmpy fills master-fmi and master-fmi-fmpy." )
parser.add_argument('--wasmfmu', '--wasmjitrunner', action='append', dest='wasmfmu', default=[], help="Export every model once as a wasm FMU (buildModelFMU with fmuType=me_cs, platforms={wasm,<this machine>}, written unzipped) and simulate that one FMU each of these ways: 'me' and 'cs' are its FMI 3.0 interfaces, 'sim' runs the translated model the way simulate() does. Comma-separated or repeated; each one fills <branch>-<name>, so --branch=wasm-jit with me,cs fills wasm-jit-me and wasm-jit-cs, beside the wasm-jit a job without this option fills. See configs/wasm-fmu-runners.json. Only for simCodeTarget=wasm-jit.")
parser.add_argument('--solver', action='append', default=[], help="Build every model once and simulate it once per solver, so that testing another solver costs a simulation rather than a build. 'default' is the model's own solver and each further name is a -s the simulation is given; comma-separated or repeated. Every solver stores its results in the table its entry names, so --branch=master with default,cvode,gbode fills master, cvode and gbode. See configs/solvers.json.")
parser.add_argument('--reference_tube', default='', help="Compare every library's results with diffSimulationResults(tube=OpenModelica.Scripting.TubeAlgorithm.<this>) (rectangle, ellipse or ellipse2014) instead of its reference_tube; the omc of --ompython_omhome needs the tube argument.")
parser.add_argument('--solverflags', default='', help="Simulate every model with the simflags of this configs/solvers.json entry, followed by --extrasimflags so that those can override them. Unlike --solver the results stay in --branch, and it works for every simCodeTarget, wasm-jit included.")
parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024)
parser.add_argument('--heavyjobs', help="How many of the models listed in --heavymodels may run at the same time. They are the handful that need gigabytes each, and running sixteen of them together is what takes the machine out of memory.", type=int, default=4)
Expand Down Expand Up @@ -762,6 +763,8 @@ def simulatorKey(libname, runner):
conf["noBuildModel"] = True
if args.coldhot:
conf["coldHot"] = True
if args.reference_tube:
conf["reference_tube"] = args.reference_tube
c=conf.copy()
del(c["configFromFile"])
if "referenceFiles" in c:
Expand Down Expand Up @@ -1634,7 +1637,7 @@ def publishLibrary(libname):
(u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))),
(u"#simFlags#", html.escape(conf.get("simFlags") or "")),
(u"#referenceFiles#", ('<p>Reference Files: %s</p>%s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""),
(u"#referenceTool#", ('<p>Verified using: %s (diffSimulationResults)</p>' % html.escape(ompython_omc_version)) if ((conf.get("referenceFiles") or "") != "") else ""),
(u"#referenceTool#", ('<p>Verified using: %s (diffSimulationResults%s)</p>' % (html.escape(ompython_omc_version), html.escape(', tube=%s' % conf["reference_tube"]) if conf.get("reference_tube") else "")) if ((conf.get("referenceFiles") or "") != "") else ""),
(u"#Total#", html.escape(str(numSucceeded[0]))),
(u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)),
(u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)),
Expand Down
6 changes: 4 additions & 2 deletions testmodel.py
Original file line number Diff line number Diff line change
Expand Up @@ -1274,8 +1274,10 @@ def verifyAgainstReference(resFile, prefix, stat):
start=monotonic()
try:
# An empty diffPrefix: no per-variable CSV, the two filtered files below are the report.
(referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' %
(resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"])
# Only passed when set: an omc without the tube argument fails the call.
tube = conf.get("reference_tube")
(referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g%s)' %
(resFile, referenceFile, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], ',tube=OpenModelica.Scripting.TubeAlgorithm.%s' % tube if tube else ""), conf["ulimitOmc"])
except TimeoutError as e:
with open(errFile, 'a+') as fp:
fp.write("Timeout error for diffSimulationResults")
Expand Down
Loading