From 63bb0442975ca2ded33b4d996f09546c920029b0 Mon Sep 17 00:00:00 2001 From: yadavchiragg Date: Mon, 16 Nov 2026 07:45:14 +0530 Subject: [PATCH] fix(packaging): ship mkconcore's runtime files and modules in the wheel, add a package CI job A regular pip install of the built wheel contained mkconcore.py but not copy_with_port_portname.py or any of the runtime files mkconcore copies into a study (concore.hpp, Dockerfile.*, .m/.v/.java/.jl helpers, mkcompile), so 'concore build' failed with ModuleNotFoundError. Add the missing module, install the runtime files next to mkconcore.py via a build_py step, include them in the sdist via MANIFEST.in, and add a CI job that builds the sdist and wheel, checks them with twine, installs the wheel in a clean venv, verifies every file mkconcore.py references is present, and builds studies with the installed command. --- .github/workflows/ci.yml | 53 ++++++++++++++++++++++++++++++++++++++ MANIFEST.in | 10 ++++++++ pyproject.toml | 8 +++++- setup.py | 55 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 MANIFEST.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb47fa00..f74d4d6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -240,3 +240,56 @@ jobs: ./TestLiteralEvalCpp g++ -std=c++11 -I. -o TestConcoreHpp TestConcoreHpp.cpp ./TestConcoreHpp + + package: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Build sdist and wheel + run: | + pip install build twine + python -m build + twine check dist/* + + - name: Install the wheel into a clean virtual environment + run: | + python -m venv "$RUNNER_TEMP/venv" + "$RUNNER_TEMP/venv/bin/pip" install dist/*.whl + + - name: Check mkconcore's runtime files are in the installed package + run: | + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/venv/bin/python" - <<'PY' + import importlib.util, os, re, sys + + repo = os.environ["GITHUB_WORKSPACE"] + installed = os.path.dirname(importlib.util.find_spec("mkconcore").origin) + source = open(os.path.join(installed, "mkconcore.py")).read() + names = set(re.findall(r'CONCOREPATH\s*\+\s*"/([^"]+)"', source)) + names |= set(re.findall(r'os\.path\.join\(CONCOREPATH,\s*"([^"]+)"', source)) + # Only files that exist in the repo are expected in the package. + missing = sorted( + n for n in names + if os.path.exists(os.path.join(repo, n)) + and not os.path.exists(os.path.join(installed, n)) + ) + if missing: + sys.exit("Referenced by mkconcore.py but missing from the wheel: " + ", ".join(missing)) + print(f"OK: {len(names)} files referenced, none missing from the wheel") + PY + + - name: Build studies with the installed concore command + run: | + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/venv/bin/concore" init proj + "$RUNNER_TEMP/venv/bin/concore" build proj/workflow.graphml --source proj/src --output posix-out --type posix + "$RUNNER_TEMP/venv/bin/concore" build proj/workflow.graphml --source proj/src --output docker-out --type docker + "$RUNNER_TEMP/venv/bin/concore" build "$GITHUB_WORKSPACE/testsou/testSM1.graphml" --source "$GITHUB_WORKSPACE/testsou" --output cpp-out --type posix + test -f cpp-out/src/concore.hpp + test -f docker-out/src/Dockerfile.script diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..b114f576 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,10 @@ +# Runtime files mkconcore.py copies into generated studies (see setup.py). +include concore*.hpp concore*.v concore*.java ConcoreJavaRuntimeCore.java +include concore*.jl +include concore_default_maxtime.m concore_initval.m concore_iport.m +include concore_literal_eval.m concore_literal_serialize.m concore_oport.m +include concore_read.m concore_unchanged.m concore_write.m +include import_concore.m import_concoredocker.m +include Dockerfile.* mkcompile +include concore.octave concore.mcr concore.sudo concore.repo concore.tools +include tools/extract_graphml.py diff --git a/pyproject.toml b/pyproject.toml index 94cb9154..a5d8f6e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,13 @@ concore = "concore_cli.cli:cli" [tool.setuptools] packages = ["concore_cli", "concore_cli.commands"] -py-modules = ["concore", "concoredocker", "concore_base", "mkconcore"] +py-modules = [ + "concore", + "concoredocker", + "concore_base", + "mkconcore", + "copy_with_port_portname", +] [tool.setuptools.dynamic] version = {attr = "concore_cli.__version__"} diff --git a/setup.py b/setup.py index ed9acf30..08795551 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,54 @@ +import glob +import os + from setuptools import setup +from setuptools.command.build_py import build_py + +# Files mkconcore.py reads from next to itself (CONCOREPATH) and copies into +# every generated study. They are not Python modules, so setuptools would +# otherwise leave them out of the wheel and `concore build` could not work +# from a regular (non-editable) install. Keep MANIFEST.in in sync. +RUNTIME_FILE_PATTERNS = [ + "concore*.hpp", + "concore*.v", + "concore*.java", + "ConcoreJavaRuntimeCore.java", + "concore*.jl", + "concore_default_maxtime.m", + "concore_initval.m", + "concore_iport.m", + "concore_literal_eval.m", + "concore_literal_serialize.m", + "concore_oport.m", + "concore_read.m", + "concore_unchanged.m", + "concore_write.m", + "import_concore.m", + "import_concoredocker.m", + "Dockerfile.*", + "mkcompile", + "concore.octave", + "concore.mcr", + "concore.sudo", + "concore.repo", + "concore.tools", + "tools/extract_graphml.py", +] + + +class build_py_with_runtime_files(build_py): + """Install the runtime files next to mkconcore.py in the built package.""" + + def run(self): + super().run() + if self.dry_run: + return + for pattern in RUNTIME_FILE_PATTERNS: + for src in sorted(glob.glob(pattern)): + dest = os.path.join(self.build_lib, src) + self.mkpath(os.path.dirname(dest)) + self.copy_file(src, dest) + -# All metadata and configuration is in pyproject.toml. -# This file exists only for legacy compatibility. -setup() +# All other metadata and configuration is in pyproject.toml. +setup(cmdclass={"build_py": build_py_with_runtime_files})